From fd83175f546ba85643a68ebb9d5e1abf4c60140b Mon Sep 17 00:00:00 2001 From: Joe Guo Date: Wed, 4 Sep 2019 13:51:59 +1200 Subject: [PATCH 001/898] charm_tests/glance: migrate Amulet tests to Zaza This patch is the peer change for: https://review.opendev.org/#/c/679599/ charm-glance currently has following Amulet tests: https://github.com/openstack/charm-glance/blob/master/tests/basic_deployment.py The 1xx, 2xx and 5xx ones are mostly covered in zaza keystone tests, or could be skipped in zaza. Add the only missing 900 one, to test glance restart on config change. Test option `debug` instead of `use-syslog` to avoid conversion between `-` and `_`, which is out the scope of this test. Signed-off-by: Joe Guo --- zaza/openstack/charm_tests/glance/tests.py | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index 001f17f..4f25d81 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -67,6 +67,44 @@ class GlanceTest(test_utils.OpenStackBaseTest): {'image_format': {'disk_formats': ['qcow2']}}, ['glance-api']) + def test_900_restart_on_config_change(self): + """Checking restart happens on config change.""" + # Expected default and alternate values + current_value = openstack_utils.get_application_config_option( + 'glance', 'debug') + # this is bool, not str + assert type(current_value) == bool + new_value = not current_value + + # convert bool to str + current_value = str(current_value) + new_value = str(new_value) + + set_default = {'debug': current_value} + set_alternate = {'debug': new_value} + default_entry = {'DEFAULT': {'debug': [current_value]}} + alternate_entry = {'DEFAULT': {'debug': [new_value]}} + + # Config file affected by juju set config change + conf_file = '/etc/glance/glance-api.conf' + + # Services which are expected to restart upon config change + services = {'glance-api': conf_file} + current_release = openstack_utils.get_os_release() + bionic_stein = openstack_utils.get_os_release('bionic_stein') + if current_release < bionic_stein: + services.update({'glance-registry': conf_file}) + + # Make config change, check for service restarts + logging.info('changing config: {}'.format(set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + services) + def test_901_pause_resume(self): """Run pause and resume tests. From 2888e62e3af43c6a71638535f9f44a136f3e13be Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 4 Sep 2019 11:42:46 +0200 Subject: [PATCH 002/898] Add ceph pg tuning test --- zaza/openstack/charm_tests/ceph/tests.py | 21 ++++++++++++++++++--- zaza/openstack/charm_tests/test_utils.py | 21 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index f8bd0c1..f9088e7 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -15,6 +15,7 @@ """Ceph Testing.""" import unittest +import json import logging from os import ( listdir, @@ -56,7 +57,7 @@ class CephLowLevelTest(test_utils.OpenStackBaseTest): } ceph_osd_processes = { - 'ceph-osd': [2, 3] + 'ceph-osd': [1, 2, 3] } # Units with process names and PID quantities expected @@ -95,6 +96,16 @@ class CephLowLevelTest(test_utils.OpenStackBaseTest): target_status='running' ) + @test_utils.skipUntilVersion('ceph-mon', 'ceph', '14.2.0') + def test_pg_tuning(self): + """Verify that auto PG tuning is enabled for Nautilus+.""" + unit_name = 'ceph-mon/0' + cmd = "ceph osd pool autoscale-status --format=json" + result = zaza_model.run_on_unit(unit_name, cmd) + self.assertEqual(result['Code'], '0') + for pool in json.loads(result['Stdout']): + self.assertEqual(pool['pg_autoscale_mode'], 'on') + class CephRelationTest(test_utils.OpenStackBaseTest): """Ceph's relations test class.""" @@ -138,7 +149,6 @@ class CephRelationTest(test_utils.OpenStackBaseTest): fsid = result.get('Stdout').strip() expected = { 'private-address': remote_ip, - 'auth': 'none', 'ceph-public-address': remote_ip, 'fsid': fsid, } @@ -408,9 +418,14 @@ class CephTest(test_utils.OpenStackBaseTest): set_default = { 'ephemeral-unmount': '', - 'osd-devices': '/dev/vdb /srv/ceph', + 'osd-devices': '/dev/vdb', } + current_release = zaza_openstack.get_os_release() + bionic_train = zaza_openstack.get_os_release('bionic_train') + if current_release < bionic_train: + set_default['osd-devices'] = '/dev/vdb /srv/ceph' + logging.info('Restoring to default configuration...') zaza_model.set_application_config(juju_service, set_default) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 88687c3..e054038 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -14,12 +14,14 @@ """Module containg base class for implementing charm tests.""" import contextlib import logging +import subprocess import unittest import zaza.model import zaza.model as model import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.utilities.generic as generic_utils def skipIfNotHA(service_name): @@ -38,6 +40,25 @@ def skipIfNotHA(service_name): return _skipIfNotHA_inner_1 +def skipUntilVersion(service, package, release): + """Run decorator to skip this test if application version is too low.""" + def _skipUntilVersion_inner_1(f): + def _skipUntilVersion_inner_2(*args, **kwargs): + package_version = generic_utils.get_pkg_version(service, package) + try: + subprocess.check_call(['dpkg', '--compare-versions', + package_version, 'ge', release], + stderr=subprocess.STDOUT, + universal_newlines=True) + return f(*args, **kwargs) + except subprocess.CalledProcessError as cp: + logging.warn("Skipping test for older ({})" + "service {}, requested {}".format( + package_version, service, release)) + return _skipUntilVersion_inner_2 + return _skipUntilVersion_inner_1 + + def audit_assertions(action, expected_passes, expected_failures=None, From 28a08cf77a1dfc20e0169527d4dac714d4e869a4 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 5 Sep 2019 16:39:04 +0200 Subject: [PATCH 003/898] Add quagga tests --- zaza/openstack/charm_tests/quagga/__init__.py | 15 +++++ zaza/openstack/charm_tests/quagga/tests.py | 63 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 zaza/openstack/charm_tests/quagga/__init__.py create mode 100644 zaza/openstack/charm_tests/quagga/tests.py diff --git a/zaza/openstack/charm_tests/quagga/__init__.py b/zaza/openstack/charm_tests/quagga/__init__.py new file mode 100644 index 0000000..e67cd6a --- /dev/null +++ b/zaza/openstack/charm_tests/quagga/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for testing quagga charm.""" diff --git a/zaza/openstack/charm_tests/quagga/tests.py b/zaza/openstack/charm_tests/quagga/tests.py new file mode 100644 index 0000000..67a49d6 --- /dev/null +++ b/zaza/openstack/charm_tests/quagga/tests.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulating `quagga` testing.""" + +import logging +import re +import unittest + +import zaza + + +class QuaggaTest(unittest.TestCase): + """Class for `quagga` tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for `quagga` tests.""" + super(QuaggaTest, cls).setUpClass() + + def test_bgp_peer_datapath(self): + """Get peers from BGP neighbour list and ping them.""" + status = zaza.model.get_status() + applications = (app for app in ['spine0', 'spine1', 'tor0', 'tor1', + 'tor2', 'peer0', 'peer1'] + if app in status.applications.keys()) + for application in applications: + for unit in zaza.model.get_units(application): + bgp_sum = zaza.model.run_on_unit( + unit.entity_id, + 'echo "sh bgp ipv4 unicast summary" | vtysh')['Stdout'] + r = re.compile('^(\d+\.\d+\.\d+\.\d+)') + ip_list = [] + for line in bgp_sum.splitlines(): + m = r.match(line) + if m: + ip_list.append(m.group(1)) + logging.info('unit {} neighbours {}' + .format(unit.entity_id, ip_list)) + + if not ip_list: + raise Exception('FAILED: Unit {} has no BGP peers.' + .format(unit.entity_id)) + for ip in ip_list: + result = zaza.model.run_on_unit( + unit.entity_id, + 'ping -c 3 {}'.format(ip)) + logging.info(result['Stdout']) + if result['Code'] == '1': + raise Exception('FAILED') From caed1ceeb3fdd099a56f02bd5997364f8ca091bc Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 10 Sep 2019 12:14:52 +0200 Subject: [PATCH 004/898] Use UTC when calculating certificate validity The current usage of ``datetime.today()`` is at the grace of the locale settings of the test executor and the instances spun up. The ``cryptography.x509.CertificateBuilder`` ``not_valud_before`` and ``not_valid_after`` attributes do expect [0] a UTC datetime. 0: https://cryptography.io/en/latest/x509/reference/#cryptography.x509.CertificateBuilder --- zaza/openstack/utilities/cert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/cert.py b/zaza/openstack/utilities/cert.py index 9cb0c19..180f796 100644 --- a/zaza/openstack/utilities/cert.py +++ b/zaza/openstack/utilities/cert.py @@ -94,10 +94,10 @@ def generate_cert(common_name, cryptography.x509.oid.NameOID.COMMON_NAME, issuer_name), ])) builder = builder.not_valid_before( - datetime.datetime.today() - datetime.timedelta(0, 1, 0), + datetime.datetime.utcnow() - datetime.timedelta(0, 1, 0), ) builder = builder.not_valid_after( - datetime.datetime.today() + datetime.timedelta(30, 0, 0), + datetime.datetime.utcnow() + datetime.timedelta(30, 0, 0), ) builder = builder.serial_number(cryptography.x509.random_serial_number()) builder = builder.public_key(public_key) From 63e9b4cf1b57d537daf82847ff90570c3178d905 Mon Sep 17 00:00:00 2001 From: Joe Guo Date: Fri, 6 Sep 2019 14:48:40 +1200 Subject: [PATCH 005/898] Migrate glance-simplestreams-sync tests from Amulet to Zaza Migrate tests from Amulet to Zaza in: charm-glance-simplestreams-sync/tests/basic_deployment.py Signed-off-by: Joe Guo --- .../glance_simplestreams_sync/__init__.py | 16 +++ .../glance_simplestreams_sync/tests.py | 102 ++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 zaza/openstack/charm_tests/glance_simplestreams_sync/__init__.py create mode 100644 zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/__init__.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/__init__.py new file mode 100644 index 0000000..7667e52 --- /dev/null +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/__init__.py @@ -0,0 +1,16 @@ + +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing glance-simplestreams-sync.""" diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py new file mode 100644 index 0000000..6ddb1b3 --- /dev/null +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py @@ -0,0 +1,102 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate glance-simplestreams-sync testing.""" +import json +import logging +import requests +import tenacity + +import zaza.model as zaza_model +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +@tenacity.retry( + retry=tenacity.retry_if_result(lambda images: len(images) == 0), + wait=tenacity.wait_fixed(6), # interval between retries + stop=tenacity.stop_after_attempt(100)) # retry times +def retry_image_sync(glance_client): + """Wait for image sync with retry.""" + # convert generator to list + return list(glance_client.images.list()) + + +class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): + """Glance Simple Streams Sync Test.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running glance simple streams sync tests.""" + super(GlanceSimpleStreamsSyncTest, cls).setUpClass() + # dict of OS_* env vars + overcloud_auth = openstack_utils.get_overcloud_auth() + cls.keystone_client = openstack_utils.get_keystone_client( + overcloud_auth) + cls.glance_client = openstack_utils.get_glance_session_client( + cls.keystone_session) + + def test_010_wait_for_image_sync(self): + """Wait for images to be synced. Expect at least one.""" + self.assertTrue(retry_image_sync(self.glance_client)) + + def test_050_gss_permissions_regression_check_lp1611987(self): + """Assert the intended file permissions on gss config files. + + refer: https://bugs.launchpad.net/bugs/1611987 + """ + file_paths = [ + '/etc/glance-simplestreams-sync/identity.yaml', + '/etc/glance-simplestreams-sync/mirrors.yaml', + '/var/log/glance-simplestreams-sync.log', + ] + expected_perms = '640' + + application = 'glance-simplestreams-sync' + for unit in zaza_model.get_units(application): + for file_path in file_paths: + cmd = 'stat -c %a {}'.format(file_path) + result = zaza_model.run_on_unit(unit.name, cmd, timeout=30) + # {'Code': '', 'Stderr': '', 'Stdout': '644\n'} + perms = result.get('Stdout', '').strip() + self.assertEqual(perms, expected_perms) + logging.debug( + 'Permissions on {}: {}'.format(file_path, perms)) + + def test_110_local_product_stream(self): + """Verify that the local product stream is accessible and has data.""" + logging.debug('Checking local product streams...') + expected_images = [ + 'com.ubuntu.cloud:server:14.04:amd64', + 'com.ubuntu.cloud:server:16.04:amd64', + 'com.ubuntu.cloud:server:18.04:amd64', + ] + uri = "streams/v1/auto.sync.json" + key = "url" + xenial_pike = openstack_utils.get_os_release('xenial_pike') + if openstack_utils.get_os_release() <= xenial_pike: + key = "publicURL" + + catalog = self.keystone_client.service_catalog.get_endpoints() + ps_interface = catalog["product-streams"][0][key] + url = "{}/{}".format(ps_interface, uri) + client = requests.session() + json_data = client.get(url).text + product_streams = json.loads(json_data) + images = product_streams["products"] + + for image in expected_images: + self.assertIn(image, images) + + logging.debug("Local product stream successful") From 6cab1f8b85cfd0784b7b07feaafec48480ab4fc3 Mon Sep 17 00:00:00 2001 From: Eduardo Sousa Date: Wed, 18 Sep 2019 16:09:08 +0100 Subject: [PATCH 006/898] Porting Ceilometer tests to Zaza Porting the Amulet tests from Ceilometer to the Zaza framework. The Amulet tests can be found here: https://opendev.org/openstack/charm-ceilometer/src/branch/master/tests/basic_deployment.py --- .../charm_tests/ceilometer/__init__.py | 17 +++ .../openstack/charm_tests/ceilometer/setup.py | 43 +++++++ .../openstack/charm_tests/ceilometer/tests.py | 118 ++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 zaza/openstack/charm_tests/ceilometer/__init__.py create mode 100644 zaza/openstack/charm_tests/ceilometer/setup.py create mode 100644 zaza/openstack/charm_tests/ceilometer/tests.py diff --git a/zaza/openstack/charm_tests/ceilometer/__init__.py b/zaza/openstack/charm_tests/ceilometer/__init__.py new file mode 100644 index 0000000..106535e --- /dev/null +++ b/zaza/openstack/charm_tests/ceilometer/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ceilometer.""" diff --git a/zaza/openstack/charm_tests/ceilometer/setup.py b/zaza/openstack/charm_tests/ceilometer/setup.py new file mode 100644 index 0000000..0051624 --- /dev/null +++ b/zaza/openstack/charm_tests/ceilometer/setup.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for configuring Ceilometer.""" + +import logging +import zaza.model as zaza_model +import zaza.openstack.utilities.openstack as openstack_utils + + +def basic_setup(unit='ceilometer/0'): + """Run setup for testing Ceilometer. + + Setup for testing Ceilometer is currently part of functional + tests. + """ + current_release = openstack_utils.get_os_release() + xenial_pike = openstack_utils.get_os_release('xenial_pike') + + if current_release < xenial_pike: + logging.debug('Not checking ceilometer-upgrade') + return + logging.debug('Checking ceilometer-upgrade') + + action = zaza_model.run_action( + unit, + 'ceilometer-upgrade', + raise_on_failure=True) + + return action diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py new file mode 100644 index 0000000..f57d4af --- /dev/null +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Ceilometer testing.""" + +import logging + +import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.test_utils as test_utils + + +class CeilometerTest(test_utils.OpenStackBaseTest): + """Encapsulate Ceilometer tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Ceilometer tests.""" + super(CeilometerTest, cls).setUpClass() + + # NOTE(beisner): need to add more functional tests + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change.""" + # Expected default and alternate values + current_value = openstack_utils.get_application_config_option( + 'ceilometer', 'debug' + ) + assert type(current_value) == bool + new_value = not current_value + + # Convert bool to str + current_value = str(current_value) + new_value = str(new_value) + + set_default = {'debug': current_value} + set_alternate = {'debug': new_value} + default_entry = {'DEFAULT': {'debug': [current_value]}} + alternate_entry = {'DEFAULT': {'debug': [new_value]}} + + # Config file affected by juju set config change + conf_file = '/etc/ceilometer/ceilometer.conf' + services = {} + current_release = openstack_utils.get_os_release() + xenial_pike = openstack_utils.get_os_release('xenial_pike') + xenial_ocata = openstack_utils.get_os_release('xenial_ocata') + xenial_newton = openstack_utils.get_os_release('xenial_newton') + trusty_mitaka = openstack_utils.get_os_release('trusty_mitaka') + trusty_liberty = openstack_utils.get_os_release('trusty_liberty') + + if current_release >= xenial_pike: + services = { + 'ceilometer-polling: AgentManager worker(0)': conf_file, + 'ceilometer-agent-notification: NotificationService worker(0)': + conf_file, + } + elif current_release >= xenial_ocata: + services = { + 'ceilometer-collector: CollectorService worker(0)': conf_file, + 'ceilometer-polling: AgentManager worker(0)': conf_file, + 'ceilometer-agent-notification: NotificationService worker(0)': + conf_file, + 'apache2': conf_file, + } + elif current_release >= xenial_newton: + services = { + 'ceilometer-collector - CollectorService(0)': conf_file, + 'ceilometer-polling - AgentManager(0)': conf_file, + 'ceilometer-agent-notification - NotificationService(0)': + conf_file, + 'ceilometer-api': conf_file, + } + else: + services = { + 'ceilometer-collector': conf_file, + 'ceilometer-api': conf_file, + 'ceilometer-agent-notification': conf_file, + } + + if current_release < trusty_mitaka: + services['ceilometer-alarm-notifier'] = conf_file + services['ceilometer-alarm-evaluator'] = conf_file + + if current_release >= trusty_liberty: + # Liberty and later + services['ceilometer-polling'] = conf_file + else: + # Juno and earlier + services['ceilometer-agent-central'] = conf_file + + logging.info('changing config: {}'.format(set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + services) + + def test_901_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started. + """ + self.pause_resume(['ceilometer']) From 274b644c95c8f9d5c6ffc0514baf35c33b8335f1 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 20 Sep 2019 19:09:02 +0100 Subject: [PATCH 007/898] First cut --- zaza/openstack/charm_tests/aodh/.tests.py.swp | Bin 0 -> 16384 bytes zaza/openstack/charm_tests/aodh/__init__.py | 15 +++ zaza/openstack/charm_tests/aodh/setup.py | 22 ++++ zaza/openstack/charm_tests/aodh/tests.py | 101 ++++++++++++++++++ zaza/openstack/configure/telemetry.py | 31 ++++++ 5 files changed, 169 insertions(+) create mode 100644 zaza/openstack/charm_tests/aodh/.tests.py.swp create mode 100644 zaza/openstack/charm_tests/aodh/__init__.py create mode 100644 zaza/openstack/charm_tests/aodh/setup.py create mode 100644 zaza/openstack/charm_tests/aodh/tests.py create mode 100644 zaza/openstack/configure/telemetry.py diff --git a/zaza/openstack/charm_tests/aodh/.tests.py.swp b/zaza/openstack/charm_tests/aodh/.tests.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..431d6bca4e2bc867994a800908f7a9d92ed31557 GIT binary patch literal 16384 zcmeHN&5t8T6)yq_A-l_G5L}Qd_Q9TLY|qSIHWMvc@9ZpCOlHv7vq^|nR=HiaE7R_7 zx~j)JiI&f-IKc%F2au3}gnuE2067uhg5ZGQmogBwYfos)yfn6V;?;#>)n-Oz%k$$a11yG90QI4$ADwNG4OxSfQcSrU&bgO z%0}vC=l7gBzc;(j+sQNa*{wS{1{?#90mp!2z%k$$a11yG90QI4$ADwNG4MagfEY4H z&+-@O!5;tr=g00~?Js=ztmG2l;+pdR=U za0B=h@DbqchZ%bv_!jU@pa(n&tO4)7kFnnZuLC~>UIC7P8gL#s2mA#uQGNmZ9MHfQ zfPLUm;7#!VJ>V2L2KIn;fc)|Yyo~u>{z7JD4Dq`hTKc1=mhp-5q+Vxvi)E_0j&oA@q!q>8>mblq2?v@(&;-qY=MyY+V^rg^} zfBG^Xs4!4oUmK}{Qq3(HDB<@5IWu8^E{jh{`iW7#ZjPnthq|BXsn-_T*VYPivhvUq zMm)G0+k8$-rBW$0nu$E+B99he^P#8*P2oH2qx0HXF(jr`!Hh8=L*q-7GGLB=x^(8%2^>Xdn z?)CQ3Uaz)|>!BPZV_dyZBgfTl-o9PN*Zo@Cm{@ME;V0o683)41MNL$NL5RdRixplN zjQS(x&$YMaV|YY)KMeZ#jwl$*VHu-U{){-5oJ7?a#8dTp=73Qc=T0EXNrx`kDSAVA z)LPPbE&E{-{WPhr#%S1dRQXP-o^0b{G?E_t&j+Xc#pJ~VU6{rG%RUxZKtCnTDa*H+ z?EpV>q%5{e`7Ik(`R=VKH6SC=#!107KM{T+b)k4Zgj4$wpk^XWPE8u089x}7K!J@= z-xNxAEU$d1^f3pkQ%F;s3yr8ZX-uco^C%7@F^11IHzC!4nmC5NB`L6IhbD*$#<^Sg zL|}FVnq$r{xUy+Ym<#z0O#Lz%tXs06sGkl0SehgbieVOm;n`6mS2uix6)2E#?dCli z1ffwQg$9ekN=i}Kkh-#{msict0E0oao9C=CXI~cv+g)8Vx@jHz@8dw@yUd!d*Gd8sMdglGv~DCWxlu-Z~o{@!E^guajFNl! zPbyYs3+4VtVhhg0 zMErMCjX7QaVY;CcjIQ3sved&Ohi%N)Ye@IZQMBV@t*w{#XL*80#k6-M5v|KSPJ)1z z)ihnmm;RK_LJMh4dA?%hrxG(|m{JOn-M|x(PJFE4oJ^ThmUfrhSGQS6nwA}4_3qN= zzgLmnk@1v=Fg3MPjXn@=7VI`jLK!7>Sw@>q(ICzARtnp_7jksIuLyRg{M+MZX)<+L z-YxVt4TsV%tj7X12B&Nkhf_WTD@IKv&xV%YGj@i#snFtB#LD&o@g_NgRk^RG$_Rrc zTk|9s%GmCad&uVlEt_c2m2$qwx|Oz3jZGi>kjH(bIEffbJ`PWiUXi;IHB+UvRf>HX zTAuCnK6`M~3@%#WwQ>-nlSPewww!lFBKg zHxmrbrcCmdCnG~lXxW)s;oV2 z4c>u$u^LYd-`d!GitmVko|A;n_YC~Os!tSKnhZ{Oy3Ji=^!)#IJkS3Wpyz))ZrbCI zS=nnize;uPa11yG90QI4$ADwNG2j?*3^)cH1C9a5fMejl%|QOzlFr$SrKSo-VZA~x f&Fs4z`)?xN!{D#jO8LSs7cBpey= self.xenial_ocata: + services = [ + 'apache2', + 'aodh-evaluator: AlarmEvaluationService worker(0)', + 'aodh-notifier: AlarmNotifierService worker(0)', + ('aodh-listener: EventAlarmEvaluationService' + ' worker(0)')] + elif self.release >= self.xenial_newton: + services = [ + ('/usr/bin/python /usr/bin/aodh-api --port 8032 -- ' + '--config-file=/etc/aodh/aodh.conf ' + '--log-file=/var/log/aodh/aodh-api.log'). + 'aodh-evaluator - AlarmEvaluationService(0)', + 'aodh-notifier - AlarmNotifierService(0)', + 'aodh-listener - EventAlarmEvaluationService(0)'] + else: + services = [ + 'aodh-api', + 'aodh-evaluator', + 'aodh-notifier', + 'aodh-listener'] + return services + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change. + + Change disk format and assert then change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + set_default = {'debug': 'False'} + set_alternate = {'debug': 'True'} + + # Config file affected by juju set config change + conf_file = '/etc/aodh/aodh.conf' + + # Make config change, check for service restarts + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + {'DEFAULT': {'debug': ['False']}}, + {'DEFAULT': {'debug': ['True']}}, + self.services) + + def test_901_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + xenial_ocata = openstack_utils.get_os_release('xenial_ocata') + xenial_newton = openstack_utils.get_os_release('xenial_newton') + if self.release >= bionic_stein: + pgrep_full = True + else: + pgrep_full = False + with self.pause_resume( + self.services, + pgrep_full=pgrep_full): + logging.info("Testing pause resume") diff --git a/zaza/openstack/configure/telemetry.py b/zaza/openstack/configure/telemetry.py new file mode 100644 index 0000000..54061b1 --- /dev/null +++ b/zaza/openstack/configure/telemetry.py @@ -0,0 +1,31 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Configure and manage masakari. + +Functions for managing masakari resources and simulating compute node loss +and recovery. +""" + +import logging + +import zaza.model + +def ceilometer_upgrade(application_name=None, model_name=None): + zaza.model.run_action_on_leader( + application_name or self.application_name, + 'ceilometer-upgrade', + model_name=model_name, + action_params={}) + From 8b60351be5f45aa84d4b923b15ae01302b2669a3 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 22 Sep 2019 07:28:08 +0100 Subject: [PATCH 008/898] Fix ceilometer action --- zaza/openstack/charm_tests/aodh/.tests.py.swp | Bin 16384 -> 0 bytes zaza/openstack/charm_tests/aodh/setup.py | 2 +- zaza/openstack/configure/telemetry.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 zaza/openstack/charm_tests/aodh/.tests.py.swp diff --git a/zaza/openstack/charm_tests/aodh/.tests.py.swp b/zaza/openstack/charm_tests/aodh/.tests.py.swp deleted file mode 100644 index 431d6bca4e2bc867994a800908f7a9d92ed31557..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHN&5t8T6)yq_A-l_G5L}Qd_Q9TLY|qSIHWMvc@9ZpCOlHv7vq^|nR=HiaE7R_7 zx~j)JiI&f-IKc%F2au3}gnuE2067uhg5ZGQmogBwYfos)yfn6V;?;#>)n-Oz%k$$a11yG90QI4$ADwNG4OxSfQcSrU&bgO z%0}vC=l7gBzc;(j+sQNa*{wS{1{?#90mp!2z%k$$a11yG90QI4$ADwNG4MagfEY4H z&+-@O!5;tr=g00~?Js=ztmG2l;+pdR=U za0B=h@DbqchZ%bv_!jU@pa(n&tO4)7kFnnZuLC~>UIC7P8gL#s2mA#uQGNmZ9MHfQ zfPLUm;7#!VJ>V2L2KIn;fc)|Yyo~u>{z7JD4Dq`hTKc1=mhp-5q+Vxvi)E_0j&oA@q!q>8>mblq2?v@(&;-qY=MyY+V^rg^} zfBG^Xs4!4oUmK}{Qq3(HDB<@5IWu8^E{jh{`iW7#ZjPnthq|BXsn-_T*VYPivhvUq zMm)G0+k8$-rBW$0nu$E+B99he^P#8*P2oH2qx0HXF(jr`!Hh8=L*q-7GGLB=x^(8%2^>Xdn z?)CQ3Uaz)|>!BPZV_dyZBgfTl-o9PN*Zo@Cm{@ME;V0o683)41MNL$NL5RdRixplN zjQS(x&$YMaV|YY)KMeZ#jwl$*VHu-U{){-5oJ7?a#8dTp=73Qc=T0EXNrx`kDSAVA z)LPPbE&E{-{WPhr#%S1dRQXP-o^0b{G?E_t&j+Xc#pJ~VU6{rG%RUxZKtCnTDa*H+ z?EpV>q%5{e`7Ik(`R=VKH6SC=#!107KM{T+b)k4Zgj4$wpk^XWPE8u089x}7K!J@= z-xNxAEU$d1^f3pkQ%F;s3yr8ZX-uco^C%7@F^11IHzC!4nmC5NB`L6IhbD*$#<^Sg zL|}FVnq$r{xUy+Ym<#z0O#Lz%tXs06sGkl0SehgbieVOm;n`6mS2uix6)2E#?dCli z1ffwQg$9ekN=i}Kkh-#{msict0E0oao9C=CXI~cv+g)8Vx@jHz@8dw@yUd!d*Gd8sMdglGv~DCWxlu-Z~o{@!E^guajFNl! zPbyYs3+4VtVhhg0 zMErMCjX7QaVY;CcjIQ3sved&Ohi%N)Ye@IZQMBV@t*w{#XL*80#k6-M5v|KSPJ)1z z)ihnmm;RK_LJMh4dA?%hrxG(|m{JOn-M|x(PJFE4oJ^ThmUfrhSGQS6nwA}4_3qN= zzgLmnk@1v=Fg3MPjXn@=7VI`jLK!7>Sw@>q(ICzARtnp_7jksIuLyRg{M+MZX)<+L z-YxVt4TsV%tj7X12B&Nkhf_WTD@IKv&xV%YGj@i#snFtB#LD&o@g_NgRk^RG$_Rrc zTk|9s%GmCad&uVlEt_c2m2$qwx|Oz3jZGi>kjH(bIEffbJ`PWiUXi;IHB+UvRf>HX zTAuCnK6`M~3@%#WwQ>-nlSPewww!lFBKg zHxmrbrcCmdCnG~lXxW)s;oV2 z4c>u$u^LYd-`d!GitmVko|A;n_YC~Os!tSKnhZ{Oy3Ji=^!)#IJkS3Wpyz))ZrbCI zS=nnize;uPa11yG90QI4$ADwNG2j?*3^)cH1C9a5fMejl%|QOzlFr$SrKSo-VZA~x f&Fs4z`)?xN!{D#jO8LSs7cBpey Date: Sun, 22 Sep 2019 07:38:13 +0100 Subject: [PATCH 009/898] Fix typo --- zaza/openstack/charm_tests/aodh/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index e5e2884..5a693ea 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -49,7 +49,7 @@ class AodhTest(test_utils.OpenStackBaseTest): services = [ ('/usr/bin/python /usr/bin/aodh-api --port 8032 -- ' '--config-file=/etc/aodh/aodh.conf ' - '--log-file=/var/log/aodh/aodh-api.log'). + '--log-file=/var/log/aodh/aodh-api.log'), 'aodh-evaluator - AlarmEvaluationService(0)', 'aodh-notifier - AlarmNotifierService(0)', 'aodh-listener - EventAlarmEvaluationService(0)'] From d41f54f239f5baec9aa0c6228b57f075182b5402 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 23 Sep 2019 08:27:06 +0100 Subject: [PATCH 010/898] Add bionic stein to aodh tests --- zaza/openstack/charm_tests/aodh/tests.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index 5a693ea..5879ea9 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -34,6 +34,7 @@ class AodhTest(test_utils.OpenStackBaseTest): super(AodhTest, cls).setUpClass() cls.xenial_ocata = openstack_utils.get_os_release('xenial_ocata') cls.xenial_newton = openstack_utils.get_os_release('xenial_newton') + cls.bionic_stein = openstack_utils.get_os_release('bionic_stein') cls.release = openstack_utils.get_os_release() @property @@ -89,9 +90,7 @@ class AodhTest(test_utils.OpenStackBaseTest): Pause service and check services are stopped then resume and check they are started """ - xenial_ocata = openstack_utils.get_os_release('xenial_ocata') - xenial_newton = openstack_utils.get_os_release('xenial_newton') - if self.release >= bionic_stein: + if self.release >= self.bionic_stein: pgrep_full = True else: pgrep_full = False From af856138eee8d354fd695d3439eafaa7f2f5a770 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 23 Sep 2019 15:15:11 +0100 Subject: [PATCH 011/898] Increase wait time for guest to boot. Zaza seems to currently wait for 64+32+16+8+4+2+1 seconds (127) for a guest to boot. This is not quite enough sometimes, so increase this to 511 seconds. --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 6769de6..1c3789b 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1981,7 +1981,7 @@ def get_ports_from_device_id(neutron_client, device_id): @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), - reraise=True, stop=tenacity.stop_after_attempt(8)) + reraise=True, stop=tenacity.stop_after_attempt(10)) def cloud_init_complete(nova_client, vm_id, bootstring): """Wait for cloud init to complete on the given vm. From 022102ab911a631d4dd93f11dfb29db88c19070d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Sep 2019 08:19:19 +0100 Subject: [PATCH 012/898] Add security actions --- zaza/openstack/charm_tests/aodh/tests.py | 39 ++++++++++++++++++++++-- zaza/openstack/configure/telemetry.py | 5 ++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index 5879ea9..8042cea 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -16,12 +16,10 @@ """Encapsulate masakari testing.""" -from datetime import datetime import logging import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -39,13 +37,14 @@ class AodhTest(test_utils.OpenStackBaseTest): @property def services(self): + """Return a list of the service that should be running.""" if self.release >= self.xenial_ocata: services = [ 'apache2', 'aodh-evaluator: AlarmEvaluationService worker(0)', 'aodh-notifier: AlarmNotifierService worker(0)', ('aodh-listener: EventAlarmEvaluationService' - ' worker(0)')] + ' worker(0)')] elif self.release >= self.xenial_newton: services = [ ('/usr/bin/python /usr/bin/aodh-api --port 8032 -- ' @@ -98,3 +97,37 @@ class AodhTest(test_utils.OpenStackBaseTest): self.services, pgrep_full=pgrep_full): logging.info("Testing pause resume") + + +class SecurityTest(test_utils.OpenStackBaseTest): + """Neutron APIsecurity tests tests.""" + + def test_security_checklist(self): + """Verify expected state with security-checklist.""" + # Changes fixing the below expected failures will be made following + # this initial work to get validation in. There will be bugs targeted + # to each one and resolved independently where possible. + + expected_failures = [ + 'validate-enables-tls', + 'validate-uses-tls-for-keystone', + ] + expected_passes = [ + 'validate-file-ownership', + 'validate-file-permissions', + 'validate-uses-keystone', + ] + + for unit in zaza.model.get_units('aodh', + model_name=self.model_name): + logging.info('Running `security-checklist` action' + ' on unit {}'.format(unit.entity_id)) + test_utils.audit_assertions( + zaza.model.run_action( + unit.entity_id, + 'security-checklist', + model_name=self.model_name, + action_params={}), + expected_passes, + expected_failures, + expected_to_pass=False) diff --git a/zaza/openstack/configure/telemetry.py b/zaza/openstack/configure/telemetry.py index 880395d..144d447 100644 --- a/zaza/openstack/configure/telemetry.py +++ b/zaza/openstack/configure/telemetry.py @@ -18,14 +18,13 @@ Functions for managing masakari resources and simulating compute node loss and recovery. """ -import logging - import zaza.model + def ceilometer_upgrade(application_name=None, model_name=None): + """Run ceilometer upgrade action.""" zaza.model.run_action_on_leader( application_name, 'ceilometer-upgrade', model_name=model_name, action_params={}) - From 3b010b08f85b817b2477b3cb740f7235b1f1fe04 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Sep 2019 08:55:12 +0100 Subject: [PATCH 013/898] Remove security action test Remove security action test as the action has not been implemented in the charm yet. --- zaza/openstack/charm_tests/aodh/tests.py | 34 ------------------------ 1 file changed, 34 deletions(-) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index 8042cea..cae30af 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -97,37 +97,3 @@ class AodhTest(test_utils.OpenStackBaseTest): self.services, pgrep_full=pgrep_full): logging.info("Testing pause resume") - - -class SecurityTest(test_utils.OpenStackBaseTest): - """Neutron APIsecurity tests tests.""" - - def test_security_checklist(self): - """Verify expected state with security-checklist.""" - # Changes fixing the below expected failures will be made following - # this initial work to get validation in. There will be bugs targeted - # to each one and resolved independently where possible. - - expected_failures = [ - 'validate-enables-tls', - 'validate-uses-tls-for-keystone', - ] - expected_passes = [ - 'validate-file-ownership', - 'validate-file-permissions', - 'validate-uses-keystone', - ] - - for unit in zaza.model.get_units('aodh', - model_name=self.model_name): - logging.info('Running `security-checklist` action' - ' on unit {}'.format(unit.entity_id)) - test_utils.audit_assertions( - zaza.model.run_action( - unit.entity_id, - 'security-checklist', - model_name=self.model_name, - action_params={}), - expected_passes, - expected_failures, - expected_to_pass=False) From 7a519a7bc02981ba78f876b955c0d5728a45fa16 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Sep 2019 09:21:10 +0100 Subject: [PATCH 014/898] Never use pgrep as service names need escaping --- zaza/openstack/charm_tests/aodh/tests.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index cae30af..cabbac3 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -18,7 +18,6 @@ import logging -import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -89,11 +88,7 @@ class AodhTest(test_utils.OpenStackBaseTest): Pause service and check services are stopped then resume and check they are started """ - if self.release >= self.bionic_stein: - pgrep_full = True - else: - pgrep_full = False with self.pause_resume( self.services, - pgrep_full=pgrep_full): + pgrep_full=False): logging.info("Testing pause resume") From 19306d660031133dd0dbb84393c1bde58ac2dd2b Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Tue, 24 Sep 2019 16:25:53 +0200 Subject: [PATCH 015/898] WIP - add plumb_dataport_method --- zaza/openstack/configure/network.py | 11 ++++++ zaza/openstack/utilities/openstack.py | 55 +++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 68965d9..542c595 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -219,6 +219,17 @@ def setup_gateway_ext_port(network_config, keystone_session=None): dvr_mode=network_config.get("dvr_enabled", False), net_id=net_id) + current_release = openstack_utils.get_os_release() + bionic_queens = openstack_utils.get_os_release('bionic_queens') + if current_release >= bionic_queens: + logging.warn("Adding second interface for dataport to guest netplan, \ + for bionic-queens and later") + openstack_utils.plumb_guest_nic( + nova_client, + neutron_client, + dvr_mode=network_config.get("dvr_enabled", False), + net_id=net_id) + def run_from_cli(**kwargs): """Run network configurations from CLI. diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 9b2e617..87c54ef 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -55,6 +55,7 @@ import sys import tempfile import tenacity import urllib +import textwrap from zaza import model from zaza.openstack.utilities import ( @@ -483,6 +484,60 @@ def get_admin_net(neutron_client): return net +def plumb_guest_nic(novaclient, neutronclient, + dvr_mode=None, net_id=None): + + """Add port associated with net_id to netplan + + :type net_id: string + """ + + if dvr_mode: + uuids = get_ovs_uuids() + else: + uuids = get_gateway_uuids() + + for uuid in uuids: + server = novaclient.servers.get(uuid) + ext_port_name = "{}_ext-port".format(server.name) + for port in neutronclient.list_ports(device_id=server.id)['ports']: + if port['name'] == ext_port_name: + logging.info('Adding second port to netplan in guest:\ + {}'.format(port['name'])) + mac_address = port['mac_address'] + if dvr_mode: + application_name = 'neutron-openvswitch' + else: + application_name = 'neutron-gateway' + unit_name = juju_utils.get_unit_name_from_ip_address( + server.ip, application_name) + interface = model.async_run_on_unit( + unit_name, "ip addr|grep\ + {}".format( + mac_address)).split("\n")[0].split(" ")[2] + body_value = textwrap.dedent("""\ + network: + ethernets: + {}: + dhcp4: false + dhcp6: true + optional: true + match: + macaddress: {} + set-name: {} + \n + version: 2 + """.format(interface, mac_address, interface)) + netplan_file = open("60-dataport.yaml", "w") + netplan_file.write(body_value) + netplan_file.close() + model.async_scp_to_unit(unit_name, './60-dataport.yaml', + '/etc/netplan/', user="root") + subprocess.call("rm 60-dataport.yaml") + model.async_run_on_unit(unit_name, "netplan apply", + user="root") + + def configure_gateway_ext_port(novaclient, neutronclient, dvr_mode=None, net_id=None): """Configure the neturong-gateway external port. From c94fbceac1c4c60dedc4307997e53384834b09ca Mon Sep 17 00:00:00 2001 From: Eduardo Sousa Date: Tue, 24 Sep 2019 15:44:49 +0100 Subject: [PATCH 016/898] Refactoring variables and properties --- .../openstack/charm_tests/ceilometer/setup.py | 11 +- .../openstack/charm_tests/ceilometer/tests.py | 105 +++++++++--------- 2 files changed, 58 insertions(+), 58 deletions(-) diff --git a/zaza/openstack/charm_tests/ceilometer/setup.py b/zaza/openstack/charm_tests/ceilometer/setup.py index 0051624..4d716a3 100644 --- a/zaza/openstack/charm_tests/ceilometer/setup.py +++ b/zaza/openstack/charm_tests/ceilometer/setup.py @@ -17,11 +17,12 @@ """Code for configuring Ceilometer.""" import logging +import unittest import zaza.model as zaza_model import zaza.openstack.utilities.openstack as openstack_utils -def basic_setup(unit='ceilometer/0'): +def basic_setup(): """Run setup for testing Ceilometer. Setup for testing Ceilometer is currently part of functional @@ -31,12 +32,12 @@ def basic_setup(unit='ceilometer/0'): xenial_pike = openstack_utils.get_os_release('xenial_pike') if current_release < xenial_pike: - logging.debug('Not checking ceilometer-upgrade') - return + raise unittest.SkipTest('Not checking ceilometer-upgrade') + logging.debug('Checking ceilometer-upgrade') - action = zaza_model.run_action( - unit, + action = zaza_model.run_action_on_leader( + 'ceilometer', 'ceilometer-upgrade', raise_on_failure=True) diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index f57d4af..1e2e316 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -25,11 +25,59 @@ import zaza.openstack.charm_tests.test_utils as test_utils class CeilometerTest(test_utils.OpenStackBaseTest): """Encapsulate Ceilometer tests.""" + CONF_FILE = '/etc/ceilometer/ceilometer.conf' + + XENIAL_PIKE = openstack_utils.get_os_release('xenial_pike') + XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') + XENIAL_NEWTON = openstack_utils.get_os_release('xenial_newton') + TRUSTY_MITAKA = openstack_utils.get_os_release('trusty_mitaka') + TRUSTY_LIBERTY = openstack_utils.get_os_release('trusty_liberty') + @classmethod def setUpClass(cls): """Run class setup for running Ceilometer tests.""" super(CeilometerTest, cls).setUpClass() + @property + def services(self): + """Return a list services for Openstack Release.""" + current_release = openstack_utils.get_os_release() + services = [] + + if current_release >= CeilometerTest.XENIAL_PIKE: + services.append('ceilometer-polling: AgentManager worker(0)') + services.append('ceilometer-agent-notification: ' + 'NotificationService worker(0)') + elif current_release >= CeilometerTest.XENIAL_OCATA: + services.append('ceilometer-collector: CollectorService worker(0)') + services.append('ceilometer-polling: AgentManager worker(0)') + services.append('ceilometer-agent-notification: ' + 'NotificationService worker(0)') + services.append('apache2') + elif current_release >= CeilometerTest.XENIAL_NEWTON: + services.append('ceilometer-collector - CollectorService(0)') + services.append('ceilometer-polling - AgentManager(0)') + services.append('ceilometer-agent-notification - ' + 'NotificationService(0)') + services.append('ceilometer-api') + else: + services.append('ceilometer-collector') + services.append('ceilometer-api') + services.append('ceilometer-agent-notification') + + if current_release < CeilometerTest.TRUSTY_MITAKA: + services.append('ceilometer-alarm-notifier') + services.append('ceilometer-alarm-evaluator') + + if current_release >= CeilometerTest.TRUSTY_LIBERTY: + # Liberty and later + services.append('ceilometer-polling') + else: + # Juno and earlier + services.append('ceilometer-agent-central') + + return services + # NOTE(beisner): need to add more functional tests def test_900_restart_on_config_change(self): @@ -50,64 +98,14 @@ class CeilometerTest(test_utils.OpenStackBaseTest): default_entry = {'DEFAULT': {'debug': [current_value]}} alternate_entry = {'DEFAULT': {'debug': [new_value]}} - # Config file affected by juju set config change - conf_file = '/etc/ceilometer/ceilometer.conf' - services = {} - current_release = openstack_utils.get_os_release() - xenial_pike = openstack_utils.get_os_release('xenial_pike') - xenial_ocata = openstack_utils.get_os_release('xenial_ocata') - xenial_newton = openstack_utils.get_os_release('xenial_newton') - trusty_mitaka = openstack_utils.get_os_release('trusty_mitaka') - trusty_liberty = openstack_utils.get_os_release('trusty_liberty') - - if current_release >= xenial_pike: - services = { - 'ceilometer-polling: AgentManager worker(0)': conf_file, - 'ceilometer-agent-notification: NotificationService worker(0)': - conf_file, - } - elif current_release >= xenial_ocata: - services = { - 'ceilometer-collector: CollectorService worker(0)': conf_file, - 'ceilometer-polling: AgentManager worker(0)': conf_file, - 'ceilometer-agent-notification: NotificationService worker(0)': - conf_file, - 'apache2': conf_file, - } - elif current_release >= xenial_newton: - services = { - 'ceilometer-collector - CollectorService(0)': conf_file, - 'ceilometer-polling - AgentManager(0)': conf_file, - 'ceilometer-agent-notification - NotificationService(0)': - conf_file, - 'ceilometer-api': conf_file, - } - else: - services = { - 'ceilometer-collector': conf_file, - 'ceilometer-api': conf_file, - 'ceilometer-agent-notification': conf_file, - } - - if current_release < trusty_mitaka: - services['ceilometer-alarm-notifier'] = conf_file - services['ceilometer-alarm-evaluator'] = conf_file - - if current_release >= trusty_liberty: - # Liberty and later - services['ceilometer-polling'] = conf_file - else: - # Juno and earlier - services['ceilometer-agent-central'] = conf_file - logging.info('changing config: {}'.format(set_alternate)) self.restart_on_changed( - conf_file, + CeilometerTest.CONF_FILE, set_default, set_alternate, default_entry, alternate_entry, - services) + self.services) def test_901_pause_resume(self): """Run pause and resume tests. @@ -115,4 +113,5 @@ class CeilometerTest(test_utils.OpenStackBaseTest): Pause service and check services are stopped then resume and check they are started. """ - self.pause_resume(['ceilometer']) + with self.pause_resume(['ceilometer']): + logging.info("Testing pause and resume") From 2bbd4247ff68b8bd545afb2e64d477f8e230a9c4 Mon Sep 17 00:00:00 2001 From: Seyeong Kim Date: Wed, 25 Sep 2019 11:10:28 +0900 Subject: [PATCH 017/898] Increase cert ttl from 80days to 4000days Vault testing is failed if we want to increase vault's default ttl to 10 years because zaza specify cert's ttl is only 80days. https://review.opendev.org/#/c/678161 --- zaza/openstack/utilities/cert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/cert.py b/zaza/openstack/utilities/cert.py index 180f796..d138386 100644 --- a/zaza/openstack/utilities/cert.py +++ b/zaza/openstack/utilities/cert.py @@ -190,7 +190,7 @@ def sign_csr(csr, ca_private_key, ca_cert=None, issuer_name=None, datetime.datetime.today() - datetime.timedelta(1, 0, 0), ) builder = builder.not_valid_after( - datetime.datetime.today() + datetime.timedelta(80, 0, 0), + datetime.datetime.today() + datetime.timedelta(4000, 0, 0), ) builder = builder.subject_name(new_csr.subject) builder = builder.public_key(new_csr.public_key()) From 730cd788c8cd57245882bd59b29c268e1b4b962b Mon Sep 17 00:00:00 2001 From: Eduardo Sousa Date: Wed, 25 Sep 2019 13:31:40 +0100 Subject: [PATCH 018/898] Fixing skip test message --- zaza/openstack/charm_tests/ceilometer/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceilometer/setup.py b/zaza/openstack/charm_tests/ceilometer/setup.py index 4d716a3..1cd9b2d 100644 --- a/zaza/openstack/charm_tests/ceilometer/setup.py +++ b/zaza/openstack/charm_tests/ceilometer/setup.py @@ -32,7 +32,8 @@ def basic_setup(): xenial_pike = openstack_utils.get_os_release('xenial_pike') if current_release < xenial_pike: - raise unittest.SkipTest('Not checking ceilometer-upgrade') + raise unittest.SkipTest('Skipping ceilometer-upgrade as it is not ' + 'supported before Pike') logging.debug('Checking ceilometer-upgrade') From 028956dd290a8c7c8f66cc692bb847882d17800e Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 18 Sep 2019 16:14:51 +0200 Subject: [PATCH 019/898] Add initial policyd initial test Add the initial, general, 'success' test for policyd support in charms. --- .../openstack/charm_tests/policyd/__init__.py | 17 ++++ zaza/openstack/charm_tests/policyd/tests.py | 97 +++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 zaza/openstack/charm_tests/policyd/__init__.py create mode 100644 zaza/openstack/charm_tests/policyd/tests.py diff --git a/zaza/openstack/charm_tests/policyd/__init__.py b/zaza/openstack/charm_tests/policyd/__init__.py new file mode 100644 index 0000000..89ec5f1 --- /dev/null +++ b/zaza/openstack/charm_tests/policyd/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing policyd overrides across a +collection of charms. +""" diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py new file mode 100644 index 0000000..17b4369 --- /dev/null +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -0,0 +1,97 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate policyd testing.""" + +import logging +import os +import shutil +import tempfile +import zipfile + +import zaza.model as zaza_model + +import zaza.openstack.charm_tests.test_utils as test_utils + + +class PolicydTest(test_utils.OpenStackBaseTest): + """Charm operation tests. + + The policyd test needs some config from the tests.yaml in order to work + properly. A top level key of "tests_options". Under that key is + 'policyd', and then the k:v of 'service': . e.g. for keystone + + tests_options: + policyd: + service: keystone + """ + + def setUpClass(cls, application_name=None): + super().setUpClass(application_name) + cls._tmp_dir = tempfile.mkdtemp() + cls._service_name = \ + cls.test_config['tests_options']['policyd']['service'] + + def tearDownClass(cls): + super().tearDownClass() + try: + shutil.rmtree(cls._tmp_dir, ignore_errors=True) + except Exception as e: + logging.error("Removing the policyd tempdir/files failed: {}" + .format(str(e))) + + def tearDown(self): + """Ensure that the policyd config is switched off and the charm is + stable at the end of the test. + """ + self._set_config_and_wait(False) + + def _set_config_and_wait(self, state): + zaza_model.set_application_config(self.application_name, + {"use-policyd-override", state}) + zaza_model.block_until_all_units_idle() + + def _make_zip_file_from(self, name, files): + """Make a zip file from a dictionary of filename: string. + + :param name: the name of the zip file + :type name: PathLike + :param files: a dict of name: string to construct the files from. + :type files: Dict[str, str] + :returns: temp file that is the zip file. + :rtype: PathLike + """ + path = os.path.join(self._tmp_dir, name) + with zipfile.ZipFile(path, "w") as zfp: + for name, contents in files.items(): + zfp.writestr(name, contents) + + def test_policyd_good_yaml(self): + # Test that the policyd with a good zipped yaml file puts the yaml file + # in the right directory + good = { + 'file1.yaml': "{'rule1': '!'}" + } + good_zip_path = self._make_zip_file_from('good.zip', good) + zaza_model.attach_resource(self.application_name, + 'policyd-override', + good_zip_path) + zaza_model.block_until_all_units_idle() + self._set_config_and_wait(True) + # check that the file gets to the right location + path = os.path.join( + "/etc", self._service_name, "policy.d", 'file1.yaml') + zaza_model.block_until_file_has_contents(self.application_name, + path, + "'rule1': '!'") From 5e2fbf96399e395ed397305e560567142235a462 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 18 Sep 2019 16:49:18 +0200 Subject: [PATCH 020/898] Fix class super calls --- zaza/openstack/charm_tests/policyd/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 17b4369..0131b65 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -38,13 +38,13 @@ class PolicydTest(test_utils.OpenStackBaseTest): """ def setUpClass(cls, application_name=None): - super().setUpClass(application_name) + super(PolicydTest, cls).setUpClass(application_name) cls._tmp_dir = tempfile.mkdtemp() cls._service_name = \ cls.test_config['tests_options']['policyd']['service'] def tearDownClass(cls): - super().tearDownClass() + super(PolicydTest, cls).tearDownClass() try: shutil.rmtree(cls._tmp_dir, ignore_errors=True) except Exception as e: From 677de69b9a95bd323468df03718b0d41564e286f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 18 Sep 2019 16:54:31 +0200 Subject: [PATCH 021/898] More fixes to super calls --- zaza/openstack/charm_tests/policyd/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 0131b65..bff1b5c 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -37,12 +37,14 @@ class PolicydTest(test_utils.OpenStackBaseTest): service: keystone """ + @classmethod def setUpClass(cls, application_name=None): super(PolicydTest, cls).setUpClass(application_name) cls._tmp_dir = tempfile.mkdtemp() cls._service_name = \ cls.test_config['tests_options']['policyd']['service'] + @classmethod def tearDownClass(cls): super(PolicydTest, cls).tearDownClass() try: From 99db1e3eab85d589dcec6bdb31ca5c98eababe0d Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 18 Sep 2019 16:59:22 +0200 Subject: [PATCH 022/898] Fix test with None path --- zaza/openstack/charm_tests/policyd/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index bff1b5c..802453c 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -78,6 +78,7 @@ class PolicydTest(test_utils.OpenStackBaseTest): with zipfile.ZipFile(path, "w") as zfp: for name, contents in files.items(): zfp.writestr(name, contents) + return path def test_policyd_good_yaml(self): # Test that the policyd with a good zipped yaml file puts the yaml file From 8aa4102855754a0f90a234e0ee4fdcd20e5d1644 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 18 Sep 2019 17:01:44 +0200 Subject: [PATCH 023/898] Fix config setting code --- zaza/openstack/charm_tests/policyd/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 802453c..22c1977 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -61,7 +61,7 @@ class PolicydTest(test_utils.OpenStackBaseTest): def _set_config_and_wait(self, state): zaza_model.set_application_config(self.application_name, - {"use-policyd-override", state}) + {"use-policyd-override": state}) zaza_model.block_until_all_units_idle() def _make_zip_file_from(self, name, files): From 89980f1cc539c40ebcef2575affd5ab01e1d6709 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 09:11:53 +0200 Subject: [PATCH 024/898] Fix using bools with libjuju --- zaza/openstack/charm_tests/policyd/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 22c1977..7fd0c9e 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -60,8 +60,9 @@ class PolicydTest(test_utils.OpenStackBaseTest): self._set_config_and_wait(False) def _set_config_and_wait(self, state): + s = "true" if state else "false" zaza_model.set_application_config(self.application_name, - {"use-policyd-override": state}) + {"use-policyd-override": s}) zaza_model.block_until_all_units_idle() def _make_zip_file_from(self, name, files): From 214660f43d7892495c487dab71c7094a216546d5 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 10:04:27 +0200 Subject: [PATCH 025/898] Added additional logging --- zaza/openstack/charm_tests/policyd/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 7fd0c9e..5657c74 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -88,14 +88,19 @@ class PolicydTest(test_utils.OpenStackBaseTest): 'file1.yaml': "{'rule1': '!'}" } good_zip_path = self._make_zip_file_from('good.zip', good) + logging.info("About to attach the resource") zaza_model.attach_resource(self.application_name, 'policyd-override', good_zip_path) + logging.info("... waiting for idle") zaza_model.block_until_all_units_idle() + logging.info("Now setting config to true") self._set_config_and_wait(True) # check that the file gets to the right location path = os.path.join( "/etc", self._service_name, "policy.d", 'file1.yaml') + logging.info("Now checking for file contents: {}".format(path)) zaza_model.block_until_file_has_contents(self.application_name, path, "'rule1': '!'") + logging.info("...done") From 6306615a1777b59b1bd7b269c04a1b5a82319e77 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 11:55:40 +0200 Subject: [PATCH 026/898] More debugging in the test --- tox.ini | 4 ++-- zaza/openstack/charm_tests/policyd/tests.py | 21 +++++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 2447271..6a37cfe 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ deps = -r{toxinidir}/requirements.txt commands = /bin/true [flake8] -ignore = E402,E226 +ignore = E402,E226,W504 per-file-ignores = unit_tests/**: D @@ -34,4 +34,4 @@ basepython = python3 changedir = doc/source deps = -r{toxinidir}/requirements.txt -commands = sphinx-build -W -b html -d {toxinidir}/doc/build/doctrees . {toxinidir}/doc/build/html \ No newline at end of file +commands = sphinx-build -W -b html -d {toxinidir}/doc/build/doctrees . {toxinidir}/doc/build/html diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 5657c74..9db2611 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -23,6 +23,7 @@ import zipfile import zaza.model as zaza_model import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils class PolicydTest(test_utils.OpenStackBaseTest): @@ -40,6 +41,9 @@ class PolicydTest(test_utils.OpenStackBaseTest): @classmethod def setUpClass(cls, application_name=None): super(PolicydTest, cls).setUpClass(application_name) + if (openstack_utils.get_os_release() < + openstack_utils.get_os_release('xenial_queens')): + cls.SkipTest("Test not valid before xenial_queens") cls._tmp_dir = tempfile.mkdtemp() cls._service_name = \ cls.test_config['tests_options']['policyd']['service'] @@ -60,9 +64,10 @@ class PolicydTest(test_utils.OpenStackBaseTest): self._set_config_and_wait(False) def _set_config_and_wait(self, state): - s = "true" if state else "false" - zaza_model.set_application_config(self.application_name, - {"use-policyd-override": s}) + s = "True" if state else "False" + config = {"user-policyd-override": s} + logging.info("Setting config to", config) + zaza_model.set_application_config(self.application_name, config) zaza_model.block_until_all_units_idle() def _make_zip_file_from(self, name, files): @@ -102,5 +107,13 @@ class PolicydTest(test_utils.OpenStackBaseTest): logging.info("Now checking for file contents: {}".format(path)) zaza_model.block_until_file_has_contents(self.application_name, path, - "'rule1': '!'") + "rule1: '!'") + logging.info("... waiting for idle") + zaza_model.block_until_all_units_idle() + # check that the status includes "PO:" at the beginning of the status + # line + + # disable the policy override + # verify that the file no longer exists + # check that the status no longer has "PO:" on it. logging.info("...done") From f5f3c9a966f1f65949b3f7f098a8c2fda9328d7d Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 12:07:23 +0200 Subject: [PATCH 027/898] Yet more fixes to the test --- zaza/openstack/charm_tests/policyd/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 9db2611..49d03a9 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -65,7 +65,7 @@ class PolicydTest(test_utils.OpenStackBaseTest): def _set_config_and_wait(self, state): s = "True" if state else "False" - config = {"user-policyd-override": s} + config = {"use-policyd-override": s} logging.info("Setting config to", config) zaza_model.set_application_config(self.application_name, config) zaza_model.block_until_all_units_idle() From 5ba0cf211cf87475ad95e32b1941ea217f1ef8c8 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 12:14:05 +0200 Subject: [PATCH 028/898] Yet more fixes to the test1 --- zaza/openstack/charm_tests/policyd/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 49d03a9..9e92d96 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -66,7 +66,7 @@ class PolicydTest(test_utils.OpenStackBaseTest): def _set_config_and_wait(self, state): s = "True" if state else "False" config = {"use-policyd-override": s} - logging.info("Setting config to", config) + logging.info("Setting config to {}".format(config)) zaza_model.set_application_config(self.application_name, config) zaza_model.block_until_all_units_idle() From ce8c72e1a18620684ab4aeda441b56c017383980 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 12:24:35 +0200 Subject: [PATCH 029/898] tests to determine app status --- zaza/openstack/charm_tests/policyd/tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 9e92d96..bad21d7 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -21,6 +21,7 @@ import tempfile import zipfile import zaza.model as zaza_model +import zaza.utilities.juju as zaza_juju import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -112,6 +113,8 @@ class PolicydTest(test_utils.OpenStackBaseTest): zaza_model.block_until_all_units_idle() # check that the status includes "PO:" at the beginning of the status # line + app_status = zaza_juju.get_application_status(self.application_name) + logging.info("App status is: {}".format(app_status)) # disable the policy override # verify that the file no longer exists From 2beddeafefe38e768ce19e7d29007b6ec403e4c2 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 13:52:34 +0200 Subject: [PATCH 030/898] more iteration to discover workload stats --- zaza/openstack/charm_tests/policyd/tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index bad21d7..dd418e5 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -114,7 +114,10 @@ class PolicydTest(test_utils.OpenStackBaseTest): # check that the status includes "PO:" at the beginning of the status # line app_status = zaza_juju.get_application_status(self.application_name) - logging.info("App status is: {}".format(app_status)) + wl_stats = [v['workload-status']['info'] + for k, v in app_status['units'].items() + if k.split('/')[0] == self.application_name] + logging.info("App status is: {}".format(wl_stats)) # disable the policy override # verify that the file no longer exists From aae3366fc9aececf81e0e8d3c97036ec7ec89097 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 14:14:00 +0200 Subject: [PATCH 031/898] Add further disable tests --- zaza/openstack/charm_tests/policyd/tests.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index dd418e5..2dee9bb 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -118,8 +118,20 @@ class PolicydTest(test_utils.OpenStackBaseTest): for k, v in app_status['units'].items() if k.split('/')[0] == self.application_name] logging.info("App status is: {}".format(wl_stats)) + self.assertTrue(all(s.startswith("PO:") for s in wl_stats)) + logging.info("App status is valid") # disable the policy override - # verify that the file no longer exists + logging.info("Disabling policy override ...") + self._set_config_and_wait(False) # check that the status no longer has "PO:" on it. + app_status = zaza_juju.get_application_status(self.application_name) + wl_stats = [v['workload-status']['info'] + for k, v in app_status['units'].items() + if k.split('/')[0] == self.application_name] + self.assertFalse(any(s.startswith("PO:") for s in wl_stats)) + logging.info("... done") + + # verify that the file no longer exists logging.info("...done") + From 67f1650917ed8d16e16f7818d553d0949052eee3 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 14:23:19 +0200 Subject: [PATCH 032/898] Add further disable tests1 --- zaza/openstack/charm_tests/policyd/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 2dee9bb..55736a5 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -129,6 +129,7 @@ class PolicydTest(test_utils.OpenStackBaseTest): wl_stats = [v['workload-status']['info'] for k, v in app_status['units'].items() if k.split('/')[0] == self.application_name] + logging.info("App status is: {}".format(wl_stats)) self.assertFalse(any(s.startswith("PO:") for s in wl_stats)) logging.info("... done") From 1110ced7c17c45ff8ffb2578c207e0b8d12d2dcb Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 14:34:08 +0200 Subject: [PATCH 033/898] More test debugging --- zaza/openstack/charm_tests/policyd/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 55736a5..bb6fa4a 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -124,12 +124,13 @@ class PolicydTest(test_utils.OpenStackBaseTest): # disable the policy override logging.info("Disabling policy override ...") self._set_config_and_wait(False) + logging.info("Done setting false?") # check that the status no longer has "PO:" on it. app_status = zaza_juju.get_application_status(self.application_name) wl_stats = [v['workload-status']['info'] for k, v in app_status['units'].items() if k.split('/')[0] == self.application_name] - logging.info("App status is: {}".format(wl_stats)) + logging.info("After App status is: {}".format(wl_stats)) self.assertFalse(any(s.startswith("PO:") for s in wl_stats)) logging.info("... done") From 89a6d9ca39b52744ea3aad5f239c51ee98829dcc Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 14:56:43 +0200 Subject: [PATCH 034/898] More test debugging again --- zaza/openstack/charm_tests/policyd/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index bb6fa4a..b7d7f64 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -62,6 +62,7 @@ class PolicydTest(test_utils.OpenStackBaseTest): """Ensure that the policyd config is switched off and the charm is stable at the end of the test. """ + logging.info("tearDown") self._set_config_and_wait(False) def _set_config_and_wait(self, state): From 36f775390cca5aff530458d229dc898a9441124d Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 15:40:35 +0200 Subject: [PATCH 035/898] change approach to waiting on results --- zaza/openstack/charm_tests/policyd/tests.py | 72 ++++++++++++--------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index b7d7f64..f1abb00 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -20,6 +20,7 @@ import shutil import tempfile import zipfile +import zaza import zaza.model as zaza_model import zaza.utilities.juju as zaza_juju @@ -58,19 +59,11 @@ class PolicydTest(test_utils.OpenStackBaseTest): logging.error("Removing the policyd tempdir/files failed: {}" .format(str(e))) - def tearDown(self): - """Ensure that the policyd config is switched off and the charm is - stable at the end of the test. - """ - logging.info("tearDown") - self._set_config_and_wait(False) - - def _set_config_and_wait(self, state): + def _set_config(self, state): s = "True" if state else "False" config = {"use-policyd-override": s} logging.info("Setting config to {}".format(config)) zaza_model.set_application_config(self.application_name, config) - zaza_model.block_until_all_units_idle() def _make_zip_file_from(self, name, files): """Make a zip file from a dictionary of filename: string. @@ -102,7 +95,7 @@ class PolicydTest(test_utils.OpenStackBaseTest): logging.info("... waiting for idle") zaza_model.block_until_all_units_idle() logging.info("Now setting config to true") - self._set_config_and_wait(True) + self._set_config(True) # check that the file gets to the right location path = os.path.join( "/etc", self._service_name, "policy.d", 'file1.yaml') @@ -110,31 +103,52 @@ class PolicydTest(test_utils.OpenStackBaseTest): zaza_model.block_until_file_has_contents(self.application_name, path, "rule1: '!'") - logging.info("... waiting for idle") - zaza_model.block_until_all_units_idle() - # check that the status includes "PO:" at the beginning of the status - # line - app_status = zaza_juju.get_application_status(self.application_name) - wl_stats = [v['workload-status']['info'] - for k, v in app_status['units'].items() - if k.split('/')[0] == self.application_name] - logging.info("App status is: {}".format(wl_stats)) - self.assertTrue(all(s.startswith("PO:") for s in wl_stats)) + # ensure that the workload status info line starts with PO: + logging.info("Checking for workload status line starts with PO:") + block_until_wl_status_info_starts_with(self.application_name, "PO:") logging.info("App status is valid") # disable the policy override logging.info("Disabling policy override ...") - self._set_config_and_wait(False) - logging.info("Done setting false?") + self._set_config(False) # check that the status no longer has "PO:" on it. - app_status = zaza_juju.get_application_status(self.application_name) - wl_stats = [v['workload-status']['info'] - for k, v in app_status['units'].items() - if k.split('/')[0] == self.application_name] - logging.info("After App status is: {}".format(wl_stats)) - self.assertFalse(any(s.startswith("PO:") for s in wl_stats)) - logging.info("... done") + block_until_wl_status_info_starts_with( + self.application_name, "PO:", negate_match=True) # verify that the file no longer exists logging.info("...done") + +async def async_block_until_wl_status_info_starts_with( + app, status, model_name=None, negate_match=False, timeout=2700): + """Block until the all the units have a desired workload status that starts + with status. + + :param app: the application to check against + :type app: str + :param status: Status to wait for at the start of the string + :type status: str + :param model_name: Name of model to query. + :type model_name: Union[None, str] + :param negate_match: Wait until the match is not true; i.e. none match + :type negate_match: Union[None, bool] + :param timeout: Time to wait for unit to achieved desired status + :type timeout: float + """ + async def _unit_status(): + model_status = await zaza_model.async_get_status() + wl_infos = [v['workload-status']['info'] + for k, v in model_status.applications[app]['units'] + if k.split('/')[0] == app] + g = (s.startswith(status) for s in wl_infos) + if negate_match: + return not(any(g)) + else: + return all(g) + + async with zaza_model.run_in_model(model_name): + await zaza_model.async_block_until(_unit_status, timeout=timeout) + + +block_until_wl_status_info_starts_with = zaza.sync_wrapper( + async_block_until_wl_status_info_starts_with) From 3465dacb96331e7a688660e39f0526319fe33a31 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 15:49:14 +0200 Subject: [PATCH 036/898] change approach to waiting on results bug fix --- zaza/openstack/charm_tests/policyd/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index f1abb00..d42b4b7 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -138,7 +138,7 @@ async def async_block_until_wl_status_info_starts_with( async def _unit_status(): model_status = await zaza_model.async_get_status() wl_infos = [v['workload-status']['info'] - for k, v in model_status.applications[app]['units'] + for k, v in model_status.applications[app]['units'].items() if k.split('/')[0] == app] g = (s.startswith(status) for s in wl_infos) if negate_match: From 55ae4aad281c5b8267b912528a0079560fc2ecde Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 16:17:50 +0200 Subject: [PATCH 037/898] Add in check to test for removal of file --- zaza/openstack/charm_tests/policyd/tests.py | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index d42b4b7..abf997f 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -20,6 +20,8 @@ import shutil import tempfile import zipfile +from juju.errors import JujuError + import zaza import zaza.model as zaza_model import zaza.utilities.juju as zaza_juju @@ -112,10 +114,21 @@ class PolicydTest(test_utils.OpenStackBaseTest): logging.info("Disabling policy override ...") self._set_config(False) # check that the status no longer has "PO:" on it. + # we have to do it twice due to async races and that some info lines + # erase the PO: bit prior to actuall getting back to idle. The double + # check verifies that the charms have started, the idle waits until it + # is finiehed, and then the final check really makes sure they got + # switched off. + block_until_wl_status_info_starts_with( + self.application_name, "PO:", negate_match=True) + zaza_model.block_until_all_units_idle() block_until_wl_status_info_starts_with( self.application_name, "PO:", negate_match=True) # verify that the file no longer exists + logging.info("Checking that {} has been removed".format(path)) + block_until_file_missing(self.application_name, path) + logging.info("...done") @@ -152,3 +165,25 @@ async def async_block_until_wl_status_info_starts_with( block_until_wl_status_info_starts_with = zaza.sync_wrapper( async_block_until_wl_status_info_starts_with) + + +async def async_block_until_file_missing( + app, path, model_name=None, timeout=2700): + async def _check_for_file(model): + units = model.applications[app].units + results = [] + for unit in units: + try: + output = await unit.run('test -e {}; echo $?'.format(path)) + contents = output.data.get('results')['Stdout'] + results.append("1" in contents) + # libjuju throws a generic error for connection failure. So we + # cannot differentiate between a connectivity issue and a + # target file not existing error. For now just assume the + # latter. + except JujuError: + results.append(False) + return all(results) + + +block_until_file_missing = zaza.sync_wrapper(async_block_until_file_missing) From 024a4b0ff7e2cbb44256c82ab3d475ef0cc9f9d7 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 19 Sep 2019 17:26:29 +0200 Subject: [PATCH 038/898] move async code to zaza --- zaza/openstack/charm_tests/policyd/tests.py | 66 ++------------------- 1 file changed, 5 insertions(+), 61 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index abf997f..f92d633 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -107,7 +107,8 @@ class PolicydTest(test_utils.OpenStackBaseTest): "rule1: '!'") # ensure that the workload status info line starts with PO: logging.info("Checking for workload status line starts with PO:") - block_until_wl_status_info_starts_with(self.application_name, "PO:") + zaza_model.block_until_wl_status_info_starts_with( + self.application_name, "PO:") logging.info("App status is valid") # disable the policy override @@ -119,71 +120,14 @@ class PolicydTest(test_utils.OpenStackBaseTest): # check verifies that the charms have started, the idle waits until it # is finiehed, and then the final check really makes sure they got # switched off. - block_until_wl_status_info_starts_with( + zaza_model.block_until_wl_status_info_starts_with( self.application_name, "PO:", negate_match=True) zaza_model.block_until_all_units_idle() - block_until_wl_status_info_starts_with( + zaza_model.block_until_wl_status_info_starts_with( self.application_name, "PO:", negate_match=True) # verify that the file no longer exists logging.info("Checking that {} has been removed".format(path)) - block_until_file_missing(self.application_name, path) + zaza_model.block_until_file_missing(self.application_name, path) logging.info("...done") - - -async def async_block_until_wl_status_info_starts_with( - app, status, model_name=None, negate_match=False, timeout=2700): - """Block until the all the units have a desired workload status that starts - with status. - - :param app: the application to check against - :type app: str - :param status: Status to wait for at the start of the string - :type status: str - :param model_name: Name of model to query. - :type model_name: Union[None, str] - :param negate_match: Wait until the match is not true; i.e. none match - :type negate_match: Union[None, bool] - :param timeout: Time to wait for unit to achieved desired status - :type timeout: float - """ - async def _unit_status(): - model_status = await zaza_model.async_get_status() - wl_infos = [v['workload-status']['info'] - for k, v in model_status.applications[app]['units'].items() - if k.split('/')[0] == app] - g = (s.startswith(status) for s in wl_infos) - if negate_match: - return not(any(g)) - else: - return all(g) - - async with zaza_model.run_in_model(model_name): - await zaza_model.async_block_until(_unit_status, timeout=timeout) - - -block_until_wl_status_info_starts_with = zaza.sync_wrapper( - async_block_until_wl_status_info_starts_with) - - -async def async_block_until_file_missing( - app, path, model_name=None, timeout=2700): - async def _check_for_file(model): - units = model.applications[app].units - results = [] - for unit in units: - try: - output = await unit.run('test -e {}; echo $?'.format(path)) - contents = output.data.get('results')['Stdout'] - results.append("1" in contents) - # libjuju throws a generic error for connection failure. So we - # cannot differentiate between a connectivity issue and a - # target file not existing error. For now just assume the - # latter. - except JujuError: - results.append(False) - return all(results) - - -block_until_file_missing = zaza.sync_wrapper(async_block_until_file_missing) From c744b1a288f3fae05621e20857f1d385277dbd96 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 20 Sep 2019 14:00:19 +0200 Subject: [PATCH 039/898] Create generic / keystone test idea --- zaza/openstack/charm_tests/policyd/tests.py | 26 +++++++++++++-------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index f92d633..8bdacbe 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -20,17 +20,13 @@ import shutil import tempfile import zipfile -from juju.errors import JujuError - -import zaza import zaza.model as zaza_model -import zaza.utilities.juju as zaza_juju import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils -class PolicydTest(test_utils.OpenStackBaseTest): +class PolicydTest(object): """Charm operation tests. The policyd test needs some config from the tests.yaml in order to work @@ -90,13 +86,13 @@ class PolicydTest(test_utils.OpenStackBaseTest): 'file1.yaml': "{'rule1': '!'}" } good_zip_path = self._make_zip_file_from('good.zip', good) - logging.info("About to attach the resource") + logging.info("Attaching a resource.") zaza_model.attach_resource(self.application_name, 'policyd-override', good_zip_path) - logging.info("... waiting for idle") + logging.debug("... waiting for idle") zaza_model.block_until_all_units_idle() - logging.info("Now setting config to true") + logging.debug("Now setting config to true") self._set_config(True) # check that the file gets to the right location path = os.path.join( @@ -109,10 +105,10 @@ class PolicydTest(test_utils.OpenStackBaseTest): logging.info("Checking for workload status line starts with PO:") zaza_model.block_until_wl_status_info_starts_with( self.application_name, "PO:") - logging.info("App status is valid") + logging.debug("App status is valid") # disable the policy override - logging.info("Disabling policy override ...") + logging.info("Disabling policy override by setting config to false") self._set_config(False) # check that the status no longer has "PO:" on it. # we have to do it twice due to async races and that some info lines @@ -131,3 +127,13 @@ class PolicydTest(test_utils.OpenStackBaseTest): zaza_model.block_until_file_missing(self.application_name, path) logging.info("...done") + + +class KeystonePolicydTest(PolicydTest, test_utils.OpenStackBaseTest): + pass + + +class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): + pass + + From 1337531926592439c6dbc5462d9c3ad00f802834 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 20 Sep 2019 14:51:57 +0200 Subject: [PATCH 040/898] Create specific policyd override for keystone --- zaza/openstack/charm_tests/policyd/tests.py | 49 +++++++++++++++++++-- zaza/openstack/utilities/exceptions.py | 6 +++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 8bdacbe..551e188 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -20,10 +20,14 @@ import shutil import tempfile import zipfile +import keystoneauth1 + import zaza.model as zaza_model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.keystone as ch_keystone +import zaza.openstack.utilities.exceptions as zaza_exceptions class PolicydTest(object): @@ -79,6 +83,15 @@ class PolicydTest(object): zfp.writestr(name, contents) return path + def _set_policy_with(self, rules): + rules_zip_path = self._make_zip_file_from('rules.zip', rules) + zaza_model.attach_resource(self.application_name, + 'policyd-override', + rules_zip_path) + self._set_config(True) + zaza_model.block_until_wl_status_info_starts_with( + self.application_name, "PO:", negate_match=True) + def test_policyd_good_yaml(self): # Test that the policyd with a good zipped yaml file puts the yaml file # in the right directory @@ -129,11 +142,39 @@ class PolicydTest(object): logging.info("...done") -class KeystonePolicydTest(PolicydTest, test_utils.OpenStackBaseTest): - pass +class KeystonePolicydTest(PolicydTest, + ch_keystone.BaseKeystoneTest, + test_utils.OpenStackBaseTest): + + def test_disable_service(self): + self._set_policy_with({"identity:get_auth_domains": "!"}) + with self.config_change( + {'preferred-api-version': self.default_api_version}, + {'preferred-api-version': '3'}, + application_name="keystone"): + for ip in self.keystone_ips: + try: + logging.info('keystone IP {}'.format(ip)) + ks_session = openstack_utils.get_keystone_session( + openstack_utils.get_overcloud_auth(address=ip)) + ks_client = openstack_utils.get_keystone_session_client( + ks_session) + ks_client.domains.list() + raise zaza_exceptions.PolicydError( + 'Retrieve domain list as admin with project scoped ' + 'token passed and should have failed. IP = {}' + .format(ip)) + except keystoneauth1.exceptions.http.Forbidden: + logging.info("keystone IP:{} policyd override working" + .format(ip)) + finally: + self._set_config(False) + zaza_model.block_until_wl_status_info_starts_with( + self.application_name, "PO:", negate_match=True) + zaza_model.block_until_all_units_idle() + + logging.info('OK') class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): pass - - diff --git a/zaza/openstack/utilities/exceptions.py b/zaza/openstack/utilities/exceptions.py index 868a7c6..28ff095 100644 --- a/zaza/openstack/utilities/exceptions.py +++ b/zaza/openstack/utilities/exceptions.py @@ -172,3 +172,9 @@ class NovaGuestRestartFailed(Exception): """Nova guest restart failed.""" pass + + +class PolicydError(Exception): + """Policyd override failed.""" + + pass From ae6b1d78497487c74122c63ed143fe61a3544b75 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 20 Sep 2019 14:56:19 +0200 Subject: [PATCH 041/898] Try to fix mixin class set up bug --- zaza/openstack/charm_tests/policyd/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 551e188..2b85c97 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -146,6 +146,10 @@ class KeystonePolicydTest(PolicydTest, ch_keystone.BaseKeystoneTest, test_utils.OpenStackBaseTest): + @classmethod + def setUpClass(cls, application_name=None): + super(KeystonePolicydTest, cls).setUpClass(application_name) + def test_disable_service(self): self._set_policy_with({"identity:get_auth_domains": "!"}) with self.config_change( From b1e6beb919699d2d9f031136695bd788fa019bcf Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 20 Sep 2019 15:18:10 +0200 Subject: [PATCH 042/898] Further work to get keystone specific test to work --- zaza/openstack/charm_tests/keystone/__init__.py | 2 +- zaza/openstack/charm_tests/policyd/tests.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/__init__.py b/zaza/openstack/charm_tests/keystone/__init__.py index 2f6dd1f..e244e16 100644 --- a/zaza/openstack/charm_tests/keystone/__init__.py +++ b/zaza/openstack/charm_tests/keystone/__init__.py @@ -30,7 +30,7 @@ class BaseKeystoneTest(test_utils.OpenStackBaseTest): """Base for Keystone charm tests.""" @classmethod - def setUpClass(cls): + def setUpClass(cls, application_name=None): """Run class setup for running Keystone charm operation tests.""" super(BaseKeystoneTest, cls).setUpClass(application_name='keystone') # Check if we are related to Vault TLS certificates diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 2b85c97..5605bc0 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -92,7 +92,7 @@ class PolicydTest(object): zaza_model.block_until_wl_status_info_starts_with( self.application_name, "PO:", negate_match=True) - def test_policyd_good_yaml(self): + def test_001_policyd_good_yaml(self): # Test that the policyd with a good zipped yaml file puts the yaml file # in the right directory good = { @@ -151,7 +151,9 @@ class KeystonePolicydTest(PolicydTest, super(KeystonePolicydTest, cls).setUpClass(application_name) def test_disable_service(self): - self._set_policy_with({"identity:get_auth_domains": "!"}) + logging.info("Doing policyd override to disable listing domains") + self._set_policy_with( + {'rule.yaml': "{'identity:get_auth_domains': '!'}"}) with self.config_change( {'preferred-api-version': self.default_api_version}, {'preferred-api-version': '3'}, From 61f7cce1d0e6636f5a793a08ef2d5e64855fb11e Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 23 Sep 2019 16:12:47 +0100 Subject: [PATCH 043/898] Fix keystone specific test --- zaza/openstack/charm_tests/policyd/tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 5605bc0..d2cf768 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -155,8 +155,10 @@ class KeystonePolicydTest(PolicydTest, self._set_policy_with( {'rule.yaml': "{'identity:get_auth_domains': '!'}"}) with self.config_change( - {'preferred-api-version': self.default_api_version}, - {'preferred-api-version': '3'}, + {'preferred-api-version': self.default_api_version, + 'use-policyd-override': 'False'}, + {'preferred-api-version': '3', + 'use-policyd-override': 'True'}, application_name="keystone"): for ip in self.keystone_ips: try: From 8a42af4e417e693b6bac31b84be660267a070433 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 23 Sep 2019 16:35:18 +0100 Subject: [PATCH 044/898] Fix keystone specific test - more --- zaza/openstack/charm_tests/policyd/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index d2cf768..ef09196 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -176,6 +176,7 @@ class KeystonePolicydTest(PolicydTest, logging.info("keystone IP:{} policyd override working" .format(ip)) finally: + return self._set_config(False) zaza_model.block_until_wl_status_info_starts_with( self.application_name, "PO:", negate_match=True) From 6a0867e2233e036b6e80f0712cf9482f9e0c081f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 23 Sep 2019 16:47:42 +0100 Subject: [PATCH 045/898] Fixed keystone spedific test --- zaza/openstack/charm_tests/policyd/tests.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index ef09196..6b6194a 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -175,12 +175,6 @@ class KeystonePolicydTest(PolicydTest, except keystoneauth1.exceptions.http.Forbidden: logging.info("keystone IP:{} policyd override working" .format(ip)) - finally: - return - self._set_config(False) - zaza_model.block_until_wl_status_info_starts_with( - self.application_name, "PO:", negate_match=True) - zaza_model.block_until_all_units_idle() logging.info('OK') From bdd225a1ee25153e6db39c75a04d991e1dcb5e77 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 24 Sep 2019 16:10:16 +0100 Subject: [PATCH 046/898] Switch keystone user to demo and try to list projects instead --- zaza/openstack/charm_tests/policyd/tests.py | 28 ++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 6b6194a..7965ca3 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -153,21 +153,37 @@ class KeystonePolicydTest(PolicydTest, def test_disable_service(self): logging.info("Doing policyd override to disable listing domains") self._set_policy_with( - {'rule.yaml': "{'identity:get_auth_domains': '!'}"}) + {'rule.yaml': "{'identity:list_projects': '!'}"}) with self.config_change( {'preferred-api-version': self.default_api_version, 'use-policyd-override': 'False'}, {'preferred-api-version': '3', 'use-policyd-override': 'True'}, application_name="keystone"): + zaza_model.block_until_all_units_idle() for ip in self.keystone_ips: try: logging.info('keystone IP {}'.format(ip)) - ks_session = openstack_utils.get_keystone_session( - openstack_utils.get_overcloud_auth(address=ip)) - ks_client = openstack_utils.get_keystone_session_client( - ks_session) - ks_client.domains.list() + openrc = { + 'API_VERSION': 3, + 'OS_USERNAME': DEMO_ADMIN_USER, + 'OS_PASSWORD': DEMO_ADMIN_USER_PASSWORD, + 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), + 'OS_USER_DOMAIN_NAME': DEMO_DOMAIN, + 'OS_DOMAIN_NAME': DEMO_DOMAIN, + } + if self.tls_rid: + openrc['OS_CACERT'] = \ + openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_AUTH_URL'] = ( + openrc['OS_AUTH_URL'].replace('http', 'https')) + logging.info('keystone IP {}'.format(ip)) + keystone_session = openstack_utils.get_keystone_session( + openrc, scope='DOMAIN') + keystone_client = ( + openstack_utils.get_keystone_session_client( + keystone_session)) + keystone_client.projects.list() raise zaza_exceptions.PolicydError( 'Retrieve domain list as admin with project scoped ' 'token passed and should have failed. IP = {}' From d4568caf93aba659746d7585cff5e1686baaf491 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 24 Sep 2019 17:15:40 +0100 Subject: [PATCH 047/898] Fix missing prefixes from other module --- zaza/openstack/charm_tests/policyd/tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 7965ca3..b4855dd 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -166,11 +166,11 @@ class KeystonePolicydTest(PolicydTest, logging.info('keystone IP {}'.format(ip)) openrc = { 'API_VERSION': 3, - 'OS_USERNAME': DEMO_ADMIN_USER, - 'OS_PASSWORD': DEMO_ADMIN_USER_PASSWORD, + 'OS_USERNAME': ch_keystone.DEMO_ADMIN_USER, + 'OS_PASSWORD': ch_keystone.DEMO_ADMIN_USER_PASSWORD, 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), - 'OS_USER_DOMAIN_NAME': DEMO_DOMAIN, - 'OS_DOMAIN_NAME': DEMO_DOMAIN, + 'OS_USER_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + 'OS_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, } if self.tls_rid: openrc['OS_CACERT'] = \ From 9d4008c9f47b6f435951c46d1c853cbf0bb3e26d Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 24 Sep 2019 17:59:15 +0100 Subject: [PATCH 048/898] More robust test --- zaza/openstack/charm_tests/policyd/tests.py | 43 ++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index b4855dd..b7de295 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -154,6 +154,47 @@ class KeystonePolicydTest(PolicydTest, logging.info("Doing policyd override to disable listing domains") self._set_policy_with( {'rule.yaml': "{'identity:list_projects': '!'}"}) + + # verify (with the config off) that we can actually access + # these points + with self.config_change( + {'preferred-api-version': self.default_api_version}, + {'preferred-api-version': '3'}, + application_name="keystone"): + zaza_model.block_until_all_units_idle() + for ip in self.keystone_ips: + try: + logging.info('keystone IP {}'.format(ip)) + openrc = { + 'API_VERSION': 3, + 'OS_USERNAME': ch_keystone.DEMO_ADMIN_USER, + 'OS_PASSWORD': ch_keystone.DEMO_ADMIN_USER_PASSWORD, + 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), + 'OS_USER_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + 'OS_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + } + if self.tls_rid: + openrc['OS_CACERT'] = \ + openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_AUTH_URL'] = ( + openrc['OS_AUTH_URL'].replace('http', 'https')) + logging.info('keystone IP {}'.format(ip)) + keystone_session = openstack_utils.get_keystone_session( + openrc, scope='DOMAIN') + keystone_client = ( + openstack_utils.get_keystone_session_client( + keystone_session)) + keystone_client.projects.list() + logging.info("keystone IP:{} without policyd override " + "projects list working" + .format(ip)) + except keystoneauth1.exceptions.http.Forbidden: + raise zaza_exceptions.PolicydError( + 'Retrieve project list as demo user with project ' + 'scoped token passed and should have passed. IP = {}' + .format(ip)) + + # now verify that the policy.d override does disable the endpoint with self.config_change( {'preferred-api-version': self.default_api_version, 'use-policyd-override': 'False'}, @@ -192,7 +233,7 @@ class KeystonePolicydTest(PolicydTest, logging.info("keystone IP:{} policyd override working" .format(ip)) - logging.info('OK') + logging.info('OK') class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): From 086076c6fc525cfd7af65bd1fbef3a34553e76e7 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 24 Sep 2019 18:34:50 +0100 Subject: [PATCH 049/898] Still trying to find a call that works with the user --- zaza/openstack/charm_tests/policyd/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index b7de295..7538cc9 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -184,13 +184,13 @@ class KeystonePolicydTest(PolicydTest, keystone_client = ( openstack_utils.get_keystone_session_client( keystone_session)) - keystone_client.projects.list() + keystone_client.services.list() logging.info("keystone IP:{} without policyd override " - "projects list working" + "services list working" .format(ip)) except keystoneauth1.exceptions.http.Forbidden: raise zaza_exceptions.PolicydError( - 'Retrieve project list as demo user with project ' + 'Retrieve services list as demo user with project ' 'scoped token passed and should have passed. IP = {}' .format(ip)) From 060d86fe27256d9bc0514a56747e5bc843b17b90 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 24 Sep 2019 19:01:32 +0100 Subject: [PATCH 050/898] Hopefully finalising the services test --- zaza/openstack/charm_tests/policyd/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 7538cc9..915c23f 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -153,7 +153,7 @@ class KeystonePolicydTest(PolicydTest, def test_disable_service(self): logging.info("Doing policyd override to disable listing domains") self._set_policy_with( - {'rule.yaml': "{'identity:list_projects': '!'}"}) + {'rule.yaml': "{'identity:list_services': '!'}"}) # verify (with the config off) that we can actually access # these points @@ -224,9 +224,9 @@ class KeystonePolicydTest(PolicydTest, keystone_client = ( openstack_utils.get_keystone_session_client( keystone_session)) - keystone_client.projects.list() + keystone_client.services.list() raise zaza_exceptions.PolicydError( - 'Retrieve domain list as admin with project scoped ' + 'Retrieve service list as admin with project scoped ' 'token passed and should have failed. IP = {}' .format(ip)) except keystoneauth1.exceptions.http.Forbidden: From ec74e6a57ca4945a8e31474d88202b71330266f5 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 24 Sep 2019 19:21:37 +0100 Subject: [PATCH 051/898] Switch the order of the tests --- zaza/openstack/charm_tests/policyd/tests.py | 80 ++++++++++----------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 915c23f..07f4753 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -155,46 +155,7 @@ class KeystonePolicydTest(PolicydTest, self._set_policy_with( {'rule.yaml': "{'identity:list_services': '!'}"}) - # verify (with the config off) that we can actually access - # these points - with self.config_change( - {'preferred-api-version': self.default_api_version}, - {'preferred-api-version': '3'}, - application_name="keystone"): - zaza_model.block_until_all_units_idle() - for ip in self.keystone_ips: - try: - logging.info('keystone IP {}'.format(ip)) - openrc = { - 'API_VERSION': 3, - 'OS_USERNAME': ch_keystone.DEMO_ADMIN_USER, - 'OS_PASSWORD': ch_keystone.DEMO_ADMIN_USER_PASSWORD, - 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), - 'OS_USER_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, - 'OS_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, - } - if self.tls_rid: - openrc['OS_CACERT'] = \ - openstack_utils.KEYSTONE_LOCAL_CACERT - openrc['OS_AUTH_URL'] = ( - openrc['OS_AUTH_URL'].replace('http', 'https')) - logging.info('keystone IP {}'.format(ip)) - keystone_session = openstack_utils.get_keystone_session( - openrc, scope='DOMAIN') - keystone_client = ( - openstack_utils.get_keystone_session_client( - keystone_session)) - keystone_client.services.list() - logging.info("keystone IP:{} without policyd override " - "services list working" - .format(ip)) - except keystoneauth1.exceptions.http.Forbidden: - raise zaza_exceptions.PolicydError( - 'Retrieve services list as demo user with project ' - 'scoped token passed and should have passed. IP = {}' - .format(ip)) - - # now verify that the policy.d override does disable the endpoint + # verify that the policy.d override does disable the endpoint with self.config_change( {'preferred-api-version': self.default_api_version, 'use-policyd-override': 'False'}, @@ -233,6 +194,45 @@ class KeystonePolicydTest(PolicydTest, logging.info("keystone IP:{} policyd override working" .format(ip)) + # now verify (with the config off) that we can actually access + # these points + with self.config_change( + {'preferred-api-version': self.default_api_version}, + {'preferred-api-version': '3'}, + application_name="keystone"): + zaza_model.block_until_all_units_idle() + for ip in self.keystone_ips: + try: + logging.info('keystone IP {}'.format(ip)) + openrc = { + 'API_VERSION': 3, + 'OS_USERNAME': ch_keystone.DEMO_ADMIN_USER, + 'OS_PASSWORD': ch_keystone.DEMO_ADMIN_USER_PASSWORD, + 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), + 'OS_USER_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + 'OS_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + } + if self.tls_rid: + openrc['OS_CACERT'] = \ + openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_AUTH_URL'] = ( + openrc['OS_AUTH_URL'].replace('http', 'https')) + logging.info('keystone IP {}'.format(ip)) + keystone_session = openstack_utils.get_keystone_session( + openrc, scope='DOMAIN') + keystone_client = ( + openstack_utils.get_keystone_session_client( + keystone_session)) + keystone_client.services.list() + logging.info("keystone IP:{} without policyd override " + "services list working" + .format(ip)) + except keystoneauth1.exceptions.http.Forbidden: + raise zaza_exceptions.PolicydError( + 'Retrieve services list as demo user with project ' + 'scoped token passed and should have passed. IP = {}' + .format(ip)) + logging.info('OK') From 6648aaf0e9f491cfef90afff215dd4f847933aa6 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 25 Sep 2019 12:03:21 +0100 Subject: [PATCH 052/898] Add negative test for broken yaml --- zaza/openstack/charm_tests/policyd/tests.py | 33 ++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 07f4753..0e0fdce 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -99,7 +99,7 @@ class PolicydTest(object): 'file1.yaml': "{'rule1': '!'}" } good_zip_path = self._make_zip_file_from('good.zip', good) - logging.info("Attaching a resource.") + logging.info("Attaching good zip file as a resource.") zaza_model.attach_resource(self.application_name, 'policyd-override', good_zip_path) @@ -141,6 +141,37 @@ class PolicydTest(object): logging.info("...done") + def test_002_policyd_bad_yaml(self): + # Test that a bad yaml file in the zip file doesn't not put it in the + # override and puts up a status message that it is bad + bad = { + "file2.yaml": "{'rule': '!}" + } + bad_zip_path = self._make_zip_file_from('bad.zip', bad) + logging.info("Attaching bad zip file as a resource") + zaza_model.attach_resource(self.application_name, + 'policyd-override', + bad_zip_path) + logging.debug("... waiting for idle") + zaza_model.block_until_all_units_idle() + logging.debug("Now setting config to true") + self._set_config(True) + # ensure that the workload status info line starts with PO (broken): + # to show that it didn't work + logging.info("Checking for workload status line starts with PO:") + zaza_model.block_until_wl_status_info_starts_with( + self.application_name, "PO (broken):") + logging.debug("App status is valid for broken yaml file") + zaza_model.block_until_all_units_idle() + # now verify that no file got landed on the machine + path = os.path.join( + "/etc", self._service_name, "policy.d", 'file2.yaml') + logging.info("Now checking that file {} is not present.".format(path)) + zaza_model.block_until_file_missing(self.application_name, path) + self._set_config(True) + zaza_model.block_until_all_units_idle() + logging.info("...done") + class KeystonePolicydTest(PolicydTest, ch_keystone.BaseKeystoneTest, From 13128db1f4236f42dac2b47de0f0994f58d3aaa4 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 25 Sep 2019 14:31:15 +0100 Subject: [PATCH 053/898] Add docs to policyd module --- zaza/openstack/charm_tests/policyd/tests.py | 47 ++++++++++++++++++--- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 0e0fdce..cc643d9 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -12,7 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Encapsulate policyd testing.""" +"""Encapsulate policyd testing. + +The Policyd Tests test the following: + +- Two general tests in the PolicydTest class that check that a policy zip can + drop policy files in the correct service policy.d directory. One test tests + that a valid yaml file is dropped; the 2nd that an invalid one is not dropped + and the workload info status line shows that it is broken. +- A custom policyd test that is per charm and tests that a policy zip file + attached does actually disable something in the associated service (i.e. + verify that the charm has implemented policy overrides and ensured that the + service actually picks them up). + +In order to use the generic tests, just include them in the specific test +class. The KeystonePolicydTest as an example does: + + class KeystonePolicydTest(PolicydTest, + ch_keystone.BaseKeystoneTest, + test_utils.OpenStackBaseTest): + + @classmethod + def setUpClass(cls, application_name=None): + super(KeystonePolicydTest, cls).setUpClass(application_name) + +Note that the generic test class (PolicyDTest) comes first, and then the +ch_keystone.BaseKeystoneTest, followed by the test_utils.OpenStackBaseTest. +This is to get the order of super().setUpClass(...) calls to work with +application_name. + +If a charm doesn't require a specific test, then the GenericPolicydTest class +can be used that just includes the two generic tests. The config in the +tests.yaml would stil be required. See the PolicydTest class docstring for +further details. +""" import logging import os @@ -103,7 +136,6 @@ class PolicydTest(object): zaza_model.attach_resource(self.application_name, 'policyd-override', good_zip_path) - logging.debug("... waiting for idle") zaza_model.block_until_all_units_idle() logging.debug("Now setting config to true") self._set_config(True) @@ -139,7 +171,7 @@ class PolicydTest(object): logging.info("Checking that {} has been removed".format(path)) zaza_model.block_until_file_missing(self.application_name, path) - logging.info("...done") + logging.info("OK") def test_002_policyd_bad_yaml(self): # Test that a bad yaml file in the zip file doesn't not put it in the @@ -152,13 +184,13 @@ class PolicydTest(object): zaza_model.attach_resource(self.application_name, 'policyd-override', bad_zip_path) - logging.debug("... waiting for idle") zaza_model.block_until_all_units_idle() logging.debug("Now setting config to true") self._set_config(True) # ensure that the workload status info line starts with PO (broken): # to show that it didn't work - logging.info("Checking for workload status line starts with PO:") + logging.info( + "Checking for workload status line starts with PO (broken):") zaza_model.block_until_wl_status_info_starts_with( self.application_name, "PO (broken):") logging.debug("App status is valid for broken yaml file") @@ -170,7 +202,7 @@ class PolicydTest(object): zaza_model.block_until_file_missing(self.application_name, path) self._set_config(True) zaza_model.block_until_all_units_idle() - logging.info("...done") + logging.info("OK") class KeystonePolicydTest(PolicydTest, @@ -222,7 +254,8 @@ class KeystonePolicydTest(PolicydTest, 'token passed and should have failed. IP = {}' .format(ip)) except keystoneauth1.exceptions.http.Forbidden: - logging.info("keystone IP:{} policyd override working" + logging.info("keystone IP:{} policyd override disabled " + "services listing by demo user" .format(ip)) # now verify (with the config off) that we can actually access From 3e2d98b1eda545b22d2fbfb5ca9dcd02cabbf8e4 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 25 Sep 2019 14:42:38 +0100 Subject: [PATCH 054/898] Fix pep8 failures --- zaza/openstack/charm_tests/policyd/__init__.py | 4 +++- zaza/openstack/charm_tests/policyd/tests.py | 13 +++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/__init__.py b/zaza/openstack/charm_tests/policyd/__init__.py index 89ec5f1..e8c1746 100644 --- a/zaza/openstack/charm_tests/policyd/__init__.py +++ b/zaza/openstack/charm_tests/policyd/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Collection of code for setting up and testing policyd overrides across a +"""Policyd. + +Collection of code for setting up and testing policyd overrides across a collection of charms. """ diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index cc643d9..9e76654 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -77,6 +77,7 @@ class PolicydTest(object): @classmethod def setUpClass(cls, application_name=None): + """Run class setup for running Policyd charm operation tests.""" super(PolicydTest, cls).setUpClass(application_name) if (openstack_utils.get_os_release() < openstack_utils.get_os_release('xenial_queens')): @@ -87,6 +88,7 @@ class PolicydTest(object): @classmethod def tearDownClass(cls): + """Run class tearDown for running Policyd charm operation tests.""" super(PolicydTest, cls).tearDownClass() try: shutil.rmtree(cls._tmp_dir, ignore_errors=True) @@ -126,8 +128,7 @@ class PolicydTest(object): self.application_name, "PO:", negate_match=True) def test_001_policyd_good_yaml(self): - # Test that the policyd with a good zipped yaml file puts the yaml file - # in the right directory + """Test that the policyd with a good zipped yaml file.""" good = { 'file1.yaml': "{'rule1': '!'}" } @@ -174,8 +175,7 @@ class PolicydTest(object): logging.info("OK") def test_002_policyd_bad_yaml(self): - # Test that a bad yaml file in the zip file doesn't not put it in the - # override and puts up a status message that it is bad + """Test bad yaml file in the zip file is handled.""" bad = { "file2.yaml": "{'rule': '!}" } @@ -208,12 +208,15 @@ class PolicydTest(object): class KeystonePolicydTest(PolicydTest, ch_keystone.BaseKeystoneTest, test_utils.OpenStackBaseTest): + """Specific test for policyd for keystone charm.""" @classmethod def setUpClass(cls, application_name=None): + """Run class setup for running KeystonePolicydTest tests.""" super(KeystonePolicydTest, cls).setUpClass(application_name) def test_disable_service(self): + """Test that service can be disabled.""" logging.info("Doing policyd override to disable listing domains") self._set_policy_with( {'rule.yaml': "{'identity:list_services': '!'}"}) @@ -301,4 +304,6 @@ class KeystonePolicydTest(PolicydTest, class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): + """Generic policyd test for any charm without a specific test.""" + pass From ab89054847f40b6fcbe5e919a732373bc6187db0 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Wed, 25 Sep 2019 16:08:20 +0200 Subject: [PATCH 055/898] enable guest interface plumbing for bionic-queens and later --- zaza/openstack/configure/network.py | 24 +++--- zaza/openstack/utilities/openstack.py | 102 ++++++++++++++------------ 2 files changed, 67 insertions(+), 59 deletions(-) diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 542c595..50405c0 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -212,23 +212,23 @@ def setup_gateway_ext_port(network_config, keystone_session=None): else: net_id = None + # If we're using netplan, we need to add the new interface to the guest + current_release = openstack_utils.get_os_release() + bionic_queens = openstack_utils.get_os_release('bionic_queens') + if current_release >= bionic_queens: + logging.warn("Adding second interface for dataport to guest netplan " + "for bionic-queens and later") + plumb_guest_nic = True + else: + plumb_guest_nic = False + logging.info("Configuring network for OpenStack undercloud/provider") openstack_utils.configure_gateway_ext_port( nova_client, neutron_client, dvr_mode=network_config.get("dvr_enabled", False), - net_id=net_id) - - current_release = openstack_utils.get_os_release() - bionic_queens = openstack_utils.get_os_release('bionic_queens') - if current_release >= bionic_queens: - logging.warn("Adding second interface for dataport to guest netplan, \ - for bionic-queens and later") - openstack_utils.plumb_guest_nic( - nova_client, - neutron_client, - dvr_mode=network_config.get("dvr_enabled", False), - net_id=net_id) + net_id=net_id, + plumb_guest_nic=plumb_guest_nic) def run_from_cli(**kwargs): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 87c54ef..729f524 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -484,62 +484,67 @@ def get_admin_net(neutron_client): return net -def plumb_guest_nic(novaclient, neutronclient, - dvr_mode=None, net_id=None): +def _plumb_guest_nic(server_name, mac_address, dvr_mode=None): + """In guest server_name, add port with mac_address to netplan - """Add port associated with net_id to netplan - - :type net_id: string + :param server_name: Hostname of instance + :type server_name: string + :param mac_address: mac address of nic to be added to netplan + :type mac_address: string """ if dvr_mode: - uuids = get_ovs_uuids() + application_name = 'neutron-openvswitch' else: - uuids = get_gateway_uuids() + application_name = 'neutron-gateway' - for uuid in uuids: - server = novaclient.servers.get(uuid) - ext_port_name = "{}_ext-port".format(server.name) - for port in neutronclient.list_ports(device_id=server.id)['ports']: - if port['name'] == ext_port_name: - logging.info('Adding second port to netplan in guest:\ - {}'.format(port['name'])) - mac_address = port['mac_address'] - if dvr_mode: - application_name = 'neutron-openvswitch' - else: - application_name = 'neutron-gateway' - unit_name = juju_utils.get_unit_name_from_ip_address( - server.ip, application_name) - interface = model.async_run_on_unit( - unit_name, "ip addr|grep\ - {}".format( - mac_address)).split("\n")[0].split(" ")[2] - body_value = textwrap.dedent("""\ - network: - ethernets: - {}: - dhcp4: false - dhcp6: true - optional: true - match: - macaddress: {} - set-name: {} - \n - version: 2 - """.format(interface, mac_address, interface)) - netplan_file = open("60-dataport.yaml", "w") - netplan_file.write(body_value) - netplan_file.close() - model.async_scp_to_unit(unit_name, './60-dataport.yaml', - '/etc/netplan/', user="root") - subprocess.call("rm 60-dataport.yaml") - model.async_run_on_unit(unit_name, "netplan apply", - user="root") + unit_name = juju_utils.get_unit_name_from_host_name( + server_name, application_name) + + run_cmd_nic = "ip addr|grep {} -B1".format(mac_address) + interface = model.run_on_unit(unit_name, run_cmd_nic) + interface = interface['Stdout'].split(' ')[1].replace(':', '') + + run_cmd_netplan = """sudo egrep -iR '{}|{}' /etc/netplan/ + """.format(mac_address, interface) + + netplancfg = model.run_on_unit(unit_name, run_cmd_netplan) + + if (mac_address in str(netplancfg)) or (interface in str(netplancfg)): + logging.warn("mac address {} or nic {} already exists in " + "/etc/netplan".format(mac_address, interface)) + return + body_value = textwrap.dedent("""\ + network: + ethernets: + {0}: + dhcp4: false + dhcp6: true + optional: true + match: + macaddress: {1} + set-name: {0} + version: 2 + """.format(interface, mac_address)) + logging.debug("plumb guest interfae debug info:") + logging.debug("body_value: {}\nunit_name: {}\ninterface: {}\nmac_address:" + "{}\nserver_name: {}".format(body_value, unit_name, + interface, mac_address, + server_name)) + netplan_file = open("60-dataport.yaml", "w") + netplan_file.write(body_value) + netplan_file.close() + model.scp_to_unit(unit_name, './60-dataport.yaml', + '/home/ubuntu/', user="ubuntu") + run_cmd_mv = "sudo mv /home/ubuntu/60-dataport.yaml /etc/netplan" + model.run_on_unit(unit_name, run_cmd_mv) + subprocess.call(["rm", "60-dataport.yaml"]) + model.run_on_unit(unit_name, "sudo netplan apply") def configure_gateway_ext_port(novaclient, neutronclient, - dvr_mode=None, net_id=None): + dvr_mode=None, net_id=None, + plumb_guest_nic=False): """Configure the neturong-gateway external port. :param novaclient: Authenticated novaclient @@ -586,6 +591,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, port = neutronclient.create_port(body=body_value) server.interface_attach(port_id=port['port']['id'], net_id=None, fixed_ip=None) + if plumb_guest_nic: + _plumb_guest_nic(server.name, mac_address=port['mac_address'], + dvr_mode=dvr_mode) ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: if 'ext-port' in port['name']: From 889053a3dfff0005ba3970de02dfdbefd39ab9af Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Wed, 25 Sep 2019 16:14:37 +0200 Subject: [PATCH 056/898] fix docstring --- zaza/openstack/utilities/openstack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 729f524..464d49c 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -485,14 +485,13 @@ def get_admin_net(neutron_client): def _plumb_guest_nic(server_name, mac_address, dvr_mode=None): - """In guest server_name, add port with mac_address to netplan + """In guest server_name, add port with mac_address to netplan. :param server_name: Hostname of instance :type server_name: string :param mac_address: mac address of nic to be added to netplan :type mac_address: string """ - if dvr_mode: application_name = 'neutron-openvswitch' else: From 36e9e45f0b6c35a20dbe10d32dcef2a8950c4cdb Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Wed, 25 Sep 2019 16:17:08 +0200 Subject: [PATCH 057/898] add dvr_mode to docstring --- zaza/openstack/utilities/openstack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 464d49c..3eae559 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -491,6 +491,8 @@ def _plumb_guest_nic(server_name, mac_address, dvr_mode=None): :type server_name: string :param mac_address: mac address of nic to be added to netplan :type mac_address: string + :param dvr_mode: Using DVR mode or not + :type dvr_mode: boolean """ if dvr_mode: application_name = 'neutron-openvswitch' From 48a410151efcff47cbfa1b0be137528d922783d6 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Wed, 25 Sep 2019 16:19:12 +0200 Subject: [PATCH 058/898] fix typo --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 3eae559..dd0fd42 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -527,7 +527,7 @@ def _plumb_guest_nic(server_name, mac_address, dvr_mode=None): set-name: {0} version: 2 """.format(interface, mac_address)) - logging.debug("plumb guest interfae debug info:") + logging.debug("plumb guest interface debug info:") logging.debug("body_value: {}\nunit_name: {}\ninterface: {}\nmac_address:" "{}\nserver_name: {}".format(body_value, unit_name, interface, mac_address, From 2a151168680e1bbe983a2f29604267094ff4d774 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Sep 2019 17:44:14 +0000 Subject: [PATCH 059/898] Add aodh api test --- zaza/openstack/charm_tests/aodh/tests.py | 41 +++++++++++ zaza/openstack/configure/telemetry.py | 93 +++++++++++++++++++++++- zaza/openstack/utilities/openstack.py | 12 +++ 3 files changed, 145 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index cabbac3..6276bfd 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -18,13 +18,17 @@ import logging +import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.configure.telemetry as telemetry_utils class AodhTest(test_utils.OpenStackBaseTest): """Encapsulate Aodh tests.""" + RESOURCE_PREFIX = 'zaza-aodhtests' + @classmethod def setUpClass(cls): """Run class setup for running tests.""" @@ -33,6 +37,27 @@ class AodhTest(test_utils.OpenStackBaseTest): cls.xenial_newton = openstack_utils.get_os_release('xenial_newton') cls.bionic_stein = openstack_utils.get_os_release('bionic_stein') cls.release = openstack_utils.get_os_release() + cls.keystone_session = openstack_utils.get_overcloud_keystone_session() + cls.model_name = zaza.model.get_juju_model() + cls.aodh_client = openstack_utils.get_aodh_session_client( + cls.keystone_session) + + @classmethod + def tearDown(cls): + """Remove test resources.""" + logging.info('Running teardown') + cache_wait = False + for alarm in cls.aodh_client.alarm.list(): + if alarm['name'].startswith(cls.RESOURCE_PREFIX): + cache_wait = True + logging.info('Removing Alarm {}'.format(alarm['name'])) + telemetry_utils.delete_alarm( + cls.aodh_client, + alarm['name'], + cache_wait=False) + if cache_wait: + logging.info('Waiting for alarm cache to clear') + telemetry_utils.alarm_cache_wait() @property def services(self): @@ -60,6 +85,22 @@ class AodhTest(test_utils.OpenStackBaseTest): 'aodh-listener'] return services + def test_100_test_api(self): + """Check api by creating an alarm.""" + alarm_name = '{}_test_api_alarm'.format(self.RESOURCE_PREFIX) + logging.info('Creating alarm {}'.format(alarm_name)) + alarm = telemetry_utils.create_server_power_off_alarm( + self.aodh_client, + alarm_name, + 'some-uuid') + alarm_state = telemetry_utils.get_alarm_state( + self.aodh_client, + alarm['alarm_id']) + logging.info('alarm_state: {}'.format(alarm_state)) + # Until data is collected alarm come up in an 'insufficient data' + # state. + self.assertEqual(alarm_state, 'insufficient data') + def test_900_restart_on_config_change(self): """Checking restart happens on config change. diff --git a/zaza/openstack/configure/telemetry.py b/zaza/openstack/configure/telemetry.py index 144d447..86102ac 100644 --- a/zaza/openstack/configure/telemetry.py +++ b/zaza/openstack/configure/telemetry.py @@ -18,13 +18,104 @@ Functions for managing masakari resources and simulating compute node loss and recovery. """ +import time + import zaza.model def ceilometer_upgrade(application_name=None, model_name=None): - """Run ceilometer upgrade action.""" + """Run ceilometer upgrade action. + + :param application_name: Name of application to run action against. + :type application_name: str + :param model_name: Name of model application_name resides in. + :type model_name: str + """ zaza.model.run_action_on_leader( application_name, 'ceilometer-upgrade', model_name=model_name, action_params={}) + + +def get_alarm(aodh_client, alarm_name): + """Return the alarm with the given name. + + :param aodh_client: Authenticated aodh v2 client + :type aodh_client: aodhclient.v2.client.Client + :param alarm_name: Name of alarm to search for + :type alarm_name: str + :returns: Returns a dict of alarm data. + :rtype: {} or None + """ + for alarm in aodh_client.alarm.list(): + if alarm['name'] == alarm_name: + return alarm + return None + + +def alarm_cache_wait(): + """Wait for alarm cache to clear.""" + # AODH has an alarm cache (see event_alarm_cache_ttl in aodh.conf). This + # means deleted alarms can persist and fire. The default is 60s and is + # currently not configrable via the charm so 61s is a safe assumption. + time.sleep(61) + + +def delete_alarm(aodh_client, alarm_name, cache_wait=False): + """Delete alarm with given name. + + :param aodh_client: Authenticated aodh v2 client + :type aodh_client: aodhclient.v2.client.Client + :param alarm_name: Name of alarm to delete + :type alarm_name: str + :param cache_wait: Whether to wait for cache to clear after deletion. + :type cache_wait: bool + """ + alarm = get_alarm(aodh_client, alarm_name) + if alarm: + aodh_client.alarm.delete(alarm['alarm_id']) + if cache_wait: + alarm_cache_wait() + + +def get_alarm_state(aodh_client, alarm_id): + """Return the state of the alarm with the given name. + + :param aodh_client: Authenticated aodh v2 client + :type aodh_client: aodhclient.v2.client.Client + :param alarm_id: ID of provided alarm + :param alarm_id: str + :returns: State of given alarm + :rtype: str + """ + alarm = aodh_client.alarm.get(alarm_id) + return alarm['state'] + + +def create_server_power_off_alarm(aodh_client, alarm_name, server_uuid): + """Create an alarm which triggers when an instance powers off. + + :param aodh_client: Authenticated aodh v2 client + :type aodh_client: aodhclient.v2.client.Client + :param alarm_name: Name of alarm to delete + :type alarm_name: str + :param server_uuid: UUID of server to monitor + :type server_uuid: str + :returns: Dict of alarm data + :rtype: {} + """ + alarm_def = { + 'type': 'event', + 'name': alarm_name, + 'description': 'Instance powered OFF', + 'alarm_actions': ['log://'], + 'ok_actions': ['log://'], + 'insufficient_data_actions': ['log://'], + 'event_rule': { + 'event_type': 'compute.instance.power_off.*', + 'query': [{'field': 'traits.instance_id', + 'op': 'eq', + 'type': 'string', + 'value': server_uuid}]}} + return aodh_client.alarm.create(alarm_def) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 6769de6..f4b36c4 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -25,6 +25,7 @@ from .os_versions import ( from openstack import connection +from aodhclient.v2 import client as aodh_client from cinderclient import client as cinderclient from glanceclient import Client as GlanceClient @@ -288,6 +289,17 @@ def get_masakari_session_client(session, interface='internal', return conn.instance_ha +def get_aodh_session_client(session): + """Return aodh client authenticated by keystone session. + + :param session: Keystone session object + :type session: keystoneauth1.session.Session object + :returns: Authenticated aodh client + :rtype: openstack.instance_ha.v1._proxy.Proxy + """ + return aodh_client.Client(session=session) + + def get_keystone_scope(): """Return Keystone scope based on OpenStack release of the overcloud. From f5521ca6ea70bb67916a3c1f4719157f7c166dce Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 26 Sep 2019 09:17:53 +0000 Subject: [PATCH 060/898] Add aodhclient as a dependancy --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0980b96..20679d4 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ install_require = [ 'PyYAML', 'tenacity', 'oslo.config', + 'aodhclient', 'python-glanceclient', 'python-keystoneclient', 'python-novaclient', @@ -102,4 +103,4 @@ setup( 'testing': tests_require, }, tests_require=tests_require, -) \ No newline at end of file +) From 3ded7dab36351672ec0a9cc10a42b8b737f8a599 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 26 Sep 2019 09:27:11 +0000 Subject: [PATCH 061/898] Fix copy/pasta error --- zaza/openstack/charm_tests/aodh/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index 6276bfd..0ba24af 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Encapsulate masakari testing.""" +"""Encapsulate Aodh testing.""" import logging From 3752f2cc1291437b048612ad9337e826ebe6ec65 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 26 Sep 2019 10:23:56 +0000 Subject: [PATCH 062/898] Remove duplicate code for running ceilometer upgrade and fix typo --- zaza/openstack/charm_tests/aodh/__init__.py | 2 +- zaza/openstack/charm_tests/aodh/setup.py | 22 --------------------- 2 files changed, 1 insertion(+), 23 deletions(-) delete mode 100644 zaza/openstack/charm_tests/aodh/setup.py diff --git a/zaza/openstack/charm_tests/aodh/__init__.py b/zaza/openstack/charm_tests/aodh/__init__.py index 67c8b3b..2781b7b 100644 --- a/zaza/openstack/charm_tests/aodh/__init__.py +++ b/zaza/openstack/charm_tests/aodh/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Collection of code for setting up and testing masakari.""" +"""Collection of code for setting up and testing aodh.""" diff --git a/zaza/openstack/charm_tests/aodh/setup.py b/zaza/openstack/charm_tests/aodh/setup.py deleted file mode 100644 index cb02ecb..0000000 --- a/zaza/openstack/charm_tests/aodh/setup.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Code for setting up aodh tests.""" - -import zaza.openstack.configure.telemetry - - -def ceilometer_upgrade(): - """Run ceilometer upgrade action.""" - zaza.openstack.configure.telemetry.ceilometer_upgrade('ceilometer') From 028602f3659ed97b23971c14c6c03d7f4ba148dc Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Thu, 26 Sep 2019 13:47:19 +0200 Subject: [PATCH 063/898] change terminology from plumb --- zaza/openstack/configure/network.py | 6 +++--- zaza/openstack/utilities/openstack.py | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 50405c0..55bcdf3 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -218,9 +218,9 @@ def setup_gateway_ext_port(network_config, keystone_session=None): if current_release >= bionic_queens: logging.warn("Adding second interface for dataport to guest netplan " "for bionic-queens and later") - plumb_guest_nic = True + add_dataport_to_netplan = True else: - plumb_guest_nic = False + add_dataport_to_netplan = False logging.info("Configuring network for OpenStack undercloud/provider") openstack_utils.configure_gateway_ext_port( @@ -228,7 +228,7 @@ def setup_gateway_ext_port(network_config, keystone_session=None): neutron_client, dvr_mode=network_config.get("dvr_enabled", False), net_id=net_id, - plumb_guest_nic=plumb_guest_nic) + add_dataport_to_netplan=add_dataport_to_netplan) def run_from_cli(**kwargs): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index dd0fd42..94e86a8 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -484,8 +484,8 @@ def get_admin_net(neutron_client): return net -def _plumb_guest_nic(server_name, mac_address, dvr_mode=None): - """In guest server_name, add port with mac_address to netplan. +def add_interface_to_netplan(server_name, mac_address, dvr_mode=None): + """In guest server_name, add nic with mac_address to netplan. :param server_name: Hostname of instance :type server_name: string @@ -545,7 +545,7 @@ def _plumb_guest_nic(server_name, mac_address, dvr_mode=None): def configure_gateway_ext_port(novaclient, neutronclient, dvr_mode=None, net_id=None, - plumb_guest_nic=False): + add_dataport_to_netplan=False): """Configure the neturong-gateway external port. :param novaclient: Authenticated novaclient @@ -592,9 +592,10 @@ def configure_gateway_ext_port(novaclient, neutronclient, port = neutronclient.create_port(body=body_value) server.interface_attach(port_id=port['port']['id'], net_id=None, fixed_ip=None) - if plumb_guest_nic: - _plumb_guest_nic(server.name, mac_address=port['mac_address'], - dvr_mode=dvr_mode) + if add_dataport_to_netplan: + add_interface_to_netplan(server.name, + mac_address=port['mac_address'], + dvr_mode=dvr_mode) ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: if 'ext-port' in port['name']: From 6f903b73a6ea6d71d0deb4556de65b69901b9f6d Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Thu, 26 Sep 2019 13:59:59 +0200 Subject: [PATCH 064/898] fix interface and mac address matching --- zaza/openstack/utilities/openstack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 94e86a8..e966716 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -502,11 +502,11 @@ def add_interface_to_netplan(server_name, mac_address, dvr_mode=None): unit_name = juju_utils.get_unit_name_from_host_name( server_name, application_name) - run_cmd_nic = "ip addr|grep {} -B1".format(mac_address) + run_cmd_nic = "ip -f link -br -o addr|grep {}".format(mac_address) interface = model.run_on_unit(unit_name, run_cmd_nic) - interface = interface['Stdout'].split(' ')[1].replace(':', '') + interface = interface['Stdout'].split(' ')[0] - run_cmd_netplan = """sudo egrep -iR '{}|{}' /etc/netplan/ + run_cmd_netplan = """sudo egrep -iR '{}|{}$' /etc/netplan/ """.format(mac_address, interface) netplancfg = model.run_on_unit(unit_name, run_cmd_netplan) From 1af831399a8f84f432318602aa7eac2e965c1e98 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Thu, 26 Sep 2019 14:16:19 +0200 Subject: [PATCH 065/898] add context manager for netplan_file --- zaza/openstack/utilities/openstack.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index e966716..4acdc00 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -532,15 +532,14 @@ def add_interface_to_netplan(server_name, mac_address, dvr_mode=None): "{}\nserver_name: {}".format(body_value, unit_name, interface, mac_address, server_name)) - netplan_file = open("60-dataport.yaml", "w") - netplan_file.write(body_value) - netplan_file.close() - model.scp_to_unit(unit_name, './60-dataport.yaml', - '/home/ubuntu/', user="ubuntu") - run_cmd_mv = "sudo mv /home/ubuntu/60-dataport.yaml /etc/netplan" - model.run_on_unit(unit_name, run_cmd_mv) - subprocess.call(["rm", "60-dataport.yaml"]) - model.run_on_unit(unit_name, "sudo netplan apply") + with tempfile.NamedTemporaryFile(mode="w") as netplan_file: + netplan_file.write(body_value) + netplan_file.flush() + model.scp_to_unit(unit_name, netplan_file.name, + '/home/ubuntu/60-dataport.yaml', user="ubuntu") + run_cmd_mv = "sudo mv /home/ubuntu/60-dataport.yaml /etc/netplan/" + model.run_on_unit(unit_name, run_cmd_mv) + model.run_on_unit(unit_name, "sudo netplan apply") def configure_gateway_ext_port(novaclient, neutronclient, From 5ad18dfc2fddbecbbbcb9d2d7d07943b0c50ab0a Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Thu, 26 Sep 2019 14:19:37 +0200 Subject: [PATCH 066/898] fix context mgr --- zaza/openstack/utilities/openstack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 4acdc00..3956063 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -537,9 +537,9 @@ def add_interface_to_netplan(server_name, mac_address, dvr_mode=None): netplan_file.flush() model.scp_to_unit(unit_name, netplan_file.name, '/home/ubuntu/60-dataport.yaml', user="ubuntu") - run_cmd_mv = "sudo mv /home/ubuntu/60-dataport.yaml /etc/netplan/" - model.run_on_unit(unit_name, run_cmd_mv) - model.run_on_unit(unit_name, "sudo netplan apply") + run_cmd_mv = "sudo mv /home/ubuntu/60-dataport.yaml /etc/netplan/" + model.run_on_unit(unit_name, run_cmd_mv) + model.run_on_unit(unit_name, "sudo netplan apply") def configure_gateway_ext_port(novaclient, neutronclient, From f64d8fe70e6b0d218307ad9ee8104622bb74a5f2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sat, 28 Sep 2019 06:48:25 +0000 Subject: [PATCH 067/898] Stop basic_setup from raising an exception The ceilometer setup script is a setup script not a test so it should not raise a skip exception if it is not needed. The fact that it does is my fault and was implemented on my recommendation. --- zaza/openstack/charm_tests/ceilometer/setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceilometer/setup.py b/zaza/openstack/charm_tests/ceilometer/setup.py index 1cd9b2d..c966077 100644 --- a/zaza/openstack/charm_tests/ceilometer/setup.py +++ b/zaza/openstack/charm_tests/ceilometer/setup.py @@ -17,7 +17,6 @@ """Code for configuring Ceilometer.""" import logging -import unittest import zaza.model as zaza_model import zaza.openstack.utilities.openstack as openstack_utils @@ -32,8 +31,9 @@ def basic_setup(): xenial_pike = openstack_utils.get_os_release('xenial_pike') if current_release < xenial_pike: - raise unittest.SkipTest('Skipping ceilometer-upgrade as it is not ' - 'supported before Pike') + logging.info( + 'Skipping ceilometer-upgrade as it is not supported before Pike') + return logging.debug('Checking ceilometer-upgrade') From b735c5885ff62e6f99c0786fba590d2a14c69b89 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Mon, 30 Sep 2019 11:40:14 +0200 Subject: [PATCH 068/898] add tenacity with backoff to configure ext dataport method --- zaza/openstack/utilities/openstack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index f0b7ffc..706336b 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -559,6 +559,8 @@ def add_interface_to_netplan(server_name, mac_address, dvr_mode=None): model.run_on_unit(unit_name, "sudo netplan apply") +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), + reraise=True, stop=tenacity.stop_after_attempt(12)) def configure_gateway_ext_port(novaclient, neutronclient, dvr_mode=None, net_id=None, add_dataport_to_netplan=False): From 903b612ba9e4939497dfd6a65320cdf4c61cdd57 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Mon, 30 Sep 2019 12:57:17 +0200 Subject: [PATCH 069/898] move mac_address retry to smaller method --- zaza/openstack/utilities/openstack.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 706336b..60fb554 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -559,8 +559,6 @@ def add_interface_to_netplan(server_name, mac_address, dvr_mode=None): model.run_on_unit(unit_name, "sudo netplan apply") -@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), - reraise=True, stop=tenacity.stop_after_attempt(12)) def configure_gateway_ext_port(novaclient, neutronclient, dvr_mode=None, net_id=None, add_dataport_to_netplan=False): @@ -612,7 +610,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, fixed_ip=None) if add_dataport_to_netplan: add_interface_to_netplan(server.name, - mac_address=port['mac_address'], + mac_address=get_mac_from_port(port), dvr_mode=dvr_mode) ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: @@ -642,6 +640,12 @@ def configure_gateway_ext_port(novaclient, neutronclient, juju_wait.wait(wait_for_workload=True) +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), + reraise=True, retry=tenacity.retry_if_exception_type(KeyError)) +def get_mac_from_port(port): + return port['mac_address'] + + def create_project_network(neutron_client, project_id, net_name='private', shared=False, network_type='gre', domain=None): """Create the project network. From 443b2295c3a36aa641727ba8c9204c12e22e0724 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Mon, 30 Sep 2019 13:00:19 +0200 Subject: [PATCH 070/898] add doctstring --- zaza/openstack/utilities/openstack.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 60fb554..f6fb66e 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -643,6 +643,11 @@ def configure_gateway_ext_port(novaclient, neutronclient, @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, retry=tenacity.retry_if_exception_type(KeyError)) def get_mac_from_port(port): + """Get mac address from port, with tenacity due to openstack async. + + :param port: neutron port + :type port: string + """ return port['mac_address'] From e5e7d03e1fcfa996a41ed5f80147350fafcd04ca Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Mon, 30 Sep 2019 13:04:12 +0200 Subject: [PATCH 071/898] fix doctstring --- zaza/openstack/utilities/openstack.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index f6fb66e..1e533d4 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -646,7 +646,9 @@ def get_mac_from_port(port): """Get mac address from port, with tenacity due to openstack async. :param port: neutron port - :type port: string + :type port: neutron port + :returns: mac address + :rtype: string """ return port['mac_address'] From ea84bf9ed8410fecd9c83f80ee07f23b0ea28a52 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 30 Sep 2019 12:31:50 +0000 Subject: [PATCH 072/898] Check aodh api after a change. The first query against the aodh api after a restart seems to fail. So, check the api after a restart with a tenacity wrapper. --- zaza/openstack/charm_tests/aodh/tests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index 0ba24af..f43615f 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -17,6 +17,7 @@ """Encapsulate Aodh testing.""" import logging +import tenacity import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils @@ -59,6 +60,12 @@ class AodhTest(test_utils.OpenStackBaseTest): logging.info('Waiting for alarm cache to clear') telemetry_utils.alarm_cache_wait() + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), + reraise=True, stop=tenacity.stop_after_attempt(8)) + def query_aodh_api(self): + """Check that aodh api is responding.""" + self.aodh_client.alarm.list() + @property def services(self): """Return a list of the service that should be running.""" @@ -122,6 +129,7 @@ class AodhTest(test_utils.OpenStackBaseTest): {'DEFAULT': {'debug': ['False']}}, {'DEFAULT': {'debug': ['True']}}, self.services) + self.query_aodh_api() def test_901_pause_resume(self): """Run pause and resume tests. @@ -133,3 +141,4 @@ class AodhTest(test_utils.OpenStackBaseTest): self.services, pgrep_full=False): logging.info("Testing pause resume") + self.query_aodh_api() From 64ddda0541d4b2d6adc039a0ee0ab03653fbb1a7 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 30 Sep 2019 13:24:08 +0000 Subject: [PATCH 073/898] Disable add_dataport_to_netplan while issue #72 is investogated --- zaza/openstack/utilities/openstack.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 1e533d4..3a7f22c 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -608,10 +608,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, port = neutronclient.create_port(body=body_value) server.interface_attach(port_id=port['port']['id'], net_id=None, fixed_ip=None) - if add_dataport_to_netplan: - add_interface_to_netplan(server.name, - mac_address=get_mac_from_port(port), - dvr_mode=dvr_mode) + # Temporarily disable call to add_interface_to_netplan as it + # sometimes throws a KeyError + # https://github.com/openstack-charmers/zaza-openstack-tests/issues/72 ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: if 'ext-port' in port['name']: From 2a917f59084d35db552f3f516b8c20c1d10764ce Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Mon, 30 Sep 2019 16:32:10 +0200 Subject: [PATCH 074/898] refresh port in get mac method --- zaza/openstack/utilities/openstack.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 1e533d4..ab600a0 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -610,7 +610,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, fixed_ip=None) if add_dataport_to_netplan: add_interface_to_netplan(server.name, - mac_address=get_mac_from_port(port), + mac_address=get_mac_from_port(port, neutronclient), dvr_mode=dvr_mode) ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: @@ -642,15 +642,20 @@ def configure_gateway_ext_port(novaclient, neutronclient, @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, retry=tenacity.retry_if_exception_type(KeyError)) -def get_mac_from_port(port): +def get_mac_from_port(port, neutronclient): """Get mac address from port, with tenacity due to openstack async. :param port: neutron port :type port: neutron port + :param neutronclient: Authenticated neutronclient + :type neutronclient: neutronclient.Client object :returns: mac address :rtype: string """ - return port['mac_address'] + logging.info("Trying to get mac address from port:" + "{}".format(port['id'])) + refresh_port = neutronclient.show_port(port['id'])['port'] + return refresh_port['mac_address'] def create_project_network(neutron_client, project_id, net_name='private', From 34cf2f1914c41f4e43fac78903072a806a98a40a Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Tue, 1 Oct 2019 10:14:44 +0200 Subject: [PATCH 075/898] disable dataport addition --- zaza/openstack/utilities/openstack.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 465d73c..4b9a4e0 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -608,11 +608,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, port = neutronclient.create_port(body=body_value) server.interface_attach(port_id=port['port']['id'], net_id=None, fixed_ip=None) - if add_dataport_to_netplan: - mac_address = get_mac_from_port(port, neutronclient) - add_interface_to_netplan(server.name, - mac_address=mac_address, - dvr_mode=dvr_mode) + # Temporarily disable call to add_interface_to_netplan as it + # sometimes throws a KeyError + # https://github.com/openstack-charmers/zaza-openstack-tests/issues/72 ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: if 'ext-port' in port['name']: From 55ee8ffe3faea91a6aa82fda5430ffc0f9a65d9f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 1 Oct 2019 10:49:19 +0100 Subject: [PATCH 076/898] WIP - fix mac_address access issue --- zaza/openstack/utilities/openstack.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 4b9a4e0..1fd9603 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -608,9 +608,11 @@ def configure_gateway_ext_port(novaclient, neutronclient, port = neutronclient.create_port(body=body_value) server.interface_attach(port_id=port['port']['id'], net_id=None, fixed_ip=None) - # Temporarily disable call to add_interface_to_netplan as it - # sometimes throws a KeyError - # https://github.com/openstack-charmers/zaza-openstack-tests/issues/72 + if add_dataport_to_netplan: + mac_address = get_mac_from_port(port, neutronclient) + add_interface_to_netplan(server.name, + mac_address=mac_address, + dvr_mode=dvr_mode) ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: if 'ext-port' in port['name']: @@ -652,9 +654,9 @@ def get_mac_from_port(port, neutronclient): :rtype: string """ logging.info("Trying to get mac address from port:" - "{}".format(port['id'])) - refresh_port = neutronclient.show_port(port['id'])['port'] - return refresh_port['mac_address'] + "{}".format(port['port']['id'])) + refresh_port = neutronclient.show_port(port['port']['id']) + return refresh_port['port']['mac_address'] def create_project_network(neutron_client, project_id, net_name='private', From b8cbae441ab8a3dc2aa82f69fb3e140de860bb78 Mon Sep 17 00:00:00 2001 From: Eduardo Sousa Date: Sun, 29 Sep 2019 00:18:16 +0100 Subject: [PATCH 077/898] Change pause and resume from ceilometer to services --- zaza/openstack/charm_tests/ceilometer/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index 1e2e316..b6e3272 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -113,5 +113,5 @@ class CeilometerTest(test_utils.OpenStackBaseTest): Pause service and check services are stopped then resume and check they are started. """ - with self.pause_resume(['ceilometer']): + with self.pause_resume(self.services): logging.info("Testing pause and resume") From d96bdd118db4b503d7c8999d0dd3fc5daaf1daa3 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Wed, 2 Oct 2019 13:11:46 +0200 Subject: [PATCH 078/898] disable add_dataport_to_netplan for dvr deployments --- zaza/openstack/utilities/openstack.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 465d73c..e6f4879 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -575,6 +575,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, """ if dvr_mode: uuids = get_ovs_uuids() + # If dvr, do not attempt to persist nic in netplan + # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78 + add_dataport_to_netplan = False else: uuids = get_gateway_uuids() From 7b0de8d34d44f4f356745da8e6e5045bfa00b1ee Mon Sep 17 00:00:00 2001 From: Rodrigo Barbieri Date: Tue, 17 Sep 2019 15:10:08 -0300 Subject: [PATCH 079/898] Add ceph-proxy cinder-ceph setup test This change adds a functional test that verifies ceph-proxy has successfully created a cinder-ceph restricted ceph pool. Related-Bug: lp#1836408 --- zaza/openstack/charm_tests/ceph/tests.py | 24 ++++++++++++++++++++++++ zaza/openstack/utilities/exceptions.py | 6 ++++++ 2 files changed, 30 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index f8bd0c1..9391953 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -706,7 +706,31 @@ class CephProxyTest(unittest.TestCase): def test_ceph_health(self): """Make sure ceph-proxy can communicate with ceph.""" + logging.info('Wait for idle/ready status...') + zaza_model.wait_for_application_states() + self.assertEqual( zaza_model.run_on_leader("ceph-proxy", "sudo ceph health")["Code"], "0" ) + + def test_cinder_ceph_restrict_pool_setup(self): + """Make sure cinder-ceph restrict pool was created successfully.""" + logging.info('Wait for idle/ready status...') + zaza_model.wait_for_application_states() + + pools = zaza_ceph.get_ceph_pools('ceph-mon/0') + if 'cinder-ceph' not in pools: + msg = 'cinder-ceph pool was not found upon querying ceph-mon/0' + raise zaza_exceptions.CephPoolNotFound(msg) + + expected = "pool=cinder-ceph, allow class-read " \ + "object_prefix rbd_children" + cmd = "sudo ceph auth get client.cinder-ceph" + result = zaza_model.run_on_unit('ceph-mon/0', cmd) + output = result.get('Stdout').strip() + + if expected not in output: + msg = ('cinder-ceph pool restriction was not configured correctly.' + ' Found: {}'.format(output)) + raise zaza_exceptions.CephPoolNotConfigured(msg) diff --git a/zaza/openstack/utilities/exceptions.py b/zaza/openstack/utilities/exceptions.py index 868a7c6..e5a5429 100644 --- a/zaza/openstack/utilities/exceptions.py +++ b/zaza/openstack/utilities/exceptions.py @@ -162,6 +162,12 @@ class CephPoolNotFound(Exception): pass +class CephPoolNotConfigured(Exception): + """Ceph pool not configured properly.""" + + pass + + class NovaGuestMigrationFailed(Exception): """Nova guest migration failed.""" From fe70f56b269d5e818351140057be1fc321515418 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Sun, 29 Sep 2019 17:07:25 +0100 Subject: [PATCH 080/898] Add base specializer for Keystone test --- zaza/openstack/charm_tests/policyd/tests.py | 265 +++++++++++++++++--- 1 file changed, 232 insertions(+), 33 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 9e76654..8599d02 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -125,7 +125,48 @@ class PolicydTest(object): rules_zip_path) self._set_config(True) zaza_model.block_until_wl_status_info_starts_with( - self.application_name, "PO:", negate_match=True) + self.application_name, "PO:", negate_match=False) + + def test_002_policyd_bad_yaml(self): + """Test bad yaml file in the zip file is handled.""" + bad = { + "file2.yaml": "{'rule': '!}" + } + bad_zip_path = self._make_zip_file_from('bad.zip', bad) + logging.info("Attaching bad zip file as a resource") + zaza_model.attach_resource(self.application_name, + 'policyd-override', + bad_zip_path) + zaza_model.block_until_all_units_idle() + logging.debug("Now setting config to true") + self._set_config(True) + # ensure that the workload status info line starts with PO (broken): + # to show that it didn't work + logging.info( + "Checking for workload status line starts with PO (broken):") + zaza_model.block_until_wl_status_info_starts_with( + self.application_name, "PO (broken):") + logging.debug("App status is valid for broken yaml file") + zaza_model.block_until_all_units_idle() + # now verify that no file got landed on the machine + path = os.path.join( + "/etc", self._service_name, "policy.d", 'file2.yaml') + logging.info("Now checking that file {} is not present.".format(path)) + zaza_model.block_until_file_missing(self.application_name, path) + self._set_config(False) + zaza_model.block_until_all_units_idle() + logging.info("OK") + + +class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): + """Generic policyd test for any charm without a specific test. + + It includes a single additional test to test_002_policyd_bad_yaml() in the + base class PolicydTest which checks that a good yaml file is placed into + the correct directory. This additional test (test_001_policyd_good_yaml) + is not needed in the specialised tests as the feature is implicitly tested + when the specialised test fails with the rule overrided. + """ def test_001_policyd_good_yaml(self): """Test that the policyd with a good zipped yaml file.""" @@ -160,7 +201,7 @@ class PolicydTest(object): # we have to do it twice due to async races and that some info lines # erase the PO: bit prior to actuall getting back to idle. The double # check verifies that the charms have started, the idle waits until it - # is finiehed, and then the final check really makes sure they got + # is finished, and then the final check really makes sure they got # switched off. zaza_model.block_until_wl_status_info_starts_with( self.application_name, "PO:", negate_match=True) @@ -174,40 +215,186 @@ class PolicydTest(object): logging.info("OK") - def test_002_policyd_bad_yaml(self): - """Test bad yaml file in the zip file is handled.""" - bad = { - "file2.yaml": "{'rule': '!}" + +class PolicydOperationFailedException(Exception): + """This is raised by the get_client_and_attempt_operation() method. + + This is used to signal that the operation in the + get_client_and_attempt_operation() method in the BaseSpecialization class + has failed. + """ + + pass + + +class BasePolicydSpecialization(PolicydTest, + ch_keystone.BaseKeystoneTest, + test_utils.OpenStackBaseTest): + """Base test for specialising Policyd override tests + + This class is for specialization of the test to verify that a yaml file + placed in the policy.d director is observed. This is done by first calling + the get_client_and_attempt_operation() method and ensuring that it works. + This method should attempt an operation on the service that can be blocked + by the policy override in the `_rule` class variable. The method should + pass cleanly without the override in place. + + The test_003_test_overide_is_observed will then apply the override and then + call get_client_and_attempt_operation() again, and this time it should + detect the failure and raise the PolicydOperationFailedException() + exception. This will be detected as the override working and thus the test + will pass. + + The test will fail if the first call fails for any reason, or if the 2nd + call doesn't raise PolicydOperationFailedException or raises any other + exception. + """ + + # this needs to be defined as the rule that gets placed into a yaml policy + # override. It is a string of the form: 'some-rule: "!"' + # i.e. disable some policy and then try and test it. + _rule = None + + # the name to log at the beginning of the test + _test_name = None + + @classmethod + def setUpClass(cls, application_name=None): + """Run class setup for running KeystonePolicydTest tests.""" + super(BasePolicydSpecialization, cls).setUpClass(application_name) + if cls._rule is None: + cls.SkipTest("Test not valid if {}.rule is not configured" + .format(cls.__name__)) + return + + def get_client_and_attempt_operation(self, keystone_session): + """Override this method to perform the operation. + + This operation should pass normally for the demo_user, and fail when + the rule has been overriden (see the `rule` class variable. + + :param keystone_session: the keystone session to use to obtain the + client necessary for the test. + :type keystone_session: ??? + :raises: PolicydOperationFailedException if operation fails. + """ + raise NotImplementedError("This method must be overridden") + + def _get_keystone_session(self, ip): + """Return the keystone session for the IP address passed. + + :param ip: the IP address to get the session against. + :type ip: str + :returns: a keystone session to the IP address + :rtype: ??? + """ + logging.info('keystone IP {}'.format(ip)) + openrc = { + 'API_VERSION': 3, + 'OS_USERNAME': ch_keystone.DEMO_ADMIN_USER, + 'OS_PASSWORD': ch_keystone.DEMO_ADMIN_USER_PASSWORD, + 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), + 'OS_USER_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + 'OS_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, } - bad_zip_path = self._make_zip_file_from('bad.zip', bad) - logging.info("Attaching bad zip file as a resource") - zaza_model.attach_resource(self.application_name, - 'policyd-override', - bad_zip_path) + if self.tls_rid: + openrc['OS_CACERT'] = \ + openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_AUTH_URL'] = ( + openrc['OS_AUTH_URL'].replace('http', 'https')) + logging.info('keystone IP {}'.format(ip)) + keystone_session = openstack_utils.get_keystone_session( + openrc, scope='DOMAIN') + return keystone_session + + def test_003_test_overide_is_observed(self): + """Test that the override is observed by the underlying service""" + if self._test_name is None: + logging.info("Doing policyd override for {}" + .format(self._service_name)) + else: + logging.info(self._test_name) + # note policyd override only works with Xenial-queens and so keystone + # is already v3 + # get a keystone session from any keystone unit; we only need one + # to check to see if it's working. + user_keystone_session = self._get_keystone_session( + self.keystone_ips[0]) + + # verify that the operation works before performing the policyd + # override. + try: + self.get_client_and_attempt_operation(user_keystone_session) + except Exception as e: + raise zaza_exceptions.PolicydError( + 'Retrieve service action as demo user with domain scoped ' + 'token failed and should have passed. "{}"' + .format(str(e))) + + # now do the policyd override. + logging.info("Doing policyd override with: {}".format(self._rule)) + self._set_policy_with({'rule.yaml': self._rule}) zaza_model.block_until_all_units_idle() - logging.debug("Now setting config to true") - self._set_config(True) - # ensure that the workload status info line starts with PO (broken): - # to show that it didn't work - logging.info( - "Checking for workload status line starts with PO (broken):") + + # now make sure the operation fails + try: + self.get_client_and_attempt_operation(user_keystone_session) + raise zaza_exceptions.PolicydError( + 'Retrieve service action as demo user with domain scoped ' + 'token passed but should have failed due to the policy ' + 'override.') + except PolicydOperationFailedException: + pass + except Exception as e: + raise zaza_exceptions.PolicydError( + 'Retrieve service action as demo user with domain scoped ' + 'token failed: "{}"' + .format(str(e))) + + # clean out the policy and wait + self._set_config(False) + # check that the status no longer has "PO:" on it. + # we have to do it twice due to async races and that some info lines + # erase the PO: bit prior to actuall getting back to idle. The double + # check verifies that the charms have started, the idle waits until it + # is finished, and then the final check really makes sure they got + # switched off. zaza_model.block_until_wl_status_info_starts_with( - self.application_name, "PO (broken):") - logging.debug("App status is valid for broken yaml file") + self.application_name, "PO:", negate_match=True) zaza_model.block_until_all_units_idle() - # now verify that no file got landed on the machine - path = os.path.join( - "/etc", self._service_name, "policy.d", 'file2.yaml') - logging.info("Now checking that file {} is not present.".format(path)) - zaza_model.block_until_file_missing(self.application_name, path) - self._set_config(True) - zaza_model.block_until_all_units_idle() - logging.info("OK") + zaza_model.block_until_wl_status_info_starts_with( + self.application_name, "PO:", negate_match=True) + + logging.info('OK') -class KeystonePolicydTest(PolicydTest, - ch_keystone.BaseKeystoneTest, - test_utils.OpenStackBaseTest): +class KeystonePolicydTest(BasePolicydSpecialization): + """Test the policyd override using the keystone client.""" + + _rule = "{'identity:list_services': '!'}" + + def get_client_and_attempt_operation(self, keystone_session): + """Override this method to perform the operation. + + This operation should pass normally for the demo_user, and fail when + the rule has been overriden (see the `rule` class variable. + + :param keystone_session: the keystone session to use to obtain the + client necessary for the test. + :type keystone_session: ??? + :raises: PolicydOperationFailedException if operation fails. + """ + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + try: + keystone_client.services.list() + except keystoneauth1.exceptions.http.Forbidden: + raise PolicydOperationFailedException() + + +class OldKeystonePolicydTest(PolicydTest, + ch_keystone.BaseKeystoneTest, + test_utils.OpenStackBaseTest): """Specific test for policyd for keystone charm.""" @classmethod @@ -303,7 +490,19 @@ class KeystonePolicydTest(PolicydTest, logging.info('OK') -class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): - """Generic policyd test for any charm without a specific test.""" +# class NeutronAPITest(PolicydTest, + # test_utils.OpenStackBaseTest): + # """Specific test for policyd for neutron-api charm.""" + + # def test_disable_service(self): + # """Test that service can be disabled.""" + # logging.info("Doing policyd override to disable listing domains") + # self._set_policy_with( + # {'rule.yaml': "{'identity:list_services': '!'}"}) + # # Get authenticated clients + # keystone_client = openstack_utils.get_keystone_session_client( + # keystone_session) + # neutron_client = openstack_utils.get_neutron_session_client( + # keystone_session) + - pass From 6b750869571082107a29fd9d79acdd70b80fbfaf Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Sun, 29 Sep 2019 21:32:42 +0100 Subject: [PATCH 081/898] Add neutron-api charm policyd test --- zaza/openstack/charm_tests/policyd/tests.py | 242 ++++++-------------- 1 file changed, 69 insertions(+), 173 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 8599d02..7030a9a 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -25,22 +25,6 @@ The Policyd Tests test the following: verify that the charm has implemented policy overrides and ensured that the service actually picks them up). -In order to use the generic tests, just include them in the specific test -class. The KeystonePolicydTest as an example does: - - class KeystonePolicydTest(PolicydTest, - ch_keystone.BaseKeystoneTest, - test_utils.OpenStackBaseTest): - - @classmethod - def setUpClass(cls, application_name=None): - super(KeystonePolicydTest, cls).setUpClass(application_name) - -Note that the generic test class (PolicyDTest) comes first, and then the -ch_keystone.BaseKeystoneTest, followed by the test_utils.OpenStackBaseTest. -This is to get the order of super().setUpClass(...) calls to work with -application_name. - If a charm doesn't require a specific test, then the GenericPolicydTest class can be used that just includes the two generic tests. The config in the tests.yaml would stil be required. See the PolicydTest class docstring for @@ -54,6 +38,7 @@ import tempfile import zipfile import keystoneauth1 +import neutronclient.exceptions import zaza.model as zaza_model @@ -127,47 +112,6 @@ class PolicydTest(object): zaza_model.block_until_wl_status_info_starts_with( self.application_name, "PO:", negate_match=False) - def test_002_policyd_bad_yaml(self): - """Test bad yaml file in the zip file is handled.""" - bad = { - "file2.yaml": "{'rule': '!}" - } - bad_zip_path = self._make_zip_file_from('bad.zip', bad) - logging.info("Attaching bad zip file as a resource") - zaza_model.attach_resource(self.application_name, - 'policyd-override', - bad_zip_path) - zaza_model.block_until_all_units_idle() - logging.debug("Now setting config to true") - self._set_config(True) - # ensure that the workload status info line starts with PO (broken): - # to show that it didn't work - logging.info( - "Checking for workload status line starts with PO (broken):") - zaza_model.block_until_wl_status_info_starts_with( - self.application_name, "PO (broken):") - logging.debug("App status is valid for broken yaml file") - zaza_model.block_until_all_units_idle() - # now verify that no file got landed on the machine - path = os.path.join( - "/etc", self._service_name, "policy.d", 'file2.yaml') - logging.info("Now checking that file {} is not present.".format(path)) - zaza_model.block_until_file_missing(self.application_name, path) - self._set_config(False) - zaza_model.block_until_all_units_idle() - logging.info("OK") - - -class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): - """Generic policyd test for any charm without a specific test. - - It includes a single additional test to test_002_policyd_bad_yaml() in the - base class PolicydTest which checks that a good yaml file is placed into - the correct directory. This additional test (test_001_policyd_good_yaml) - is not needed in the specialised tests as the feature is implicitly tested - when the specialised test fails with the rule overrided. - """ - def test_001_policyd_good_yaml(self): """Test that the policyd with a good zipped yaml file.""" good = { @@ -215,6 +159,42 @@ class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): logging.info("OK") + def test_002_policyd_bad_yaml(self): + """Test bad yaml file in the zip file is handled.""" + bad = { + "file2.yaml": "{'rule': '!}" + } + bad_zip_path = self._make_zip_file_from('bad.zip', bad) + logging.info("Attaching bad zip file as a resource") + zaza_model.attach_resource(self.application_name, + 'policyd-override', + bad_zip_path) + zaza_model.block_until_all_units_idle() + logging.debug("Now setting config to true") + self._set_config(True) + # ensure that the workload status info line starts with PO (broken): + # to show that it didn't work + logging.info( + "Checking for workload status line starts with PO (broken):") + zaza_model.block_until_wl_status_info_starts_with( + self.application_name, "PO (broken):") + logging.debug("App status is valid for broken yaml file") + zaza_model.block_until_all_units_idle() + # now verify that no file got landed on the machine + path = os.path.join( + "/etc", self._service_name, "policy.d", 'file2.yaml') + logging.info("Now checking that file {} is not present.".format(path)) + zaza_model.block_until_file_missing(self.application_name, path) + self._set_config(False) + zaza_model.block_until_all_units_idle() + logging.info("OK") + + +class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): + """Generic policyd test for any charm without a specific test.""" + + pass + class PolicydOperationFailedException(Exception): """This is raised by the get_client_and_attempt_operation() method. @@ -248,14 +228,22 @@ class BasePolicydSpecialization(PolicydTest, The test will fail if the first call fails for any reason, or if the 2nd call doesn't raise PolicydOperationFailedException or raises any other exception. - """ + To use this class, follow the keystone example: + + class KeystonePolicydTest(BasePolicydSpecialization): + + _rule = "{'identity:list_services': '!'}" + + def get_client_and_attempt_operation(self, keystone_session): + ... etc. + """ # this needs to be defined as the rule that gets placed into a yaml policy # override. It is a string of the form: 'some-rule: "!"' # i.e. disable some policy and then try and test it. _rule = None - # the name to log at the beginning of the test + # Optional: the name to log at the beginning of the test _test_name = None @classmethod @@ -275,7 +263,7 @@ class BasePolicydSpecialization(PolicydTest, :param keystone_session: the keystone session to use to obtain the client necessary for the test. - :type keystone_session: ??? + :type keystone_session: keystoneauth1.session.Session :raises: PolicydOperationFailedException if operation fails. """ raise NotImplementedError("This method must be overridden") @@ -286,7 +274,7 @@ class BasePolicydSpecialization(PolicydTest, :param ip: the IP address to get the session against. :type ip: str :returns: a keystone session to the IP address - :rtype: ??? + :rtype: keystoneauth1.session.Session """ logging.info('keystone IP {}'.format(ip)) openrc = { @@ -374,14 +362,14 @@ class KeystonePolicydTest(BasePolicydSpecialization): _rule = "{'identity:list_services': '!'}" def get_client_and_attempt_operation(self, keystone_session): - """Override this method to perform the operation. + """Attempt to list services. If it fails, raise an exception. This operation should pass normally for the demo_user, and fail when the rule has been overriden (see the `rule` class variable. :param keystone_session: the keystone session to use to obtain the client necessary for the test. - :type keystone_session: ??? + :type keystone_session: keystoneauth1.session.Session :raises: PolicydOperationFailedException if operation fails. """ keystone_client = openstack_utils.get_keystone_session_client( @@ -392,117 +380,25 @@ class KeystonePolicydTest(BasePolicydSpecialization): raise PolicydOperationFailedException() -class OldKeystonePolicydTest(PolicydTest, - ch_keystone.BaseKeystoneTest, - test_utils.OpenStackBaseTest): - """Specific test for policyd for keystone charm.""" +class NeutronApiTest(BasePolicydSpecialization): + """Test the policyd override using the keystone client.""" - @classmethod - def setUpClass(cls, application_name=None): - """Run class setup for running KeystonePolicydTest tests.""" - super(KeystonePolicydTest, cls).setUpClass(application_name) + _rule = "{'get_network': '!'}" - def test_disable_service(self): - """Test that service can be disabled.""" - logging.info("Doing policyd override to disable listing domains") - self._set_policy_with( - {'rule.yaml': "{'identity:list_services': '!'}"}) - - # verify that the policy.d override does disable the endpoint - with self.config_change( - {'preferred-api-version': self.default_api_version, - 'use-policyd-override': 'False'}, - {'preferred-api-version': '3', - 'use-policyd-override': 'True'}, - application_name="keystone"): - zaza_model.block_until_all_units_idle() - for ip in self.keystone_ips: - try: - logging.info('keystone IP {}'.format(ip)) - openrc = { - 'API_VERSION': 3, - 'OS_USERNAME': ch_keystone.DEMO_ADMIN_USER, - 'OS_PASSWORD': ch_keystone.DEMO_ADMIN_USER_PASSWORD, - 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), - 'OS_USER_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, - 'OS_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, - } - if self.tls_rid: - openrc['OS_CACERT'] = \ - openstack_utils.KEYSTONE_LOCAL_CACERT - openrc['OS_AUTH_URL'] = ( - openrc['OS_AUTH_URL'].replace('http', 'https')) - logging.info('keystone IP {}'.format(ip)) - keystone_session = openstack_utils.get_keystone_session( - openrc, scope='DOMAIN') - keystone_client = ( - openstack_utils.get_keystone_session_client( - keystone_session)) - keystone_client.services.list() - raise zaza_exceptions.PolicydError( - 'Retrieve service list as admin with project scoped ' - 'token passed and should have failed. IP = {}' - .format(ip)) - except keystoneauth1.exceptions.http.Forbidden: - logging.info("keystone IP:{} policyd override disabled " - "services listing by demo user" - .format(ip)) - - # now verify (with the config off) that we can actually access - # these points - with self.config_change( - {'preferred-api-version': self.default_api_version}, - {'preferred-api-version': '3'}, - application_name="keystone"): - zaza_model.block_until_all_units_idle() - for ip in self.keystone_ips: - try: - logging.info('keystone IP {}'.format(ip)) - openrc = { - 'API_VERSION': 3, - 'OS_USERNAME': ch_keystone.DEMO_ADMIN_USER, - 'OS_PASSWORD': ch_keystone.DEMO_ADMIN_USER_PASSWORD, - 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), - 'OS_USER_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, - 'OS_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, - } - if self.tls_rid: - openrc['OS_CACERT'] = \ - openstack_utils.KEYSTONE_LOCAL_CACERT - openrc['OS_AUTH_URL'] = ( - openrc['OS_AUTH_URL'].replace('http', 'https')) - logging.info('keystone IP {}'.format(ip)) - keystone_session = openstack_utils.get_keystone_session( - openrc, scope='DOMAIN') - keystone_client = ( - openstack_utils.get_keystone_session_client( - keystone_session)) - keystone_client.services.list() - logging.info("keystone IP:{} without policyd override " - "services list working" - .format(ip)) - except keystoneauth1.exceptions.http.Forbidden: - raise zaza_exceptions.PolicydError( - 'Retrieve services list as demo user with project ' - 'scoped token passed and should have passed. IP = {}' - .format(ip)) - - logging.info('OK') - - -# class NeutronAPITest(PolicydTest, - # test_utils.OpenStackBaseTest): - # """Specific test for policyd for neutron-api charm.""" - - # def test_disable_service(self): - # """Test that service can be disabled.""" - # logging.info("Doing policyd override to disable listing domains") - # self._set_policy_with( - # {'rule.yaml': "{'identity:list_services': '!'}"}) - # # Get authenticated clients - # keystone_client = openstack_utils.get_keystone_session_client( - # keystone_session) - # neutron_client = openstack_utils.get_neutron_session_client( - # keystone_session) + def get_client_and_attempt_operation(self, keystone_session): + """Attempt to ???? + This operation should pass normally for the demo_user, and fail when + the rule has been overriden (see the `rule` class variable. + :param keystone_session: the keystone session to use to obtain the + client necessary for the test. + :type keystone_session: keystoneauth1.session.Session + :raises: PolicydOperationFailedException if operation fails. + """ + neutron_client = openstack_utils.get_neutron_session_client( + keystone_session) + try: + neutron_client.list_networks() + except neutronclient.exceptions.Forbidden: + raise PolicydOperationFailedException() From 909a8d8969f49bce4d35664b2d9de052d3c74dc4 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 3 Oct 2019 17:18:52 +0100 Subject: [PATCH 082/898] Add remaining policyd tests for 19.10 release This adds the modified neutron-api, glance, cinder policyd tests. In particular, the test verifies that an operation can be overriden and removed and that the api calls works, doesn't work, and then works again. --- zaza/openstack/charm_tests/policyd/tests.py | 223 ++++++++++++++++---- 1 file changed, 182 insertions(+), 41 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 7030a9a..396d3df 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -38,7 +38,8 @@ import tempfile import zipfile import keystoneauth1 -import neutronclient.exceptions +import glanceclient.common.exceptions +import cinderclient.exceptions import zaza.model as zaza_model @@ -210,7 +211,7 @@ class PolicydOperationFailedException(Exception): class BasePolicydSpecialization(PolicydTest, ch_keystone.BaseKeystoneTest, test_utils.OpenStackBaseTest): - """Base test for specialising Policyd override tests + """Base test for specialising Policyd override tests. This class is for specialization of the test to verify that a yaml file placed in the policy.d director is observed. This is done by first calling @@ -238,6 +239,7 @@ class BasePolicydSpecialization(PolicydTest, def get_client_and_attempt_operation(self, keystone_session): ... etc. """ + # this needs to be defined as the rule that gets placed into a yaml policy # override. It is a string of the form: 'some-rule: "!"' # i.e. disable some policy and then try and test it. @@ -268,23 +270,20 @@ class BasePolicydSpecialization(PolicydTest, """ raise NotImplementedError("This method must be overridden") - def _get_keystone_session(self, ip): + def _get_keystone_session(self, ip, openrc, scope='DOMAIN'): """Return the keystone session for the IP address passed. :param ip: the IP address to get the session against. :type ip: str + :param openrc: the params to authenticate with. + :type openrc: Dict[str, str] + :param scope: the scope of the token + :type scope: str :returns: a keystone session to the IP address :rtype: keystoneauth1.session.Session """ - logging.info('keystone IP {}'.format(ip)) - openrc = { - 'API_VERSION': 3, - 'OS_USERNAME': ch_keystone.DEMO_ADMIN_USER, - 'OS_PASSWORD': ch_keystone.DEMO_ADMIN_USER_PASSWORD, - 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), - 'OS_USER_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, - 'OS_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, - } + logging.info('Authentication for {} on keystone IP {}' + .format(openrc['OS_USERNAME'], ip)) if self.tls_rid: openrc['OS_CACERT'] = \ openstack_utils.KEYSTONE_LOCAL_CACERT @@ -292,11 +291,64 @@ class BasePolicydSpecialization(PolicydTest, openrc['OS_AUTH_URL'].replace('http', 'https')) logging.info('keystone IP {}'.format(ip)) keystone_session = openstack_utils.get_keystone_session( - openrc, scope='DOMAIN') + openrc, scope=scope) return keystone_session + def get_keystone_session_demo_user(self, ip, scope='PROJECT'): + """Return the keystone session for demo user. + + :param ip: the IP address to get the session against. + :type ip: str + :param scope: the scope of the token + :type scope: str + :returns: a keystone session to the IP address + :rtype: keystoneauth1.session.Session + """ + return self._get_keystone_session(ip, { + 'API_VERSION': 3, + 'OS_USERNAME': ch_keystone.DEMO_USER, + 'OS_PASSWORD': ch_keystone.DEMO_PASSWORD, + 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), + 'OS_USER_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + 'OS_PROJECT_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + 'OS_PROJECT_NAME': ch_keystone.DEMO_PROJECT, + 'OS_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + }, scope) + + def get_keystone_session_demo_admin_user(self, ip, scope='PROJECT'): + """Return the keystone session demo_admin user. + + :param ip: the IP address to get the session against. + :type ip: str + :param scope: the scope of the token + :type scope: str + :returns: a keystone session to the IP address + :rtype: keystoneauth1.session.Session + """ + return self._get_keystone_session(ip, { + 'API_VERSION': 3, + 'OS_USERNAME': ch_keystone.DEMO_ADMIN_USER, + 'OS_PASSWORD': ch_keystone.DEMO_ADMIN_USER_PASSWORD, + 'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip), + 'OS_USER_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + 'OS_PROJECT_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + 'OS_PROJECT_NAME': ch_keystone.DEMO_PROJECT, + 'OS_DOMAIN_NAME': ch_keystone.DEMO_DOMAIN, + }, scope) + + def get_keystone_session_admin_user(self, ip): + """Return the keystone session admin user. + + :param ip: the IP address to get the session against. + :type ip: str + :returns: a keystone session to the IP address + :rtype: keystoneauth1.session.Session + """ + return openstack_utils.get_keystone_session( + openstack_utils.get_overcloud_auth(address=ip)) + def test_003_test_overide_is_observed(self): - """Test that the override is observed by the underlying service""" + """Test that the override is observed by the underlying service.""" if self._test_name is None: logging.info("Doing policyd override for {}" .format(self._service_name)) @@ -304,19 +356,18 @@ class BasePolicydSpecialization(PolicydTest, logging.info(self._test_name) # note policyd override only works with Xenial-queens and so keystone # is already v3 - # get a keystone session from any keystone unit; we only need one - # to check to see if it's working. - user_keystone_session = self._get_keystone_session( - self.keystone_ips[0]) # verify that the operation works before performing the policyd # override. + zaza_model.block_until_wl_status_info_starts_with( + self.application_name, "PO:", negate_match=True) + zaza_model.block_until_all_units_idle() + logging.info("First verify that operation works prior to override") try: - self.get_client_and_attempt_operation(user_keystone_session) + self.get_client_and_attempt_operation(self.keystone_ips[0]) except Exception as e: raise zaza_exceptions.PolicydError( - 'Retrieve service action as demo user with domain scoped ' - 'token failed and should have passed. "{}"' + 'Service action failed and should have passed. "{}"' .format(str(e))) # now do the policyd override. @@ -325,15 +376,20 @@ class BasePolicydSpecialization(PolicydTest, zaza_model.block_until_all_units_idle() # now make sure the operation fails + logging.info("Now verify that operation doesn't work with override") try: - self.get_client_and_attempt_operation(user_keystone_session) + self.get_client_and_attempt_operation(self.keystone_ips[0]) raise zaza_exceptions.PolicydError( - 'Retrieve service action as demo user with domain scoped ' - 'token passed but should have failed due to the policy ' - 'override.') + "Service action passed and should have failed.") except PolicydOperationFailedException: pass + except zaza_exceptions.PolicydError: + logging.info("Policy override worked.") + raise except Exception as e: + logging.info("exception was: {}".format(e.__class__.__name__)) + import traceback + logging.info(traceback.format_exc()) raise zaza_exceptions.PolicydError( 'Retrieve service action as demo user with domain scoped ' 'token failed: "{}"' @@ -353,52 +409,137 @@ class BasePolicydSpecialization(PolicydTest, zaza_model.block_until_wl_status_info_starts_with( self.application_name, "PO:", negate_match=True) + # Finally make sure it works again! + logging.info("Finally verify that operation works after removing the " + "override.") + try: + self.get_client_and_attempt_operation(self.keystone_ips[0]) + except Exception as e: + raise zaza_exceptions.PolicydError( + 'Service action failed and should have passed after removing ' + 'policy override: "{}"' + .format(str(e))) + logging.info('OK') -class KeystonePolicydTest(BasePolicydSpecialization): +class KeystoneTests(BasePolicydSpecialization): """Test the policyd override using the keystone client.""" _rule = "{'identity:list_services': '!'}" - def get_client_and_attempt_operation(self, keystone_session): + @classmethod + def setUpClass(cls, application_name=None): + """Run class setup for running NeutronApiTest charm operation tests.""" + super(KeystoneTests, cls).setUpClass( + application_name="keystone") + + def get_client_and_attempt_operation(self, ip): """Attempt to list services. If it fails, raise an exception. This operation should pass normally for the demo_user, and fail when the rule has been overriden (see the `rule` class variable. - :param keystone_session: the keystone session to use to obtain the - client necessary for the test. - :type keystone_session: keystoneauth1.session.Session + :param ip: the IP address to get the session against. + :type ip: str :raises: PolicydOperationFailedException if operation fails. """ keystone_client = openstack_utils.get_keystone_session_client( - keystone_session) + self.get_keystone_session_demo_admin_user(ip)) try: keystone_client.services.list() except keystoneauth1.exceptions.http.Forbidden: raise PolicydOperationFailedException() -class NeutronApiTest(BasePolicydSpecialization): - """Test the policyd override using the keystone client.""" +class NeutronApiTests(BasePolicydSpecialization): + """Test the policyd override using the neutron client.""" _rule = "{'get_network': '!'}" - def get_client_and_attempt_operation(self, keystone_session): - """Attempt to ???? + @classmethod + def setUpClass(cls, application_name=None): + """Run class setup for running NeutronApiTest charm operation tests.""" + super(NeutronApiTests, cls).setUpClass(application_name="neutron-api") + cls.application_name = "neutron-api" + + def get_client_and_attempt_operation(self, ip): + """Attempt to list the networks as a policyd override. This operation should pass normally for the demo_user, and fail when the rule has been overriden (see the `rule` class variable. - :param keystone_session: the keystone session to use to obtain the - client necessary for the test. - :type keystone_session: keystoneauth1.session.Session + :param ip: the IP address to get the session against. + :type ip: str :raises: PolicydOperationFailedException if operation fails. """ neutron_client = openstack_utils.get_neutron_session_client( - keystone_session) + self.get_keystone_session_demo_user(ip)) try: - neutron_client.list_networks() - except neutronclient.exceptions.Forbidden: + # If we are allowed to list networks, this will return something. + # if the policyd override is present, then no error is generated, + # but no networks are returned. + networks = neutron_client.list_networks() + logging.debug("networks: {}".format(networks)) + if len(networks['networks']) == 0: + raise PolicydOperationFailedException() + except Exception: + raise PolicydOperationFailedException() + + +class GlanceTests(BasePolicydSpecialization): + """Test the policyd override using the glance client.""" + + _rule = "{'get_images': '!'}" + + @classmethod + def setUpClass(cls, application_name=None): + """Run class setup for running GlanceTests charm operation tests.""" + super(GlanceTests, cls).setUpClass(application_name="glance") + cls.application_name = "glance" + + def get_client_and_attempt_operation(self, ip): + """Attempt to list the images as a policyd override. + + This operation should pass normally for the demo_user, and fail when + the rule has been overriden (see the `rule` class variable. + + :param ip: the IP address to get the session against. + :type ip: str + :raises: PolicydOperationFailedException if operation fails. + """ + glance_client = openstack_utils.get_glance_session_client( + self.get_keystone_session_demo_user(ip)) + try: + glance_client.images.list() + except glanceclient.common.exceptions.HTTPForbidden: + raise PolicydOperationFailedException() + + +class CinderTests(BasePolicydSpecialization): + """Test the policyd override using the cinder client.""" + + _rule = "{'volume:get_all': '!'}" + + @classmethod + def setUpClass(cls, application_name=None): + """Run class setup for running CinderTests charm operation tests.""" + super(CinderTests, cls).setUpClass(application_name="cinder") + cls.application_name = "cinder" + + def get_client_and_attempt_operation(self, ip): + """Attempt to list the images as a policyd override. + + This operation should pass normally for the demo_user, and fail when + the rule has been overriden (see the `rule` class variable. + + :param ip: the IP address to get the session against. + :type ip: str + :raises: PolicydOperationFailedException if operation fails. + """ + cinder_client = openstack_utils.get_cinder_session_client( + self.get_keystone_session_admin_user(ip)) + try: + cinder_client.volumes.list() + except cinderclient.exceptions.Forbidden: raise PolicydOperationFailedException() From bb162918db2824c70711e739a7378ca41b6e078a Mon Sep 17 00:00:00 2001 From: Eduardo Sousa Date: Thu, 3 Oct 2019 15:27:28 +0100 Subject: [PATCH 083/898] Disabling pause_resume This test is being disabled because of bug #1846390. https://bugs.launchpad.net/charms/+source/ceilometer/+bug/1846390 --- .../openstack/charm_tests/ceilometer/tests.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index b6e3272..23f4bbd 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -40,23 +40,28 @@ class CeilometerTest(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list services for Openstack Release.""" + """Return a list services for Openstack Release. + + Note: disabling ceilometer-polling and ceilometer-agent-central due to + bug #1846390. + https://bugs.launchpad.net/charms/+source/ceilometer/+bug/1846390 + """ current_release = openstack_utils.get_os_release() services = [] if current_release >= CeilometerTest.XENIAL_PIKE: - services.append('ceilometer-polling: AgentManager worker(0)') + # services.append('ceilometer-polling: AgentManager worker(0)') services.append('ceilometer-agent-notification: ' 'NotificationService worker(0)') elif current_release >= CeilometerTest.XENIAL_OCATA: services.append('ceilometer-collector: CollectorService worker(0)') - services.append('ceilometer-polling: AgentManager worker(0)') + # services.append('ceilometer-polling: AgentManager worker(0)') services.append('ceilometer-agent-notification: ' 'NotificationService worker(0)') services.append('apache2') elif current_release >= CeilometerTest.XENIAL_NEWTON: services.append('ceilometer-collector - CollectorService(0)') - services.append('ceilometer-polling - AgentManager(0)') + # services.append('ceilometer-polling - AgentManager(0)') services.append('ceilometer-agent-notification - ' 'NotificationService(0)') services.append('ceilometer-api') @@ -69,12 +74,12 @@ class CeilometerTest(test_utils.OpenStackBaseTest): services.append('ceilometer-alarm-notifier') services.append('ceilometer-alarm-evaluator') - if current_release >= CeilometerTest.TRUSTY_LIBERTY: + # if current_release >= CeilometerTest.TRUSTY_LIBERTY: # Liberty and later - services.append('ceilometer-polling') - else: + # services.append('ceilometer-polling') + # else: # Juno and earlier - services.append('ceilometer-agent-central') + # services.append('ceilometer-agent-central') return services From 001635e23bae6b3b6db56deb83000d8456939878 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 7 Oct 2019 10:10:27 +0100 Subject: [PATCH 084/898] Fix skiptest on policyd The wrong approach was used. It was supposed to be a raise rather than just a cls call. This PR fixes-up the SkipTest in policyd. --- zaza/openstack/charm_tests/policyd/tests.py | 26 +++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 396d3df..601dd86 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -35,6 +35,7 @@ import logging import os import shutil import tempfile +import unittest import zipfile import keystoneauth1 @@ -65,9 +66,6 @@ class PolicydTest(object): def setUpClass(cls, application_name=None): """Run class setup for running Policyd charm operation tests.""" super(PolicydTest, cls).setUpClass(application_name) - if (openstack_utils.get_os_release() < - openstack_utils.get_os_release('xenial_queens')): - cls.SkipTest("Test not valid before xenial_queens") cls._tmp_dir = tempfile.mkdtemp() cls._service_name = \ cls.test_config['tests_options']['policyd']['service'] @@ -194,7 +192,15 @@ class PolicydTest(object): class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): """Generic policyd test for any charm without a specific test.""" - pass + @classmethod + def setUpClass(cls, application_name=None): + """Run class setup for running KeystonePolicydTest tests.""" + super(GenericPolicydTest, cls).setUpClass(application_name) + if (openstack_utils.get_os_release() < + openstack_utils.get_os_release('xenial_queens')): + raise unittest.SkipTest( + "zaza.openstack.charm_tests.policyd.tests.GenericPolicydTest " + "not valid before xenial_queens") class PolicydOperationFailedException(Exception): @@ -252,10 +258,16 @@ class BasePolicydSpecialization(PolicydTest, def setUpClass(cls, application_name=None): """Run class setup for running KeystonePolicydTest tests.""" super(BasePolicydSpecialization, cls).setUpClass(application_name) + if (openstack_utils.get_os_release() < + openstack_utils.get_os_release('xenial_queens')): + raise unittest.SkipTest( + "zaza.openstack.charm_tests.policyd.tests.* " + "not valid before xenial_queens") if cls._rule is None: - cls.SkipTest("Test not valid if {}.rule is not configured" - .format(cls.__name__)) - return + raise unittest.SkipTest( + "zaza.openstack.charm_tests.policyd.tests.* " + "not valid if {}.rule is not configured" + .format(cls.__name__)) def get_client_and_attempt_operation(self, keystone_session): """Override this method to perform the operation. From 81c4432136a2469eb65ad1a180d11e8c5a4391f5 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 7 Oct 2019 22:06:21 +0100 Subject: [PATCH 085/898] Improve logging on the policyd tests This has no 'testing' changes, just improves the logging output. --- zaza/openstack/charm_tests/policyd/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 601dd86..e1b99fc 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -395,16 +395,15 @@ class BasePolicydSpecialization(PolicydTest, "Service action passed and should have failed.") except PolicydOperationFailedException: pass - except zaza_exceptions.PolicydError: - logging.info("Policy override worked.") + except zaza_exceptions.PolicydError as e: + logging.info("{}".format(str(e))) raise except Exception as e: logging.info("exception was: {}".format(e.__class__.__name__)) import traceback logging.info(traceback.format_exc()) raise zaza_exceptions.PolicydError( - 'Retrieve service action as demo user with domain scoped ' - 'token failed: "{}"' + 'Service action failed in an unexpected way: {}' .format(str(e))) # clean out the policy and wait @@ -523,7 +522,8 @@ class GlanceTests(BasePolicydSpecialization): glance_client = openstack_utils.get_glance_session_client( self.get_keystone_session_demo_user(ip)) try: - glance_client.images.list() + images = list(glance_client.images.list()) + logging.debug("images is: {}".format(images)) except glanceclient.common.exceptions.HTTPForbidden: raise PolicydOperationFailedException() From b33afe9cb350d2cc7bf01cc03af25c5daa45dc3d Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 7 Oct 2019 23:06:22 +0100 Subject: [PATCH 086/898] Add note re: GlanceTest function --- zaza/openstack/charm_tests/policyd/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index e1b99fc..8779149 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -522,6 +522,10 @@ class GlanceTests(BasePolicydSpecialization): glance_client = openstack_utils.get_glance_session_client( self.get_keystone_session_demo_user(ip)) try: + # NOTE(ajkavanagh) - it turns out that the list() is very important + # as it forces the generator to iterate which only then checkes if + # the api call is authorized. Just getting the generator (from + # .list()) doesn't perform the API call. images = list(glance_client.images.list()) logging.debug("images is: {}".format(images)) except glanceclient.common.exceptions.HTTPForbidden: From 0e932ed61ecac915b4ded2fac2e4f54c2ba506cd Mon Sep 17 00:00:00 2001 From: Natalia Litvinova Date: Mon, 30 Sep 2019 11:17:45 +0300 Subject: [PATCH 087/898] Porting Cinder Backup tests to Zaza Porting the Amulet tests from Cinder Backup to the Zaza framework. The Amulet tests can be found here: https://opendev.org/openstack/charm-cinder-backup/src/commit/4273738b94b22cca187487af2bad982be49fdd69/tests/basic_deployment.py --- .../charm_tests/cinder_backup/__init__.py | 17 ++ .../charm_tests/cinder_backup/tests.py | 194 ++++++++++++++++++ zaza/openstack/utilities/ceph.py | 50 ++++- 3 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 zaza/openstack/charm_tests/cinder_backup/__init__.py create mode 100644 zaza/openstack/charm_tests/cinder_backup/tests.py diff --git a/zaza/openstack/charm_tests/cinder_backup/__init__.py b/zaza/openstack/charm_tests/cinder_backup/__init__.py new file mode 100644 index 0000000..6501e55 --- /dev/null +++ b/zaza/openstack/charm_tests/cinder_backup/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing cinder-backup.""" diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py new file mode 100644 index 0000000..2fa12db --- /dev/null +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate cinder-backup testing.""" +import logging + +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.ceph as ceph_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +class CinderBackupTest(test_utils.OpenStackBaseTest): + """Encapsulate Cinder Backup tests.""" + + RESOURCE_PREFIX = 'zaza-cinderbackuptests' + + @classmethod + def setUpClass(cls): + """Run class setup for running Cinder Backup tests.""" + super(CinderBackupTest, cls).setUpClass() + cls.cinder_client = openstack_utils.get_cinder_session_client( + cls.keystone_session) + + @property + def services(self): + """Return a list services for Openstack Release.""" + current_release = openstack_utils.get_os_release() + services = ['cinder-scheduler', 'cinder-volume'] + if (current_release >= + openstack_utils.get_os_release('xenial_ocata')): + services.append('apache2') + else: + services.append('cinder-api') + return services + + def test_100_volume_create_extend_delete(self): + """Test creating, extending a volume.""" + vol_new = openstack_utils.create_volume( + self.cinder_client, + name='{}-100-vol'.format(self.RESOURCE_PREFIX), + size=1) + self.cinder_client.volumes.extend( + vol_new.id, + '2') + openstack_utils.resource_reaches_status( + self.cinder_client.volumes, + vol_new.id, + expected_status="available", + msg="Volume status wait") + + def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self): + """Create, backup, delete, restore a ceph-backed cinder volume. + + Create, backup, delete, restore a ceph-backed cinder volume, and + inspect ceph cinder pool object count as the volume is created + and deleted. + """ + unit_name = zaza.model.get_lead_unit_name('ceph-mon') + obj_count_samples = [] + pool_size_samples = [] + pools = ceph_utils.get_ceph_pools(unit_name) + expected_pool = 'cinder-ceph' + cinder_ceph_pool = pools[expected_pool] + + # Check ceph cinder pool object count, disk space usage and pool name + logging.info('Checking ceph cinder pool original samples...') + pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample( + unit_name, cinder_ceph_pool) + + obj_count_samples.append(obj_count) + pool_size_samples.append(kb_used) + + self.assertEqual(pool_name, expected_pool) + + # Create ceph-backed cinder volume + cinder_vol = self.cinder_client.volumes.create( + name='{}-410-vol'.format(self.RESOURCE_PREFIX), + size=1) + openstack_utils.resource_reaches_status( + self.cinder_client.volumes, + cinder_vol.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='Volume status wait') + + # Backup the volume + vol_backup = self.cinder_client.backups.create( + cinder_vol.id, + name='{}-410-backup-vol'.format(self.RESOURCE_PREFIX)) + openstack_utils.resource_reaches_status( + self.cinder_client.backups, + vol_backup.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='Volume status wait') + # Delete the volume + openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) + # Restore the volume + self.cinder_client.restores.restore(vol_backup.id) + openstack_utils.resource_reaches_status( + self.cinder_client.backups, + vol_backup.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='Backup status wait') + # Delete the backup + openstack_utils.delete_volume_backup( + self.cinder_client, + vol_backup.id) + openstack_utils.resource_removed( + self.cinder_client.backups, + vol_backup.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + msg="Backup volume") + + # Re-check ceph cinder pool object count and disk usage + logging.info('Checking ceph cinder pool samples ' + 'after volume create...') + pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample( + unit_name, cinder_ceph_pool, self.model_name) + + obj_count_samples.append(obj_count) + pool_size_samples.append(kb_used) + + name = '{}-410-vol'.format(self.RESOURCE_PREFIX) + vols = self.cinder_client.volumes.list() + try: + cinder_vols = [v for v in vols if v.name == name] + except AttributeError: + cinder_vols = [v for v in vols if v.display_name == name] + if not cinder_vols: + # NOTE(hopem): it appears that at some point cinder-backup stopped + # restoring volume metadata properly so revert to default name if + # original is not found + name = "restore_backup_{}".format(vol_backup.id) + try: + cinder_vols = [v for v in vols if v.name == name] + except AttributeError: + cinder_vols = [v for v in vols if v.display_name == name] + + self.assertTrue(cinder_vols) + + cinder_vol = cinder_vols[0] + + # Delete restored cinder volume + openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) + openstack_utils.resource_removed( + self.cinder_client.volumes, + cinder_vol.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + msg="Volume") + + # Final check, ceph cinder pool object count and disk usage + logging.info('Checking ceph cinder pool after volume delete...') + pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample( + unit_name, cinder_ceph_pool, self.model_name) + + obj_count_samples.append(obj_count) + pool_size_samples.append(kb_used) + + # Validate ceph cinder pool object count samples over time + original, created, deleted = range(3) + self.assertFalse(obj_count_samples[created] <= + obj_count_samples[original]) + self.assertFalse(obj_count_samples[deleted] >= + obj_count_samples[created]) + + # Luminous (pike) ceph seems more efficient at disk usage so we cannot + # grantee the ordering of kb_used + if (openstack_utils.get_os_release() < + openstack_utils.get_os_release('xenial_mitaka')): + self.assertFalse(pool_size_samples[created] <= + pool_size_samples[original]) + self.assertFalse(pool_size_samples[deleted] >= + pool_size_samples[created]) diff --git a/zaza/openstack/utilities/ceph.py b/zaza/openstack/utilities/ceph.py index 2c54512..61d7d66 100644 --- a/zaza/openstack/utilities/ceph.py +++ b/zaza/openstack/utilities/ceph.py @@ -1,5 +1,5 @@ """Module containing Ceph related utilities.""" - +import json import logging import zaza.openstack.utilities.openstack as openstack_utils @@ -97,6 +97,54 @@ def get_ceph_pools(unit_name, model_name=None): return pools +def get_ceph_df(unit_name, model_name=None): + """Return dict of ceph df json output, including ceph pool state. + + :param unit_name: Name of the unit to get ceph df + :type unit_name: string + :param model_name: Name of model to operate in + :type model_name: str + :returns: Dict of ceph df output + :rtype: dict + :raise: zaza.model.CommandRunFailed + """ + cmd = 'sudo ceph df --format=json' + result = zaza_model.run_on_unit(unit_name, cmd, model_name=model_name) + if result.get('Code') != '0': + raise zaza_model.CommandRunFailed(cmd, result) + return json.loads(result.get('Stdout')) + + +def get_ceph_pool_sample(unit_name, pool_id=0, model_name=None): + """Return list of ceph pool attributes. + + Take a sample of attributes of a ceph pool, returning ceph + pool name, object count and disk space used for the specified + pool ID number. + + :param unit_name: Name of the unit to get the pool sample + :type unit_name: string + :param pool_id: Ceph pool ID + :type pool_id: int + :param model_name: Name of model to operate in + :type model_name: str + :returns: List of pool name, object count, kb disk space used + :rtype: list + :raises: zaza.model.CommandRunFailed + """ + df = get_ceph_df(unit_name, model_name) + for pool in df['pools']: + if pool['id'] == pool_id: + pool_name = pool['name'] + obj_count = pool['stats']['objects'] + kb_used = pool['stats']['kb_used'] + + logging.debug('Ceph {} pool (ID {}): {} objects, ' + '{} kb used'.format(pool_name, pool_id, + obj_count, kb_used)) + return pool_name, obj_count, kb_used + + def get_rbd_hash(unit_name, pool, image, model_name=None): """Get SHA512 hash of RBD image. From 0080d84c33f579703aa68972bc9b158c98e55164 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 10 Oct 2019 11:21:59 +0000 Subject: [PATCH 088/898] dist upgrade before series upgrade --- .../utilities/test_zaza_utilities_generic.py | 11 ++++++++++ zaza/openstack/utilities/generic.py | 21 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 4712d75..4869ae4 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -166,6 +166,17 @@ class TestGenericUtils(ut_utils.BaseTestCase): _yaml_dict) self._open.assert_called_once_with(_filename, "r") + def test_dist_upgrade(self): + _unit = "app/2" + generic_utils.dist_upgrade(_unit) + dist_upgrade_cmd = ( + """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + self.model.run_on_unit.assert_has_calls([ + mock.call(_unit, 'sudo apt update'), + mock.call(_unit, dist_upgrade_cmd)]) + def test_do_release_upgrade(self): _unit = "app/2" generic_utils.do_release_upgrade(_unit) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 9bb4bc9..901d9e6 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -374,6 +374,7 @@ def series_upgrade(unit_name, machine_num, model.block_until_unit_wl_status(unit_name, "blocked") logging.info("Waiting for model idleness") model.block_until_all_units_idle() + dist_upgrade(unit_name) wrap_do_release_upgrade(unit_name, from_series=from_series, to_series=to_series, files=files, workaround_script=workaround_script) @@ -481,6 +482,26 @@ def run_via_ssh(unit_name, cmd): logging.warn(e) +def dist_upgrade(unit_name): + """Run dist-upgrade on unit after update package db. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Updating package db ' + unit_name) + update_cmd = 'sudo apt update' + model.run_on_unit(unit_name, update_cmd) + + logging.info('Updating existing packages ' + unit_name) + dist_upgrade_cmd = ( + """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + model.run_on_unit(unit_name, dist_upgrade_cmd) + + def do_release_upgrade(unit_name): """Run do-release-upgrade noninteractive. From 7fd66a7a147277f01434b84f09cd75f986831da9 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 10 Oct 2019 12:22:18 +0000 Subject: [PATCH 089/898] Move dist upgrade earlier so agent is still running --- zaza/openstack/utilities/generic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 901d9e6..dd9e599 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -367,6 +367,7 @@ def series_upgrade(unit_name, machine_num, logging.info("Series upgrade {}".format(unit_name)) application = unit_name.split('/')[0] set_dpkg_non_interactive_on_unit(unit_name) + dist_upgrade(unit_name) logging.info("Prepare series upgrade on {}".format(machine_num)) model.prepare_series_upgrade(machine_num, to_series=to_series) logging.info("Waiting for workload status 'blocked' on {}" @@ -374,7 +375,6 @@ def series_upgrade(unit_name, machine_num, model.block_until_unit_wl_status(unit_name, "blocked") logging.info("Waiting for model idleness") model.block_until_all_units_idle() - dist_upgrade(unit_name) wrap_do_release_upgrade(unit_name, from_series=from_series, to_series=to_series, files=files, workaround_script=workaround_script) From c05f784669a04c35b88b803fc72c442790cab8fc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 11 Oct 2019 11:30:16 +0000 Subject: [PATCH 090/898] Saml tests rely on lxml --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 20679d4..7fe8b81 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ install_require = [ 'jinja2', 'juju', 'juju-wait', + 'lxml', 'PyYAML', 'tenacity', 'oslo.config', From 9f73b218acf87fc056558049693cac3e3e0a9d4e Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 11 Oct 2019 11:40:10 +0000 Subject: [PATCH 091/898] Ensure units are idle before prepare_series_upgrade Ensure all units are idel before running prepare_series_upgrade. Closes issue #88 --- zaza/openstack/utilities/generic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index dd9e599..7eb175c 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -368,6 +368,7 @@ def series_upgrade(unit_name, machine_num, application = unit_name.split('/')[0] set_dpkg_non_interactive_on_unit(unit_name) dist_upgrade(unit_name) + model.block_until_all_units_idle() logging.info("Prepare series upgrade on {}".format(machine_num)) model.prepare_series_upgrade(machine_num, to_series=to_series) logging.info("Waiting for workload status 'blocked' on {}" From f0d2853aec58d4f81a2e262fc1805806500fde6e Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sat, 12 Oct 2019 07:31:06 +0000 Subject: [PATCH 092/898] No origin for designate-bind series upgrade --- zaza/openstack/charm_tests/series_upgrade/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 5c088e9..b5c1872 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -70,6 +70,8 @@ class SeriesUpgradeTest(unittest.TestCase): origin = "source" pause_non_leader_primary = False pause_non_leader_subordinate = False + if "designate-bind" in applications[application]["charm"]: + origin = None if "memcached" in applications[application]["charm"]: origin = None pause_non_leader_primary = False From 34a8cc14b88c3b58d1c520598041865d496cee39 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 13 Oct 2019 09:50:21 +0000 Subject: [PATCH 093/898] No origin for designate-bind series upgrade --- zaza/openstack/charm_tests/series_upgrade/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index b5c1872..6d75fb9 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -72,6 +72,8 @@ class SeriesUpgradeTest(unittest.TestCase): pause_non_leader_subordinate = False if "designate-bind" in applications[application]["charm"]: origin = None + if "tempest" in applications[application]["charm"]: + origin = None if "memcached" in applications[application]["charm"]: origin = None pause_non_leader_primary = False From 9bceee55c8c1aa04b545b878e5e7c8187f2fdc11 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 12:17:55 +1000 Subject: [PATCH 094/898] Add function that returns a map of unit names -> hostnames --- zaza/openstack/utilities/generic.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 7eb175c..b2f7354 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -75,6 +75,16 @@ def get_network_config(net_topology, ignore_env_vars=False, return net_info +def get_unit_hostnames(units): + """Return a dict of juju unit names to hostnames.""" + host_names = {} + for unit in units: + output = model.run_on_unit(unit.entity_id, 'hostname') + hostname = output['Stdout'].strip() + host_names[unit.entity_id] = hostname + return host_names + + def get_pkg_version(application, pkg): """Return package version. From 27bd15a9e0cac4958a85b7eba9999e387dc7e808 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 12:18:43 +1000 Subject: [PATCH 095/898] Add a function that returns contents of a file from a unit --- zaza/openstack/utilities/generic.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index b2f7354..8b562ab 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -716,6 +716,11 @@ def get_ubuntu_release(ubuntu_name): return index +def get_file_contents(unit, f): + return model.run_on_unit(unit.entity_id, + "cat {}".format(f))['Stdout'] + + def is_port_open(port, address): """Determine if TCP port is accessible. From 2888cb5e92615f9f8c44c1be7fe6c099f0919472 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 12:19:25 +1000 Subject: [PATCH 096/898] Add a function that checks port access given a list of units --- zaza/openstack/utilities/generic.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 8b562ab..c8e942e 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -744,3 +744,21 @@ def is_port_open(port, address): logging.error("connection refused connecting" " to {}:{}".format(address, port)) return False + + +def port_knock_units(units, port=22, expect_success=True): + """Open a TCP socket to check for a listening sevice on each + listed juju unit. + :param units: list of unit pointers + :param port: TCP port number, default to 22 + :param timeout: Connect timeout, default to 15 seconds + :expect_success: True by default, set False to invert logic + :returns: None if successful, Failure message otherwise + """ + for u in units: + host = u.public_address + connected = is_port_open(port, host) + if not connected and expect_success: + return 'Socket connect failed.' + elif connected and not expect_success: + return 'Socket connected unexpectedly.' From 31c71f4061def1a4ed363a534c1788d58087d8d3 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 12:20:26 +1000 Subject: [PATCH 097/898] Add function that runs/checks a command on all units, expecting exit 0 --- zaza/openstack/utilities/generic.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index c8e942e..f026321 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -513,6 +513,31 @@ def dist_upgrade(unit_name): model.run_on_unit(unit_name, dist_upgrade_cmd) +def check_commands_on_units(commands, units): + """Check that all commands in a list exit zero on all + units in a list. + :param commands: list of bash commands + :param units: list of unit pointers + :returns: None if successful; Failure message otherwise + """ + logging.debug('Checking exit codes for {} commands on {} ' + 'units...'.format(len(commands), + len(units))) + + for u in units: + for cmd in commands: + output = model.run_on_unit(u.entity_id, cmd) + if int(output['Code']) == 0: + logging.debug('{} `{}` returned {} ' + '(OK)'.format(u.entity_id, + cmd, output['Code'])) + else: + return ('{} `{}` returned {} ' + '{}'.format(u.entity_id, + cmd, output['Code'], output)) + return None + + def do_release_upgrade(unit_name): """Run do-release-upgrade noninteractive. From cf91a9dab940f7dfdd305eacc416e03a0d05f6e9 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:07:22 +1000 Subject: [PATCH 098/898] Create rabbitmq_server.utils file (add a license as a starter) --- .../charm_tests/rabbitmq_server/__init__.py | 15 +++++++++++++++ .../charm_tests/rabbitmq_server/utils.py | 13 +++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 zaza/openstack/charm_tests/rabbitmq_server/__init__.py create mode 100644 zaza/openstack/charm_tests/rabbitmq_server/utils.py diff --git a/zaza/openstack/charm_tests/rabbitmq_server/__init__.py b/zaza/openstack/charm_tests/rabbitmq_server/__init__.py new file mode 100644 index 0000000..cdac408 --- /dev/null +++ b/zaza/openstack/charm_tests/rabbitmq_server/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing rabbitmq server.""" diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py new file mode 100644 index 0000000..e429521 --- /dev/null +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -0,0 +1,13 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 2fb7dcbdfc76626a949ea8267e53ea4b47d8c441 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:09:09 +1000 Subject: [PATCH 099/898] Add function that waits until Rmq units are clustered and ready --- .../charm_tests/rabbitmq_server/utils.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index e429521..7599511 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -11,3 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import zaza.model + + +def wait_for_cluster(model_name=None, timeout=1200): + """Wait for rmq units extended status to show cluster readiness, + after an optional initial sleep period. Initial sleep is likely + necessary to be effective following a config change, as status + message may not instantly update to non-ready.""" + states = { + 'rabbitmq-server': { + 'workload-status-messages': 'Unit is ready and clustered' + } + } + + zaza.model.wait_for_application_states(model_name=model_name, + states=states, + timeout=timeout) + + From 47657d6113a2686323aec00c97970632c25be13f Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:12:10 +1000 Subject: [PATCH 100/898] Add a function for getting Rmq cluster_status info from a unit --- .../openstack/charm_tests/rabbitmq_server/utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 7599511..3ef9836 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import zaza.model @@ -31,3 +32,16 @@ def wait_for_cluster(model_name=None, timeout=1200): timeout=timeout) +def get_cluster_status(unit): + """Execute rabbitmq cluster status command on a unit and return + the full output. + :param unit: unit + :returns: String containing console output of cluster status command + """ + cmd = 'rabbitmqctl cluster_status' + output = zaza.model.run_on_unit(unit.entity_id, cmd)['Stdout'].strip() + logging.debug('{} cluster_status:\n{}'.format( + unit.entity_id, output)) + return str(output) + + From fa68f09e721282e05701ad6fcaff55d3276e26ac Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:13:56 +1000 Subject: [PATCH 101/898] Add a function providing all units/nodes Rmq cluster_status in JSON --- .../charm_tests/rabbitmq_server/utils.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 3ef9836..66cb1c8 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import zaza.model @@ -45,3 +46,22 @@ def get_cluster_status(unit): return str(output) +def get_cluster_running_nodes(unit): + """Parse rabbitmqctl cluster_status output string, return list of + running rabbitmq cluster nodes. + :param unit: unit pointer + :returns: List containing node names of running nodes + """ + # NOTE(beisner): rabbitmqctl cluster_status output is not + # json-parsable, do string chop foo, then json.loads that. + str_stat = get_cluster_status(unit) + if 'running_nodes' in str_stat: + pos_start = str_stat.find("{running_nodes,") + 15 + pos_end = str_stat.find("]},", pos_start) + 1 + str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"') + run_nodes = json.loads(str_run_nodes) + return run_nodes + else: + return [] + + From 6749ff763ed9574b60a31305ea62fc9a0d9a092b Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:25:32 +1000 Subject: [PATCH 102/898] Add a function for creating connection to Rmq instance on unit --- .../charm_tests/rabbitmq_server/utils.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 66cb1c8..734c8bb 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -14,8 +14,11 @@ import json import logging + +import pika import zaza.model +import ssl as libssl def wait_for_cluster(model_name=None, timeout=1200): """Wait for rmq units extended status to show cluster readiness, @@ -65,3 +68,55 @@ def get_cluster_running_nodes(unit): return [] +def connect_amqp_by_unit(unit, ssl=False, + port=None, fatal=True, + username="testuser1", password="changeme"): + """Establish and return a pika amqp connection to the rabbitmq service + running on a rmq juju unit. + :param unit: unit pointer + :param ssl: boolean, default to False + :param port: amqp port, use defaults if None + :param fatal: boolean, default to True (raises on connect error) + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :returns: pika amqp connection pointer or None if failed and non-fatal + """ + host = unit.public_address + unit_name = unit.entity_id + + if ssl: + ssl_options = pika.SSLOptions(libssl.SSLContext()) + else: + ssl_options = None + + # Default port logic if port is not specified + if ssl and not port: + port = 5671 + elif not ssl and not port: + port = 5672 + + logging.debug('Connecting to amqp on {}:{} ({}) as ' + '{}...'.format(host, port, unit_name, username)) + + try: + credentials = pika.PlainCredentials(username, password) + parameters = pika.ConnectionParameters(host=host, port=port, + credentials=credentials, + ssl_options=ssl_options, + connection_attempts=3, + retry_delay=5, + socket_timeout=1) + connection = pika.BlockingConnection(parameters) + assert connection.is_open is True + logging.debug('Connect OK') + return connection + except Exception as e: + msg = ('amqp connection failed to {}:{} as ' + '{} ({})'.format(host, port, username, str(e))) + if fatal: + raise Exception(msg) + else: + logging.warn(msg) + return None + + From efb781b04817d5c09c7afb67f7e951f5abd80c4d Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:39:35 +1000 Subject: [PATCH 103/898] Create rabbitmq_server.tests file (add a license as a starter) --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 zaza/openstack/charm_tests/rabbitmq_server/tests.py diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py new file mode 100644 index 0000000..e429521 --- /dev/null +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -0,0 +1,13 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From e0707fac044562bb2aa1aabfd5f481e7d03a904e Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:41:29 +1000 Subject: [PATCH 104/898] Add an RmqTests class (tests to follow...) --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index e429521..d50d8c2 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -11,3 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import zaza.openstack.charm_tests.test_utils as test_utils + + +class RmqTests(test_utils.OpenStackBaseTest): + """Zaza tests on a basic rabbitmq cluster deployment. Verify + relations, service status, users and endpoint service catalog.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(RmqTests, cls).setUpClass() + From 313f07bd0c267034efb4f5fdb14c6b04db5bbefe Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:42:08 +1000 Subject: [PATCH 105/898] Add test confirming that all Rmq units are part of cluster --- .../charm_tests/rabbitmq_server/tests.py | 17 ++++++++++ .../charm_tests/rabbitmq_server/utils.py | 32 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index d50d8c2..1168759 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging +import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils +from . import utils as rmq_utils + class RmqTests(test_utils.OpenStackBaseTest): """Zaza tests on a basic rabbitmq cluster deployment. Verify @@ -24,3 +28,16 @@ class RmqTests(test_utils.OpenStackBaseTest): """Run class setup for running tests.""" super(RmqTests, cls).setUpClass() + def test_400_rmq_cluster_running_nodes(self): + """Verify that cluster status from each rmq juju unit shows + every cluster node as a running member in that cluster.""" + logging.debug('Checking that all units are in cluster_status ' + 'running nodes...') + + units = zaza.model.get_units(self.application_name) + + ret = rmq_utils.validate_cluster_running_nodes(units) + self.assertIsNone(ret) + + logging.info('OK\n') + diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 734c8bb..1335dd6 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -19,6 +19,8 @@ import pika import zaza.model import ssl as libssl +import zaza.openstack.utilities.generic as generic_utils + def wait_for_cluster(model_name=None, timeout=1200): """Wait for rmq units extended status to show cluster readiness, @@ -68,6 +70,36 @@ def get_cluster_running_nodes(unit): return [] +def validate_cluster_running_nodes(units): + """Check that all rmq unit hostnames are represented in the + cluster_status output of all units. + :param host_names: dict of juju unit names to host names + :param units: list of unit pointers (all rmq units) + :returns: None if successful, otherwise return error message + """ + host_names = generic_utils.get_unit_hostnames(units) + errors = [] + + # Query every unit for cluster_status running nodes + for query_unit in units: + query_unit_name = query_unit.entity_id + running_nodes = get_cluster_running_nodes(query_unit) + + # Confirm that every unit is represented in the queried unit's + # cluster_status running nodes output. + for validate_unit in units: + val_host_name = host_names[validate_unit.entity_id] + val_node_name = 'rabbit@{}'.format(val_host_name) + + if val_node_name not in running_nodes: + errors.append('Cluster member check failed on {}: {} not ' + 'in {}\n'.format(query_unit_name, + val_node_name, + running_nodes)) + if errors: + return ''.join(errors) + + def connect_amqp_by_unit(unit, ssl=False, port=None, fatal=True, username="testuser1", password="changeme"): From 3a64bec97aebd178901ff71285679e1d8da8ca56 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:55:31 +1000 Subject: [PATCH 106/898] Add function for adding a user to an Rmq cluster --- .../charm_tests/rabbitmq_server/utils.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 1335dd6..2ccebe3 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -38,6 +38,43 @@ def wait_for_cluster(model_name=None, timeout=1200): timeout=timeout) +def add_user(units, username="testuser1", password="changeme"): + """Add a user via the first rmq juju unit, check connection as + the new user against all units. + :param units: list of unit pointers + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :returns: None if successful. Raise on error. + """ + logging.debug('Adding rmq user ({})...'.format(username)) + + # Check that user does not already exist + cmd_user_list = 'rabbitmqctl list_users' + cmd_result = zaza.model.run_on_unit(units[0].entity_id, cmd_user_list) + output = cmd_result['Stdout'].strip() + if username in output: + logging.warning('User ({}) already exists, returning ' + 'gracefully.'.format(username)) + return + + perms = '".*" ".*" ".*"' + cmds = ['rabbitmqctl add_user {} {}'.format(username, password), + 'rabbitmqctl set_permissions {} {}'.format(username, perms)] + + # Add user via first unit + for cmd in cmds: + cmd_result = zaza.model.run_on_unit(units[0].entity_id, cmd) + output = cmd_result['Stdout'].strip() + + # Check connection against the other units + logging.debug('Checking user connect against units...') + for u in units: + connection = connect_amqp_by_unit(u, ssl=False, + username=username, + password=password) + connection.close() + + def get_cluster_status(unit): """Execute rabbitmq cluster status command on a unit and return the full output. From 6d3714aa06da7b56f8c075b81528680cd0a0a49a Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:55:59 +1000 Subject: [PATCH 107/898] Add a function for deleting a user from a Rmq cluster --- .../charm_tests/rabbitmq_server/utils.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 2ccebe3..0380d70 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -75,6 +75,30 @@ def add_user(units, username="testuser1", password="changeme"): connection.close() +def delete_user(units, username="testuser1"): + """Delete a rabbitmq user via the first rmq juju unit. + :param units: list of unit pointers + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :returns: None if successful or no such user. + """ + logging.debug('Deleting rmq user ({})...'.format(username)) + + # Check that the user exists + cmd_user_list = 'rabbitmqctl list_users' + output = zaza.model.run_on_unit(units[0].entity_id, + cmd_user_list)['Stdout'].strip() + + if username not in output: + logging.warning('User ({}) does not exist, returning ' + 'gracefully.'.format(username)) + return + + # Delete the user + cmd_user_del = 'rabbitmqctl delete_user {}'.format(username) + output = zaza.model.run_on_unit(units[0].entity_id, cmd_user_del) + + def get_cluster_status(unit): """Execute rabbitmq cluster status command on a unit and return the full output. From 79a9f179dfd962447b6cc159061ac5708c8404fb Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:05:28 +1000 Subject: [PATCH 108/898] Add function for determing if SSL is enabled in unit's Rmq config --- .../charm_tests/rabbitmq_server/utils.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 0380d70..e52479c 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -161,6 +161,42 @@ def validate_cluster_running_nodes(units): return ''.join(errors) +def is_ssl_enabled_on_unit(unit, port=None): + """Check a single juju rmq unit for ssl and port in the config file.""" + host = unit.public_address + unit_name = unit.entity_id + + conf_file = '/etc/rabbitmq/rabbitmq.config' + conf_contents = str(generic_utils.get_file_contents(unit, + conf_file)) + # Checks + conf_ssl = 'ssl' in conf_contents + conf_port = str(port) in conf_contents + + # Port explicitly checked in config + if port and conf_port and conf_ssl: + logging.debug('SSL is enabled @{}:{} ' + '({})'.format(host, port, unit_name)) + return True + elif port and not conf_port and conf_ssl: + logging.debug('SSL is enabled @{} but not on port {} ' + '({})'.format(host, port, unit_name)) + return False + # Port not checked (useful when checking that ssl is disabled) + elif not port and conf_ssl: + logging.debug('SSL is enabled @{}:{} ' + '({})'.format(host, port, unit_name)) + return True + elif not conf_ssl: + logging.debug('SSL not enabled @{}:{} ' + '({})'.format(host, port, unit_name)) + return False + else: + msg = ('Unknown condition when checking SSL status @{}:{} ' + '({})'.format(host, port, unit_name)) + raise ValueError(msg) + + def connect_amqp_by_unit(unit, ssl=False, port=None, fatal=True, username="testuser1", password="changeme"): From c917c5b370c72a9b9b77553f01344fe1d0b17d93 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:06:51 +1000 Subject: [PATCH 109/898] Add function validating if SSL is enabled on Rmq units --- zaza/openstack/charm_tests/rabbitmq_server/utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index e52479c..8ee5b17 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -161,6 +161,19 @@ def validate_cluster_running_nodes(units): return ''.join(errors) +def validate_ssl_enabled_units(units, port=None): + """Check that ssl is enabled on rmq juju units. + :param units: list of all rmq units + :param port: optional ssl port override to validate + :returns: None if successful, otherwise return error message + """ + for u in units: + if not is_ssl_enabled_on_unit(u, port=port): + return ('Unexpected condition: ssl is disabled on unit ' + '({})'.format(u.info['unit_name'])) + return None + + def is_ssl_enabled_on_unit(unit, port=None): """Check a single juju rmq unit for ssl and port in the config file.""" host = unit.public_address From 97cc2b8e6040ddff9bb01c235234fb437c244d3e Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:07:25 +1000 Subject: [PATCH 110/898] Add function validating is SSL is disabled on Rmq units --- zaza/openstack/charm_tests/rabbitmq_server/utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 8ee5b17..99545e1 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -174,6 +174,18 @@ def validate_ssl_enabled_units(units, port=None): return None +def validate_ssl_disabled_units(units): + """Check that ssl is enabled on listed rmq juju units. + :param units: list of all rmq units + :returns: True if successful. Raise on error. + """ + for u in units: + if is_ssl_enabled_on_unit(u): + return ('Unexpected condition: ssl is enabled on unit ' + '({})'.format(u.entity_id)) + return None + + def is_ssl_enabled_on_unit(unit, port=None): """Check a single juju rmq unit for ssl and port in the config file.""" host = unit.public_address From 79d70e4f8d9ee3710e221504b532a34be97659dc Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:12:00 +1000 Subject: [PATCH 111/898] Add a function for enabling SSL on Rmq units --- .../charm_tests/rabbitmq_server/utils.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 99545e1..2c945e7 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -14,6 +14,7 @@ import json import logging +import time import pika import zaza.model @@ -186,6 +187,43 @@ def validate_ssl_disabled_units(units): return None +def configure_ssl_on(units, model_name=None, + port=None, max_wait=60): + """Turn ssl charm config option on, with optional non-default + ssl port specification. Confirm that it is enabled on every + unit. + :param units: list of units + :param port: amqp port, use defaults if None + :param max_wait: maximum time to wait in seconds to confirm + :returns: None if successful. Raise on error. + """ + logging.debug('Setting ssl charm config option: on') + + # Enable RMQ SSL + config = {'ssl': 'on'} + if port: + config['ssl_port'] = str(port) + + zaza.model.set_application_config('rabbitmq-server', + config, + model_name=model_name) + + # Wait for unit status + wait_for_cluster(model_name) + + # Confirm + tries = 0 + ret = validate_ssl_enabled_units(units, port=port) + while ret and tries < (max_wait / 4): + time.sleep(4) + logging.debug('Attempt {}: {}'.format(tries, ret)) + ret = validate_ssl_enabled_units(units, port=port) + tries += 1 + + if ret: + raise Exception(ret) + + def is_ssl_enabled_on_unit(unit, port=None): """Check a single juju rmq unit for ssl and port in the config file.""" host = unit.public_address From e09660f7c12eb761525664e468c6bf0f96722f35 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:12:27 +1000 Subject: [PATCH 112/898] Add a function for disabling SSL on Rmq units --- .../charm_tests/rabbitmq_server/utils.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 2c945e7..450855a 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -224,6 +224,37 @@ def configure_ssl_on(units, model_name=None, raise Exception(ret) +def configure_ssl_off(units, model_name=None, max_wait=60): + """Turn ssl charm config option off, confirm that it is disabled + on every unit. + :param units: list of units + :param max_wait: maximum time to wait in seconds to confirm + :returns: None if successful. Raise on error. + """ + logging.debug('Setting ssl charm config option: off') + + # Disable RMQ SSL + config = {'ssl': 'off'} + zaza.model.set_application_config('rabbitmq-server', + config, + model_name=model_name) + + # Wait for unit status + wait_for_cluster(model_name) + + # Confirm + tries = 0 + ret = validate_ssl_disabled_units(units) + while ret and tries < (max_wait / 4): + time.sleep(4) + logging.debug('Attempt {}: {}'.format(tries, ret)) + ret = validate_ssl_disabled_units(units) + tries += 1 + + if ret: + raise Exception(ret) + + def is_ssl_enabled_on_unit(unit, port=None): """Check a single juju rmq unit for ssl and port in the config file.""" host = unit.public_address From 20aad97033def58a677e2b041051ad5e7aed286c Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:28:56 +1000 Subject: [PATCH 113/898] Add a function for publishing a message to an Rmq unit --- .../charm_tests/rabbitmq_server/utils.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 450855a..fd08ee5 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -343,3 +343,40 @@ def connect_amqp_by_unit(unit, ssl=False, return None +def publish_amqp_message_by_unit(unit, message, + queue="test", ssl=False, + username="testuser1", + password="changeme", + port=None): + """Publish an amqp message to a rmq juju unit. + :param unit: unit pointer + :param message: amqp message string + :param queue: message queue, default to test + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :param ssl: boolean, default to False + :param port: amqp port, use defaults if None + :returns: None. Raises exception if publish failed. + """ + logging.debug('Publishing message to {} queue:\n{}'.format(queue, + message)) + connection = connect_amqp_by_unit(unit, ssl=ssl, + port=port, + username=username, + password=password) + + # NOTE(beisner): extra debug here re: pika hang potential: + # https://github.com/pika/pika/issues/297 + # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw + logging.debug('Defining channel...') + channel = connection.channel() + logging.debug('Declaring queue...') + channel.queue_declare(queue=queue, auto_delete=False, durable=True) + logging.debug('Publishing message...') + channel.basic_publish(exchange='', routing_key=queue, body=message) + logging.debug('Closing channel...') + channel.close() + logging.debug('Closing connection...') + connection.close() + + From 1acbfd874163112fbd7a681f152b348d1f08fc41 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:29:56 +1000 Subject: [PATCH 114/898] Add a function for retrieving a message to an Rmq unit --- .../charm_tests/rabbitmq_server/utils.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index fd08ee5..2447f8f 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -380,3 +380,34 @@ def publish_amqp_message_by_unit(unit, message, connection.close() +def get_amqp_message_by_unit(unit, queue="test", + username="testuser1", + password="changeme", + ssl=False, port=None): + """Get an amqp message from a rmq juju unit. + :param unit: unit pointer + :param queue: message queue, default to test + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :param ssl: boolean, default to False + :param port: amqp port, use defaults if None + :returns: amqp message body as string. Raise if get fails. + """ + connection = connect_amqp_by_unit(unit, ssl=ssl, + port=port, + username=username, + password=password) + channel = connection.channel() + method_frame, _, body = channel.basic_get(queue) + body = body.decode() + + if method_frame: + logging.debug('Retreived message from {} queue:\n{}'.format(queue, + body)) + channel.basic_ack(method_frame.delivery_tag) + channel.close() + connection.close() + return body + else: + msg = 'No message retrieved.' + raise Exception(msg) From 1baff7281cba7d29b4f0aca60acb1ab92a667775 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:31:52 +1000 Subject: [PATCH 115/898] Helper function for testing if messages move across cluster successfully --- .../charm_tests/rabbitmq_server/tests.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 1168759..9808150 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -13,8 +13,13 @@ # limitations under the License. import logging +import time +import uuid + import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.generic as generic_utils + from . import utils as rmq_utils @@ -28,6 +33,86 @@ class RmqTests(test_utils.OpenStackBaseTest): """Run class setup for running tests.""" super(RmqTests, cls).setUpClass() + def _get_uuid_epoch_stamp(self): + """Returns a stamp string based on uuid4 and epoch time. Useful in + generating test messages which need to be unique-ish.""" + return '[{}-{}]'.format(uuid.uuid4(), time.time()) + + def _test_rmq_amqp_messages_all_units(self, units, + ssl=False, port=None): + """Reusable test to send amqp messages to every listed rmq unit + and check every listed rmq unit for messages. + + :param units: list of units + :returns: None if successful. Raise on error. + """ + + # Add test user if it does not already exist + rmq_utils.add_user(units) + + # Handle ssl (includes wait-for-cluster) + if ssl: + rmq_utils.configure_ssl_on(units, port=port) + else: + rmq_utils.configure_ssl_off(units) + + # Publish and get amqp messages in all possible unit combinations. + # Qty of checks == (qty of units) ^ 2 + amqp_msg_counter = 1 + host_names = generic_utils.get_unit_hostnames(units) + + for dest_unit in units: + dest_unit_name = dest_unit.entity_id + dest_unit_host = dest_unit.public_address + dest_unit_host_name = host_names[dest_unit_name] + + for check_unit in units: + check_unit_name = check_unit.entity_id + check_unit_host = check_unit.public_address + check_unit_host_name = host_names[check_unit_name] + + amqp_msg_stamp = self._get_uuid_epoch_stamp() + amqp_msg = ('Message {}@{} {}'.format(amqp_msg_counter, + dest_unit_host, + amqp_msg_stamp)).upper() + # Publish amqp message + logging.debug('Publish message to: {} ' + '({} {})'.format(dest_unit_host, + dest_unit_name, + dest_unit_host_name)) + + rmq_utils.publish_amqp_message_by_unit(dest_unit, + amqp_msg, ssl=ssl, + port=port) + + # Wait a bit before checking for message + time.sleep(10) + + # Get amqp message + logging.debug('Get message from: {} ' + '({} {})'.format(check_unit_host, + check_unit_name, + check_unit_host_name)) + + amqp_msg_rcvd = rmq_utils.get_amqp_message_by_unit(check_unit, + ssl=ssl, + port=port) + + # Validate amqp message content + if amqp_msg == amqp_msg_rcvd: + logging.debug('Message {} received ' + 'OK.'.format(amqp_msg_counter)) + else: + logging.error('Expected: {}'.format(amqp_msg)) + logging.error('Actual: {}'.format(amqp_msg_rcvd)) + msg = 'Message {} mismatch.'.format(amqp_msg_counter) + raise Exception(msg) + + amqp_msg_counter += 1 + + # Delete the test user + rmq_utils.delete_user(units) + def test_400_rmq_cluster_running_nodes(self): """Verify that cluster status from each rmq juju unit shows every cluster node as a running member in that cluster.""" From 79c816ed0f49358c51dd2c3f9cc411c9995cd03c Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:47:50 +1000 Subject: [PATCH 116/898] Test: messages can be sent/retrieved to/from Rmq cluster without SSL --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 9808150..37375d2 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -126,3 +126,13 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') + def test_406_rmq_amqp_messages_all_units_ssl_off(self): + """Send amqp messages to every rmq unit and check every rmq unit + for messages. Standard amqp tcp port, no ssl.""" + logging.debug('Checking amqp message publish/get on all units ' + '(ssl off)...') + + units = zaza.model.get_units(self.application_name) + self._test_rmq_amqp_messages_all_units(units, ssl=False) + logging.info('OK\n') + From 405cdf63885b5da6642c08d55dbce597c598f873 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:50:18 +1000 Subject: [PATCH 117/898] Test: RMQ management plugin enabling/disabling exposes/hides ports --- .../charm_tests/rabbitmq_server/tests.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 37375d2..33d52f5 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -136,3 +136,54 @@ class RmqTests(test_utils.OpenStackBaseTest): self._test_rmq_amqp_messages_all_units(units, ssl=False) logging.info('OK\n') + def test_412_rmq_management_plugin(self): + """Enable and check management plugin.""" + logging.debug('Checking tcp socket connect to management plugin ' + 'port on all rmq units...') + + units = zaza.model.get_units(self.application_name) + mgmt_port = 15672 + + # Enable management plugin + logging.debug('Enabling management_plugin charm config option...') + config = {'management_plugin': 'True'} + zaza.model.set_application_config('rabbitmq-server', config) + rmq_utils.wait_for_cluster() + + # Check tcp connect to management plugin port + max_wait = 600 + tries = 0 + ret = generic_utils.port_knock_units(units, mgmt_port) + while ret and tries < (max_wait / 30): + time.sleep(30) + logging.debug('Attempt {}: {}'.format(tries, ret)) + ret = generic_utils.port_knock_units(units, mgmt_port) + tries += 1 + + self.assertIsNone(ret) + logging.debug('Connect to all units (OK)\n') + + # Disable management plugin + logging.debug('Disabling management_plugin charm config option...') + config = {'management_plugin': 'False'} + zaza.model.set_application_config('rabbitmq-server', config) + rmq_utils.wait_for_cluster() + + # Negative check - tcp connect to management plugin port + logging.info('Expect tcp connect fail since charm config ' + 'option is disabled.') + tries = 0 + ret = generic_utils.port_knock_units(units, + mgmt_port, + expect_success=False) + + while ret and tries < (max_wait / 30): + time.sleep(30) + logging.debug('Attempt {}: {}'.format(tries, ret)) + ret = generic_utils.port_knock_units(units, mgmt_port, + expect_success=False) + tries += 1 + + self.assertIsNone(ret) + logging.info('Confirm mgmt port closed on all units (OK)\n') + From c5aba589ce0f8223bec012b2b4d1b289d0b012bf Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:14:04 +1000 Subject: [PATCH 118/898] Test: existence NRPE configs/data on Rmq units --- .../charm_tests/rabbitmq_server/tests.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 33d52f5..6faf015 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -187,3 +187,42 @@ class RmqTests(test_utils.OpenStackBaseTest): self.assertIsNone(ret) logging.info('Confirm mgmt port closed on all units (OK)\n') + def test_414_rmq_nrpe_monitors(self): + """Check rabbimq-server nrpe monitor basic functionality.""" + units = zaza.model.get_units(self.application_name) + host_names = generic_utils.get_unit_hostnames(units) + + # check_rabbitmq monitor + logging.debug('Checking nrpe check_rabbitmq on units...') + cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_rabbitmq.cfg'] + ret = generic_utils.check_commands_on_units(cmds, units) + self.assertIsNone(ret) + + logging.debug('Sleeping 2ms for 1m cron job to run...') + time.sleep(120) + + # check_rabbitmq_queue monitor + logging.debug('Checking nrpe check_rabbitmq_queue on units...') + cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_rabbitmq_queue.cfg'] + ret = generic_utils.check_commands_on_units(cmds, units) + self.assertIsNone(ret) + + # check dat file existence + logging.debug('Checking nrpe dat file existence on units...') + for u in units: + unit_host_name = host_names[u.entity_id] + + cmds = [ + 'stat /var/lib/rabbitmq/data/{}_general_stats.dat'.format( + unit_host_name), + 'stat /var/lib/rabbitmq/data/{}_queue_stats.dat'.format( + unit_host_name) + ] + + ret = generic_utils.check_commands_on_units(cmds, [u]) + self.assertIsNone(ret) + + logging.info('OK\n') + From 9e7d76df2f655bd2d0a810780ec41428a712e306 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:17:34 +1000 Subject: [PATCH 119/898] Test: Rmq unit action 'cluster-status' returns a Juju action --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 6faf015..58ab127 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -16,6 +16,7 @@ import logging import time import uuid +import juju import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils @@ -226,3 +227,13 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') + def test_911_cluster_status(self): + """ rabbitmqctl cluster_status action can be returned. """ + logging.debug('Checking cluster status action...') + + unit = zaza.model.get_units(self.application_name)[0] + action = zaza.model.run_action(unit.entity_id, "cluster-status") + self.assertIsInstance(action, juju.action.Action) + + logging.debug('OK') + From 2a0b7bf8e28bb8cbd0396a54aa4164374ffa5f5b Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:18:47 +1000 Subject: [PATCH 120/898] Test: Rmq unit action 'check-queues' returns a Juju action --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 58ab127..6127522 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -237,3 +237,11 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('OK') + def test_912_check_queues(self): + """ rabbitmqctl check_queues action can be returned. """ + logging.debug('Checking cluster status action...') + + unit = zaza.model.get_units(self.application_name)[0] + action = zaza.model.run_action(unit.entity_id, "check-queues") + self.assertIsInstance(action, juju.action.Action) + From a738efd7f215fe1477f7419b107e04314c672708 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:24:32 +1000 Subject: [PATCH 121/898] Test: Rmq unit action 'list-unconsumed-queues' returns a Juju action --- .../charm_tests/rabbitmq_server/tests.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 6127522..32ff23c 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import time import uuid @@ -245,3 +246,31 @@ class RmqTests(test_utils.OpenStackBaseTest): action = zaza.model.run_action(unit.entity_id, "check-queues") self.assertIsInstance(action, juju.action.Action) + def test_913_list_unconsumed_queues(self): + """ rabbitmqctl list-unconsumed-queues action can be returned. """ + logging.debug('Checking list-unconsumed-queues action...') + + unit = zaza.model.get_units(self.application_name)[0] + self._test_rmq_amqp_messages_all_units([unit]) + action = zaza.model.run_action(unit.entity_id, + 'list-unconsumed-queues') + self.assertIsInstance(action, juju.action.Action) + + queue_count = int(action.results['unconsumed-queue-count']) + assert queue_count > 0, 'Did not find any unconsumed queues.' + + queue_name = 'test' # publish_amqp_message_by_unit default queue name + for i in range(queue_count): + queue_data = json.loads( + action.results['unconsumed-queues'][str(i)]) + if queue_data['name'] == queue_name: + break + else: + assert False, 'Did not find expected queue in result.' + + # Since we just reused _test_rmq_amqp_messages_all_units, we should + # have created the queue if it didn't already exist, but all messages + # should have already been consumed. + assert queue_data['messages'] == 0, 'Found unexpected message count.' + + logging.debug('OK') From 19c9e86b192633ed8a3a031317ea6cef43315dec Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:26:31 +1000 Subject: [PATCH 122/898] Test: Rmq pause/resume. Note: This test may have exposed a bug, where the `block_until_unit_wl_status` returns once it reaches the "maintenance" state, but subsequent queries to `unit.workload_status == "maintenance"` fail. Recreating the unit object (via `zaza.model.get_unit_from_name`) returns the correct workload_status when queried. --- .../charm_tests/rabbitmq_server/tests.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 32ff23c..dae8009 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -228,6 +228,30 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') + def test_910_pause_and_resume(self): + """The services can be paused and resumed. """ + + logging.debug('Checking pause and resume actions...') + + unit = zaza.model.get_units(self.application_name)[0] + assert unit.workload_status == "active" + + zaza.model.run_action(unit.entity_id, "pause") + zaza.model.block_until_unit_wl_status(unit.entity_id, "maintenance") + # TODO: investigate possible bug (the following line is + # required, otherwise it looks like workload_status is + # reporting cached information, no matter how long you sleep) + unit = zaza.model.get_unit_from_name(unit.entity_id) + assert unit.workload_status == "maintenance" + + zaza.model.run_action(unit.entity_id, "resume") + zaza.model.block_until_unit_wl_status(unit.entity_id, "active") + unit = zaza.model.get_unit_from_name(unit.entity_id) + assert unit.workload_status == "active" + + rmq_utils.wait_for_cluster() + logging.debug('OK') + def test_911_cluster_status(self): """ rabbitmqctl cluster_status action can be returned. """ logging.debug('Checking cluster status action...') From ce45a47dab9200841244daabf914096b8a75db7a Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:32:01 +1000 Subject: [PATCH 123/898] Test: messages can be sent/retrieved to/from Rmq cluster with SSL on As per the code: is there a function to determine unit's release? Otherwise, I'll just implement a generic function that run_on_unit lsb_release -cs --- .../charm_tests/rabbitmq_server/tests.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index dae8009..b13f186 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -138,6 +138,26 @@ class RmqTests(test_utils.OpenStackBaseTest): self._test_rmq_amqp_messages_all_units(units, ssl=False) logging.info('OK\n') + def test_408_rmq_amqp_messages_all_units_ssl_on(self): + """Send amqp messages with ssl enabled, to every rmq unit and + check every rmq unit for messages. Standard ssl tcp port.""" + # http://pad.lv/1625044 + # TODO: exsdev: find out if there's a function to determine unit's release + # Otherwise run_on_unit: lsb_release -cs + # if (CompareHostReleases(self.client_series) >= 'xenial' and + # CompareHostReleases(self.series) <= 'trusty'): + # logging.info('SKIP') + # logging.info('Skipping SSL tests due to client' + # ' compatibility issues') + # return + logging.debug('Checking amqp message publish/get on all units ' + '(ssl on)...') + + units = zaza.model.get_units(self.application_name) + self._test_rmq_amqp_messages_all_units(units, + ssl=True, port=5671) + logging.info('OK\n') + def test_412_rmq_management_plugin(self): """Enable and check management plugin.""" logging.debug('Checking tcp socket connect to management plugin ' From 1835d957a1c9a9f78c13b8cb30c2e481f0b8b476 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:32:16 +1000 Subject: [PATCH 124/898] Test: msgs can be sent/retrieved to/from Rmq cluster with SSL+alt port Same message as my previous commit: As per the code: is there a function to determine unit's release? Otherwise, I'll just implement a generic function that run_on_unit lsb_release -cs --- .../charm_tests/rabbitmq_server/tests.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index b13f186..55f0049 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -158,6 +158,26 @@ class RmqTests(test_utils.OpenStackBaseTest): ssl=True, port=5671) logging.info('OK\n') + def test_410_rmq_amqp_messages_all_units_ssl_alt_port(self): + """Send amqp messages with ssl on, to every rmq unit and check + every rmq unit for messages. Custom ssl tcp port.""" + # http://pad.lv/1625044 + # TODO: exsdev: find out if there's a function to determine unit's release + # Otherwise run_on_unit: lsb_release -cs + # if (CompareHostReleases(self.client_series) >= 'xenial' and + # CompareHostReleases(self.series) <= 'trusty'): + # logging.info('SKIP') + # logging.info('Skipping SSL tests due to client' + # ' compatibility issues') + # return + logging.debug('Checking amqp message publish/get on all units ' + '(ssl on)...') + + units = zaza.model.get_units(self.application_name) + self._test_rmq_amqp_messages_all_units(units, + ssl=True, port=5999) + logging.info('OK\n') + def test_412_rmq_management_plugin(self): """Enable and check management plugin.""" logging.debug('Checking tcp socket connect to management plugin ' From 0e163593306dd3407895ebb468d51c473ffa5bd5 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 20 Sep 2019 13:57:05 +1000 Subject: [PATCH 125/898] Check client+unit OS versions before running Rmq tests with SSL enabled --- .../charm_tests/rabbitmq_server/tests.py | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 55f0049..3e3e0f0 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -22,6 +22,10 @@ import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils +from charmhelpers.core.host import ( + lsb_release, + CompareHostReleases, +) from . import utils as rmq_utils @@ -138,18 +142,26 @@ class RmqTests(test_utils.OpenStackBaseTest): self._test_rmq_amqp_messages_all_units(units, ssl=False) logging.info('OK\n') + def _series(self, unit): + result = zaza.model.run_on_unit(unit.entity_id, + "lsb_release -cs") + return result['Stdout'].strip() + + def _client_series(self): + return lsb_release()['DISTRIB_CODENAME'] + def test_408_rmq_amqp_messages_all_units_ssl_on(self): """Send amqp messages with ssl enabled, to every rmq unit and check every rmq unit for messages. Standard ssl tcp port.""" + units = zaza.model.get_units(self.application_name) + # http://pad.lv/1625044 - # TODO: exsdev: find out if there's a function to determine unit's release - # Otherwise run_on_unit: lsb_release -cs - # if (CompareHostReleases(self.client_series) >= 'xenial' and - # CompareHostReleases(self.series) <= 'trusty'): - # logging.info('SKIP') - # logging.info('Skipping SSL tests due to client' - # ' compatibility issues') - # return + if (CompareHostReleases(self._client_series()) >= 'xenial' and + CompareHostReleases(self._series(units[0])) <= 'trusty'): + logging.info('SKIP') + logging.info('Skipping SSL tests due to client' + ' compatibility issues') + return logging.debug('Checking amqp message publish/get on all units ' '(ssl on)...') @@ -161,15 +173,15 @@ class RmqTests(test_utils.OpenStackBaseTest): def test_410_rmq_amqp_messages_all_units_ssl_alt_port(self): """Send amqp messages with ssl on, to every rmq unit and check every rmq unit for messages. Custom ssl tcp port.""" + units = zaza.model.get_units(self.application_name) + # http://pad.lv/1625044 - # TODO: exsdev: find out if there's a function to determine unit's release - # Otherwise run_on_unit: lsb_release -cs - # if (CompareHostReleases(self.client_series) >= 'xenial' and - # CompareHostReleases(self.series) <= 'trusty'): - # logging.info('SKIP') - # logging.info('Skipping SSL tests due to client' - # ' compatibility issues') - # return + if (CompareHostReleases(self._client_series()) >= 'xenial' and + CompareHostReleases(self._series(units[0])) <= 'trusty'): + logging.info('SKIP') + logging.info('Skipping SSL tests due to client' + ' compatibility issues') + return logging.debug('Checking amqp message publish/get on all units ' '(ssl on)...') From 4792a527c9503ec27ebc8d15f1c89e5bb473d22c Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Tue, 24 Sep 2019 16:00:05 +1000 Subject: [PATCH 126/898] PROTOCOL_TLS is not available until Py3.6, use alternative --- zaza/openstack/charm_tests/rabbitmq_server/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 2447f8f..37c70e5 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -308,7 +308,10 @@ def connect_amqp_by_unit(unit, ssl=False, unit_name = unit.entity_id if ssl: - ssl_options = pika.SSLOptions(libssl.SSLContext()) + # TODO: when Python3.5 support is removed, investigate + # changing protocol to PROTOCOL_TLS + context = libssl.SSLContext(protocol=libssl.PROTOCOL_TLSv1_2) + ssl_options = pika.SSLOptions(context) else: ssl_options = None From 18726c341a94a129e5c79e2a8efb662ed84eccaa Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Wed, 25 Sep 2019 10:29:25 +1000 Subject: [PATCH 127/898] Fix lint errors found in docstrings --- .../charm_tests/rabbitmq_server/tests.py | 58 ++++++++++++------- .../charm_tests/rabbitmq_server/utils.py | 45 ++++++++++---- zaza/openstack/utilities/generic.py | 10 ++-- 3 files changed, 77 insertions(+), 36 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 3e3e0f0..898c078 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""RabbitMQ Testing.""" + import json import logging import time @@ -31,8 +33,7 @@ from . import utils as rmq_utils class RmqTests(test_utils.OpenStackBaseTest): - """Zaza tests on a basic rabbitmq cluster deployment. Verify - relations, service status, users and endpoint service catalog.""" + """Zaza tests on a basic rabbitmq cluster deployment.""" @classmethod def setUpClass(cls): @@ -40,19 +41,22 @@ class RmqTests(test_utils.OpenStackBaseTest): super(RmqTests, cls).setUpClass() def _get_uuid_epoch_stamp(self): - """Returns a stamp string based on uuid4 and epoch time. Useful in - generating test messages which need to be unique-ish.""" + """Return a string based on uuid4 and epoch time. + + Useful in generating test messages which need to be unique-ish. + """ return '[{}-{}]'.format(uuid.uuid4(), time.time()) def _test_rmq_amqp_messages_all_units(self, units, ssl=False, port=None): - """Reusable test to send amqp messages to every listed rmq unit - and check every listed rmq unit for messages. + """Reusable test to send/check amqp messages to every listed rmq unit. + Reusable test to send amqp messages to every listed rmq + unit. Checks every listed rmq unit for messages. :param units: list of units :returns: None if successful. Raise on error. - """ + """ # Add test user if it does not already exist rmq_utils.add_user(units) @@ -120,8 +124,7 @@ class RmqTests(test_utils.OpenStackBaseTest): rmq_utils.delete_user(units) def test_400_rmq_cluster_running_nodes(self): - """Verify that cluster status from each rmq juju unit shows - every cluster node as a running member in that cluster.""" + """Verify cluster status shows every cluster node as running member.""" logging.debug('Checking that all units are in cluster_status ' 'running nodes...') @@ -133,8 +136,12 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') def test_406_rmq_amqp_messages_all_units_ssl_off(self): - """Send amqp messages to every rmq unit and check every rmq unit - for messages. Standard amqp tcp port, no ssl.""" + """Send (and check) amqp messages to every rmq unit. + + Sends amqp messages to every rmq unit, and check every rmq + unit for messages. Uses Standard amqp tcp port, no ssl. + + """ logging.debug('Checking amqp message publish/get on all units ' '(ssl off)...') @@ -151,8 +158,12 @@ class RmqTests(test_utils.OpenStackBaseTest): return lsb_release()['DISTRIB_CODENAME'] def test_408_rmq_amqp_messages_all_units_ssl_on(self): - """Send amqp messages with ssl enabled, to every rmq unit and - check every rmq unit for messages. Standard ssl tcp port.""" + """Send (and check) amqp messages to every rmq unit (ssl enabled). + + Sends amqp messages to every rmq unit, and check every rmq + unit for messages. Uses Standard ssl tcp port. + + """ units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 @@ -160,7 +171,7 @@ class RmqTests(test_utils.OpenStackBaseTest): CompareHostReleases(self._series(units[0])) <= 'trusty'): logging.info('SKIP') logging.info('Skipping SSL tests due to client' - ' compatibility issues') + ' compatibility issues') return logging.debug('Checking amqp message publish/get on all units ' '(ssl on)...') @@ -171,8 +182,12 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') def test_410_rmq_amqp_messages_all_units_ssl_alt_port(self): - """Send amqp messages with ssl on, to every rmq unit and check - every rmq unit for messages. Custom ssl tcp port.""" + """Send (and check) amqp messages to every rmq unit (alt ssl port). + + Send amqp messages with ssl on, to every rmq unit and check + every rmq unit for messages. Custom ssl tcp port. + + """ units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 @@ -180,7 +195,7 @@ class RmqTests(test_utils.OpenStackBaseTest): CompareHostReleases(self._series(units[0])) <= 'trusty'): logging.info('SKIP') logging.info('Skipping SSL tests due to client' - ' compatibility issues') + ' compatibility issues') return logging.debug('Checking amqp message publish/get on all units ' '(ssl on)...') @@ -281,8 +296,7 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') def test_910_pause_and_resume(self): - """The services can be paused and resumed. """ - + """The services can be paused and resumed.""" logging.debug('Checking pause and resume actions...') unit = zaza.model.get_units(self.application_name)[0] @@ -305,7 +319,7 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('OK') def test_911_cluster_status(self): - """ rabbitmqctl cluster_status action can be returned. """ + """Test rabbitmqctl cluster_status action can be returned.""" logging.debug('Checking cluster status action...') unit = zaza.model.get_units(self.application_name)[0] @@ -315,7 +329,7 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('OK') def test_912_check_queues(self): - """ rabbitmqctl check_queues action can be returned. """ + """Test rabbitmqctl check_queues action can be returned.""" logging.debug('Checking cluster status action...') unit = zaza.model.get_units(self.application_name)[0] @@ -323,7 +337,7 @@ class RmqTests(test_utils.OpenStackBaseTest): self.assertIsInstance(action, juju.action.Action) def test_913_list_unconsumed_queues(self): - """ rabbitmqctl list-unconsumed-queues action can be returned. """ + """Test rabbitmqctl list-unconsumed-queues action can be returned.""" logging.debug('Checking list-unconsumed-queues action...') unit = zaza.model.get_units(self.application_name)[0] diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 37c70e5..5a942e1 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""RabbitMQ Testing utility functions.""" + import json import logging import time @@ -24,10 +26,13 @@ import zaza.openstack.utilities.generic as generic_utils def wait_for_cluster(model_name=None, timeout=1200): - """Wait for rmq units extended status to show cluster readiness, + """Wait for Rmq cluster status to show cluster readiness. + + Wait for rmq units extended status to show cluster readiness, after an optional initial sleep period. Initial sleep is likely necessary to be effective following a config change, as status - message may not instantly update to non-ready.""" + message may not instantly update to non-ready. + """ states = { 'rabbitmq-server': { 'workload-status-messages': 'Unit is ready and clustered' @@ -40,7 +45,9 @@ def wait_for_cluster(model_name=None, timeout=1200): def add_user(units, username="testuser1", password="changeme"): - """Add a user via the first rmq juju unit, check connection as + """Add a user to a RabbitMQ cluster. + + Add a user via the first rmq juju unit, check connection as the new user against all units. :param units: list of unit pointers :param username: amqp user name, default to testuser1 @@ -77,7 +84,9 @@ def add_user(units, username="testuser1", password="changeme"): def delete_user(units, username="testuser1"): - """Delete a rabbitmq user via the first rmq juju unit. + """Delete a user from a RabbitMQ cluster. + + Delete a rabbitmq user via the first rmq juju unit. :param units: list of unit pointers :param username: amqp user name, default to testuser1 :param password: amqp user password @@ -101,7 +110,9 @@ def delete_user(units, username="testuser1"): def get_cluster_status(unit): - """Execute rabbitmq cluster status command on a unit and return + """Get RabbitMQ cluster status output. + + Execute rabbitmq cluster status command on a unit and return the full output. :param unit: unit :returns: String containing console output of cluster status command @@ -114,7 +125,9 @@ def get_cluster_status(unit): def get_cluster_running_nodes(unit): - """Parse rabbitmqctl cluster_status output string, return list of + """Get a list of RabbitMQ cluster's running nodes. + + Parse rabbitmqctl cluster_status output string, return list of running rabbitmq cluster nodes. :param unit: unit pointer :returns: List containing node names of running nodes @@ -133,7 +146,9 @@ def get_cluster_running_nodes(unit): def validate_cluster_running_nodes(units): - """Check that all rmq unit hostnames are represented in the + """Check all rmq unit hostnames are represented in cluster_status. + + Check that all rmq unit hostnames are represented in the cluster_status output of all units. :param host_names: dict of juju unit names to host names :param units: list of unit pointers (all rmq units) @@ -164,6 +179,7 @@ def validate_cluster_running_nodes(units): def validate_ssl_enabled_units(units, port=None): """Check that ssl is enabled on rmq juju units. + :param units: list of all rmq units :param port: optional ssl port override to validate :returns: None if successful, otherwise return error message @@ -177,6 +193,7 @@ def validate_ssl_enabled_units(units, port=None): def validate_ssl_disabled_units(units): """Check that ssl is enabled on listed rmq juju units. + :param units: list of all rmq units :returns: True if successful. Raise on error. """ @@ -189,7 +206,9 @@ def validate_ssl_disabled_units(units): def configure_ssl_on(units, model_name=None, port=None, max_wait=60): - """Turn ssl charm config option on, with optional non-default + """Turn RabbitMQ charm SSL config option on. + + Turn ssl charm config option on, with optional non-default ssl port specification. Confirm that it is enabled on every unit. :param units: list of units @@ -225,7 +244,9 @@ def configure_ssl_on(units, model_name=None, def configure_ssl_off(units, model_name=None, max_wait=60): - """Turn ssl charm config option off, confirm that it is disabled + """Turn RabbitMQ charm SSL config option off. + + Turn ssl charm config option off, confirm that it is disabled on every unit. :param units: list of units :param max_wait: maximum time to wait in seconds to confirm @@ -294,7 +315,9 @@ def is_ssl_enabled_on_unit(unit, port=None): def connect_amqp_by_unit(unit, ssl=False, port=None, fatal=True, username="testuser1", password="changeme"): - """Establish and return a pika amqp connection to the rabbitmq service + """Establish and return a pika amqp connection to the rabbitmq service. + + Establish and return a pika amqp connection to the rabbitmq service running on a rmq juju unit. :param unit: unit pointer :param ssl: boolean, default to False @@ -352,6 +375,7 @@ def publish_amqp_message_by_unit(unit, message, password="changeme", port=None): """Publish an amqp message to a rmq juju unit. + :param unit: unit pointer :param message: amqp message string :param queue: message queue, default to test @@ -388,6 +412,7 @@ def get_amqp_message_by_unit(unit, queue="test", password="changeme", ssl=False, port=None): """Get an amqp message from a rmq juju unit. + :param unit: unit pointer :param queue: message queue, default to test :param username: amqp user name, default to testuser1 diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index f026321..ed5aa0f 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -514,8 +514,8 @@ def dist_upgrade(unit_name): def check_commands_on_units(commands, units): - """Check that all commands in a list exit zero on all - units in a list. + """Check that all commands in a list exit zero on all units in a list. + :param commands: list of bash commands :param units: list of unit pointers :returns: None if successful; Failure message otherwise @@ -742,6 +742,7 @@ def get_ubuntu_release(ubuntu_name): def get_file_contents(unit, f): + """Get contents of a file on a remote unit.""" return model.run_on_unit(unit.entity_id, "cat {}".format(f))['Stdout'] @@ -772,8 +773,9 @@ def is_port_open(port, address): def port_knock_units(units, port=22, expect_success=True): - """Open a TCP socket to check for a listening sevice on each - listed juju unit. + """Check if specific port is open on units. + + Open a TCP socket to check for a listening sevice on each listed juju unit. :param units: list of unit pointers :param port: TCP port number, default to 22 :param timeout: Connect timeout, default to 15 seconds From b5f60126bcc87942cc328a6b44f18cd62eb261d5 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Mon, 30 Sep 2019 09:09:29 +1000 Subject: [PATCH 128/898] Remove duplicate call to function --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 898c078..18d6461 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -176,7 +176,6 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('Checking amqp message publish/get on all units ' '(ssl on)...') - units = zaza.model.get_units(self.application_name) self._test_rmq_amqp_messages_all_units(units, ssl=True, port=5671) logging.info('OK\n') From 60e84e4c276955a47ea0d38cd583241e3f5b8fd8 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Mon, 30 Sep 2019 09:12:47 +1000 Subject: [PATCH 129/898] Port Rmq charm func test removing a unit from a Rmq cluster This function was previously called test_901_remove_unit, but had to be renamed (moved to the end of the func tests); The way in which unit removal is now performed (by running the "stop" hook) puts the the removed unit in a "waiting" state -- which consequently causes wait_for_cluster() (e.g. used in 910) to fail (timeout). --- .../charm_tests/rabbitmq_server/tests.py | 41 +++++++++++++++++++ .../charm_tests/rabbitmq_server/utils.py | 23 +++++++++++ 2 files changed, 64 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 18d6461..3a144ff 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -363,3 +363,44 @@ class RmqTests(test_utils.OpenStackBaseTest): assert queue_data['messages'] == 0, 'Found unexpected message count.' logging.debug('OK') + + def test_921_remove_unit(self): + """Test if unit cleans up when removed from Rmq cluster. + + Test if a unit correctly cleans up by removing itself from the + RabbitMQ cluster on removal + + """ + logging.debug('Checking that units correctly clean up after ' + 'themselves on unit removal...') + config = {'min-cluster-size': '2'} + zaza.model.set_application_config('rabbitmq-server', config) + rmq_utils.wait_for_cluster() + + units = zaza.model.get_units(self.application_name) + removed_unit = units[-1] + left_units = units[:-1] + + zaza.model.run_on_unit(removed_unit.entity_id, 'hooks/stop') + zaza.model.block_until_unit_wl_status(removed_unit.entity_id, + "waiting") + + unit_host_names = generic_utils.get_unit_hostnames(left_units) + unit_node_names = [] + for unit in unit_host_names: + unit_node_names.append('rabbit@{}'.format(unit_host_names[unit])) + errors = [] + + for u in left_units: + e = rmq_utils.check_unit_cluster_nodes(u, unit_node_names) + if e: + # NOTE: cluster status may not have been updated yet so wait a + # little and try one more time. Need to find a better way to do + # this. + time.sleep(10) + e = rmq_utils.check_unit_cluster_nodes(u, unit_node_names) + if e: + errors.append(e) + + self.assertFalse(errors) + logging.debug('OK') diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 5a942e1..027ca50 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -439,3 +439,26 @@ def get_amqp_message_by_unit(unit, queue="test", else: msg = 'No message retrieved.' raise Exception(msg) + + +def check_unit_cluster_nodes(unit, unit_node_names): + """Check if unit exists in list of Rmq cluster node names.""" + unit_name = unit.entity_id + nodes = [] + errors = [] + str_stat = get_cluster_status(unit) + # make the interesting part of rabbitmqctl cluster_status output + # json-parseable. + if 'nodes,[{disc,' in str_stat: + pos_start = str_stat.find('nodes,[{disc,') + 13 + pos_end = str_stat.find(']}]},', pos_start) + 1 + str_nodes = str_stat[pos_start:pos_end].replace("'", '"') + nodes = json.loads(str_nodes) + for node in nodes: + if node not in unit_node_names: + errors.append('Cluster registration check failed on {}: ' + '{} should not be registered with RabbitMQ ' + 'after unit removal.\n' + ''.format(unit_name, node)) + + return errors From 1a9449766a58bb6aa74238e6cf252459deb75895 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Wed, 2 Oct 2019 08:51:06 +1000 Subject: [PATCH 130/898] Remove superfluous newlines at the end of debug messages --- .../openstack/charm_tests/rabbitmq_server/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 3a144ff..150a39b 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -133,7 +133,7 @@ class RmqTests(test_utils.OpenStackBaseTest): ret = rmq_utils.validate_cluster_running_nodes(units) self.assertIsNone(ret) - logging.info('OK\n') + logging.info('OK') def test_406_rmq_amqp_messages_all_units_ssl_off(self): """Send (and check) amqp messages to every rmq unit. @@ -147,7 +147,7 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) self._test_rmq_amqp_messages_all_units(units, ssl=False) - logging.info('OK\n') + logging.info('OK') def _series(self, unit): result = zaza.model.run_on_unit(unit.entity_id, @@ -178,7 +178,7 @@ class RmqTests(test_utils.OpenStackBaseTest): self._test_rmq_amqp_messages_all_units(units, ssl=True, port=5671) - logging.info('OK\n') + logging.info('OK') def test_410_rmq_amqp_messages_all_units_ssl_alt_port(self): """Send (and check) amqp messages to every rmq unit (alt ssl port). @@ -202,7 +202,7 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) self._test_rmq_amqp_messages_all_units(units, ssl=True, port=5999) - logging.info('OK\n') + logging.info('OK') def test_412_rmq_management_plugin(self): """Enable and check management plugin.""" @@ -229,7 +229,7 @@ class RmqTests(test_utils.OpenStackBaseTest): tries += 1 self.assertIsNone(ret) - logging.debug('Connect to all units (OK)\n') + logging.debug('Connect to all units (OK)') # Disable management plugin logging.debug('Disabling management_plugin charm config option...') @@ -253,7 +253,7 @@ class RmqTests(test_utils.OpenStackBaseTest): tries += 1 self.assertIsNone(ret) - logging.info('Confirm mgmt port closed on all units (OK)\n') + logging.info('Confirm mgmt port closed on all units (OK)') def test_414_rmq_nrpe_monitors(self): """Check rabbimq-server nrpe monitor basic functionality.""" @@ -292,7 +292,7 @@ class RmqTests(test_utils.OpenStackBaseTest): ret = generic_utils.check_commands_on_units(cmds, [u]) self.assertIsNone(ret) - logging.info('OK\n') + logging.info('OK') def test_910_pause_and_resume(self): """The services can be paused and resumed.""" From a0769e5dcb33b6ca3a713cccc61ecc15c2c965ed Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Wed, 2 Oct 2019 09:14:10 +1000 Subject: [PATCH 131/898] Remove TODO (behaviour confirmed not to be a bug) --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 150a39b..f93bd2d 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -303,9 +303,6 @@ class RmqTests(test_utils.OpenStackBaseTest): zaza.model.run_action(unit.entity_id, "pause") zaza.model.block_until_unit_wl_status(unit.entity_id, "maintenance") - # TODO: investigate possible bug (the following line is - # required, otherwise it looks like workload_status is - # reporting cached information, no matter how long you sleep) unit = zaza.model.get_unit_from_name(unit.entity_id) assert unit.workload_status == "maintenance" From 4eb068ab236449d9f76a01bc6a130cef7b414246 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Wed, 2 Oct 2019 09:12:43 +1000 Subject: [PATCH 132/898] Provide a message if assert fails --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index f93bd2d..a5a764a 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -131,7 +131,7 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) ret = rmq_utils.validate_cluster_running_nodes(units) - self.assertIsNone(ret) + self.assertIsNone(ret, msg=ret) logging.info('OK') @@ -228,7 +228,7 @@ class RmqTests(test_utils.OpenStackBaseTest): ret = generic_utils.port_knock_units(units, mgmt_port) tries += 1 - self.assertIsNone(ret) + self.assertIsNone(ret, msg=ret) logging.debug('Connect to all units (OK)') # Disable management plugin @@ -252,7 +252,7 @@ class RmqTests(test_utils.OpenStackBaseTest): expect_success=False) tries += 1 - self.assertIsNone(ret) + self.assertIsNone(ret, msg=ret) logging.info('Confirm mgmt port closed on all units (OK)') def test_414_rmq_nrpe_monitors(self): @@ -275,7 +275,7 @@ class RmqTests(test_utils.OpenStackBaseTest): cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' 'check_rabbitmq_queue.cfg'] ret = generic_utils.check_commands_on_units(cmds, units) - self.assertIsNone(ret) + self.assertIsNone(ret, msg=ret) # check dat file existence logging.debug('Checking nrpe dat file existence on units...') @@ -290,7 +290,7 @@ class RmqTests(test_utils.OpenStackBaseTest): ] ret = generic_utils.check_commands_on_units(cmds, [u]) - self.assertIsNone(ret) + self.assertIsNone(ret, msg=ret) logging.info('OK') From 5e170cd1ba87814473ced9681db0fe6f14ff1ab1 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 09:50:45 +1000 Subject: [PATCH 133/898] Make get_series/get_client_series available as public util functions --- .../charm_tests/rabbitmq_server/tests.py | 24 +++++++------------ zaza/openstack/utilities/generic.py | 13 ++++++++++ 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index a5a764a..2ae080a 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -24,9 +24,11 @@ import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils -from charmhelpers.core.host import ( - lsb_release, - CompareHostReleases, +from charmhelpers.core.host import CompareHostReleases + +from zaza.openstack.utilities.generic import ( + get_series, + get_client_series, ) from . import utils as rmq_utils @@ -149,14 +151,6 @@ class RmqTests(test_utils.OpenStackBaseTest): self._test_rmq_amqp_messages_all_units(units, ssl=False) logging.info('OK') - def _series(self, unit): - result = zaza.model.run_on_unit(unit.entity_id, - "lsb_release -cs") - return result['Stdout'].strip() - - def _client_series(self): - return lsb_release()['DISTRIB_CODENAME'] - def test_408_rmq_amqp_messages_all_units_ssl_on(self): """Send (and check) amqp messages to every rmq unit (ssl enabled). @@ -167,8 +161,8 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 - if (CompareHostReleases(self._client_series()) >= 'xenial' and - CompareHostReleases(self._series(units[0])) <= 'trusty'): + if (CompareHostReleases(get_client_series()) >= 'xenial' and + CompareHostReleases(get_series(units[0])) <= 'trusty'): logging.info('SKIP') logging.info('Skipping SSL tests due to client' ' compatibility issues') @@ -190,8 +184,8 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 - if (CompareHostReleases(self._client_series()) >= 'xenial' and - CompareHostReleases(self._series(units[0])) <= 'trusty'): + if (CompareHostReleases(get_client_series()) >= 'xenial' and + CompareHostReleases(get_series(units[0])) <= 'trusty'): logging.info('SKIP') logging.info('Skipping SSL tests due to client' ' compatibility issues') diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index ed5aa0f..6e3809d 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -22,6 +22,7 @@ import telnetlib import yaml from zaza import model +from charmhelpers.core.host import lsb_release from zaza.openstack.utilities import juju as juju_utils from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE @@ -789,3 +790,15 @@ def port_knock_units(units, port=22, expect_success=True): return 'Socket connect failed.' elif connected and not expect_success: return 'Socket connected unexpectedly.' + + +def get_series(unit): + """Ubuntu release name running on unit.""" + result = model.run_on_unit(unit.entity_id, + "lsb_release -cs") + return result['Stdout'].strip() + + +def get_client_series(): + """Ubuntu release name of machine running this function.""" + return lsb_release()['DISTRIB_CODENAME'] From 814ff7ddee24bc11373ebcade2a9c8d6197877b0 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 09:59:01 +1000 Subject: [PATCH 134/898] Don't sleep before attempting get AMQP msg. Retry w/ tenacity --- .../charm_tests/rabbitmq_server/tests.py | 20 +++++++++++++------ .../charm_tests/rabbitmq_server/utils.py | 8 +++++++- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 2ae080a..3aae1f0 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -20,6 +20,7 @@ import time import uuid import juju +import tenacity import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils @@ -32,6 +33,7 @@ from zaza.openstack.utilities.generic import ( ) from . import utils as rmq_utils +from .utils import RmqNoMessageException class RmqTests(test_utils.OpenStackBaseTest): @@ -49,6 +51,15 @@ class RmqTests(test_utils.OpenStackBaseTest): """ return '[{}-{}]'.format(uuid.uuid4(), time.time()) + @tenacity.retry( + retry=tenacity.retry_if_exception_type(RmqNoMessageException), + wait=tenacity.wait_fixed(10), + stop=tenacity.stop_after_attempt(2)) + def _retry_get_amqp_message(self, check_unit, ssl=None, port=None): + return rmq_utils.get_amqp_message_by_unit(check_unit, + ssl=ssl, + port=port) + def _test_rmq_amqp_messages_all_units(self, units, ssl=False, port=None): """Reusable test to send/check amqp messages to every listed rmq unit. @@ -97,18 +108,15 @@ class RmqTests(test_utils.OpenStackBaseTest): amqp_msg, ssl=ssl, port=port) - # Wait a bit before checking for message - time.sleep(10) - # Get amqp message logging.debug('Get message from: {} ' '({} {})'.format(check_unit_host, check_unit_name, check_unit_host_name)) - amqp_msg_rcvd = rmq_utils.get_amqp_message_by_unit(check_unit, - ssl=ssl, - port=port) + amqp_msg_rcvd = self._retry_get_amqp_message(check_unit, + ssl=ssl, + port=port) # Validate amqp message content if amqp_msg == amqp_msg_rcvd: diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 027ca50..cfb2f4b 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -25,6 +25,12 @@ import ssl as libssl import zaza.openstack.utilities.generic as generic_utils +class RmqNoMessageException(Exception): + """Message retrieval from Rmq resulted in no message.""" + + pass + + def wait_for_cluster(model_name=None, timeout=1200): """Wait for Rmq cluster status to show cluster readiness. @@ -438,7 +444,7 @@ def get_amqp_message_by_unit(unit, queue="test", return body else: msg = 'No message retrieved.' - raise Exception(msg) + raise RmqNoMessageException(msg) def check_unit_cluster_nodes(unit, unit_node_names): From e8068a83565f3e018e0da348a9da4a8a57587eab Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 10:09:22 +1000 Subject: [PATCH 135/898] Port validating Rmq SSL on from time.sleep to Tenacity --- .../charm_tests/rabbitmq_server/utils.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index cfb2f4b..edae102 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -19,6 +19,7 @@ import logging import time import pika +import tenacity import zaza.model import ssl as libssl @@ -31,6 +32,11 @@ class RmqNoMessageException(Exception): pass +def _log_tenacity_retry(retry_state): + logging.info('Attempt {}: {}'.format(retry_state.attempt_number, + retry_state.outcome.result())) + + def wait_for_cluster(model_name=None, timeout=1200): """Wait for Rmq cluster status to show cluster readiness. @@ -210,8 +216,16 @@ def validate_ssl_disabled_units(units): return None -def configure_ssl_on(units, model_name=None, - port=None, max_wait=60): +@tenacity.retry( + retry=tenacity.retry_if_result(lambda errors: bool(errors)), + wait=tenacity.wait_fixed(4), + stop=tenacity.stop_after_attempt(15), + after=_log_tenacity_retry) +def _retry_validate_ssl_enabled_units(units, port=None): + return validate_ssl_enabled_units(units, port=port) + + +def configure_ssl_on(units, model_name=None, port=None): """Turn RabbitMQ charm SSL config option on. Turn ssl charm config option on, with optional non-default @@ -219,7 +233,6 @@ def configure_ssl_on(units, model_name=None, unit. :param units: list of units :param port: amqp port, use defaults if None - :param max_wait: maximum time to wait in seconds to confirm :returns: None if successful. Raise on error. """ logging.debug('Setting ssl charm config option: on') @@ -236,15 +249,7 @@ def configure_ssl_on(units, model_name=None, # Wait for unit status wait_for_cluster(model_name) - # Confirm - tries = 0 - ret = validate_ssl_enabled_units(units, port=port) - while ret and tries < (max_wait / 4): - time.sleep(4) - logging.debug('Attempt {}: {}'.format(tries, ret)) - ret = validate_ssl_enabled_units(units, port=port) - tries += 1 - + ret = _retry_validate_ssl_enabled_units(units, port=port) if ret: raise Exception(ret) From d780f76797233b9298d15776388f194753681076 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 10:12:36 +1000 Subject: [PATCH 136/898] Port validating Rmq SSL off from time.sleep to Tenacity --- .../charm_tests/rabbitmq_server/utils.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index edae102..dba2d92 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -16,7 +16,6 @@ import json import logging -import time import pika import tenacity @@ -254,6 +253,15 @@ def configure_ssl_on(units, model_name=None, port=None): raise Exception(ret) +@tenacity.retry( + retry=tenacity.retry_if_result(lambda errors: bool(errors)), + wait=tenacity.wait_fixed(4), + stop=tenacity.stop_after_attempt(15), + after=_log_tenacity_retry) +def _retry_validate_ssl_disabled_units(units): + return validate_ssl_disabled_units(units) + + def configure_ssl_off(units, model_name=None, max_wait=60): """Turn RabbitMQ charm SSL config option off. @@ -274,14 +282,7 @@ def configure_ssl_off(units, model_name=None, max_wait=60): # Wait for unit status wait_for_cluster(model_name) - # Confirm - tries = 0 - ret = validate_ssl_disabled_units(units) - while ret and tries < (max_wait / 4): - time.sleep(4) - logging.debug('Attempt {}: {}'.format(tries, ret)) - ret = validate_ssl_disabled_units(units) - tries += 1 + ret = _retry_validate_ssl_disabled_units(units) if ret: raise Exception(ret) From 215b3c87e3b10e56d9ef11f447b2e006954fff67 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 10:14:36 +1000 Subject: [PATCH 137/898] Don't time.sleep waiting for Nagios. Port to tenacity. --- .../charm_tests/rabbitmq_server/tests.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 3aae1f0..3d30d8a 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -257,6 +257,14 @@ class RmqTests(test_utils.OpenStackBaseTest): self.assertIsNone(ret, msg=ret) logging.info('Confirm mgmt port closed on all units (OK)') + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + # sleep for 2mins to allow 1min cron job to run... + wait=tenacity.wait_fixed(120), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_commands_on_units(self, cmds, units): + return generic_utils.check_commands_on_units(cmds, units) + def test_414_rmq_nrpe_monitors(self): """Check rabbimq-server nrpe monitor basic functionality.""" units = zaza.model.get_units(self.application_name) @@ -266,17 +274,14 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('Checking nrpe check_rabbitmq on units...') cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' 'check_rabbitmq.cfg'] - ret = generic_utils.check_commands_on_units(cmds, units) - self.assertIsNone(ret) - - logging.debug('Sleeping 2ms for 1m cron job to run...') - time.sleep(120) + ret = self._retry_check_commands_on_units(cmds, units) + self.assertIsNone(ret, msg=ret) # check_rabbitmq_queue monitor logging.debug('Checking nrpe check_rabbitmq_queue on units...') cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' 'check_rabbitmq_queue.cfg'] - ret = generic_utils.check_commands_on_units(cmds, units) + ret = self._retry_check_commands_on_units(cmds, units) self.assertIsNone(ret, msg=ret) # check dat file existence From 060f7392d91fe44278f38cadb220207fcb8b61f7 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 10:32:51 +1000 Subject: [PATCH 138/898] Don't sleep while waiting for cluster status to update. Use tenacity --- .../charm_tests/rabbitmq_server/tests.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 3d30d8a..0c28074 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -368,6 +368,13 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('OK') + @tenacity.retry( + retry=tenacity.retry_if_result(lambda errors: bool(errors)), + wait=tenacity.wait_fixed(10), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_unit_cluster_nodes(self, u, unit_node_names): + return rmq_utils.check_unit_cluster_nodes(u, unit_node_names) + def test_921_remove_unit(self): """Test if unit cleans up when removed from Rmq cluster. @@ -396,15 +403,10 @@ class RmqTests(test_utils.OpenStackBaseTest): errors = [] for u in left_units: - e = rmq_utils.check_unit_cluster_nodes(u, unit_node_names) + e = self._retry_check_unit_cluster_nodes(u, + unit_node_names) if e: - # NOTE: cluster status may not have been updated yet so wait a - # little and try one more time. Need to find a better way to do - # this. - time.sleep(10) - e = rmq_utils.check_unit_cluster_nodes(u, unit_node_names) - if e: - errors.append(e) + errors.append(e) - self.assertFalse(errors) + self.assertFalse(errors, msg=errors) logging.debug('OK') From 1ea0be044058ce0809dcdf57c99a971cbcf4b6fa Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 10:37:55 +1000 Subject: [PATCH 139/898] Don't time.sleep when checking Rmq ports after toggling mgmt plugin. --- .../charm_tests/rabbitmq_server/tests.py | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 0c28074..c1ef9de 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -206,6 +206,15 @@ class RmqTests(test_utils.OpenStackBaseTest): ssl=True, port=5999) logging.info('OK') + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + wait=tenacity.wait_fixed(30), + stop=tenacity.stop_after_attempt(20), + after=rmq_utils._log_tenacity_retry) + def _retry_port_knock_units(self, units, port, expect_success=True): + return generic_utils.port_knock_units(units, port, + expect_success=expect_success) + def test_412_rmq_management_plugin(self): """Enable and check management plugin.""" logging.debug('Checking tcp socket connect to management plugin ' @@ -221,14 +230,7 @@ class RmqTests(test_utils.OpenStackBaseTest): rmq_utils.wait_for_cluster() # Check tcp connect to management plugin port - max_wait = 600 - tries = 0 - ret = generic_utils.port_knock_units(units, mgmt_port) - while ret and tries < (max_wait / 30): - time.sleep(30) - logging.debug('Attempt {}: {}'.format(tries, ret)) - ret = generic_utils.port_knock_units(units, mgmt_port) - tries += 1 + ret = self._retry_port_knock_units(units, mgmt_port) self.assertIsNone(ret, msg=ret) logging.debug('Connect to all units (OK)') @@ -242,17 +244,9 @@ class RmqTests(test_utils.OpenStackBaseTest): # Negative check - tcp connect to management plugin port logging.info('Expect tcp connect fail since charm config ' 'option is disabled.') - tries = 0 - ret = generic_utils.port_knock_units(units, - mgmt_port, - expect_success=False) - - while ret and tries < (max_wait / 30): - time.sleep(30) - logging.debug('Attempt {}: {}'.format(tries, ret)) - ret = generic_utils.port_knock_units(units, mgmt_port, - expect_success=False) - tries += 1 + ret = self._retry_port_knock_units(units, + mgmt_port, + expect_success=False) self.assertIsNone(ret, msg=ret) logging.info('Confirm mgmt port closed on all units (OK)') From d2e12abb9153cea04d7563fd6b81a6fb140a842d Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 10 Oct 2019 11:06:26 +1100 Subject: [PATCH 140/898] Remove condition: check client series will always be true. All CI test runners are >= xenial. The condition will always be true. --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 12 +++--------- zaza/openstack/utilities/generic.py | 6 ------ 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index c1ef9de..b114314 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -26,11 +26,7 @@ import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils from charmhelpers.core.host import CompareHostReleases - -from zaza.openstack.utilities.generic import ( - get_series, - get_client_series, -) +from zaza.openstack.utilities.generic import get_series from . import utils as rmq_utils from .utils import RmqNoMessageException @@ -169,8 +165,7 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 - if (CompareHostReleases(get_client_series()) >= 'xenial' and - CompareHostReleases(get_series(units[0])) <= 'trusty'): + if CompareHostReleases(get_series(units[0])) <= 'trusty': logging.info('SKIP') logging.info('Skipping SSL tests due to client' ' compatibility issues') @@ -192,8 +187,7 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 - if (CompareHostReleases(get_client_series()) >= 'xenial' and - CompareHostReleases(get_series(units[0])) <= 'trusty'): + if CompareHostReleases(get_series(units[0])) <= 'trusty': logging.info('SKIP') logging.info('Skipping SSL tests due to client' ' compatibility issues') diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 6e3809d..174b47a 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -22,7 +22,6 @@ import telnetlib import yaml from zaza import model -from charmhelpers.core.host import lsb_release from zaza.openstack.utilities import juju as juju_utils from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE @@ -797,8 +796,3 @@ def get_series(unit): result = model.run_on_unit(unit.entity_id, "lsb_release -cs") return result['Stdout'].strip() - - -def get_client_series(): - """Ubuntu release name of machine running this function.""" - return lsb_release()['DISTRIB_CODENAME'] From e0c10b5457412612db792c2b0a1279c64a2ebe96 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 14 Oct 2019 07:50:55 +0000 Subject: [PATCH 141/898] Search for vault credetials file. During testing the vault credentials are stored on the lead unit. Unfortunatly which unit is the designated leader can change during the life of the test (particularly if the test involves rebooting vault units). This change adds a function to search the units for the credentials file rather than assuming its on the lead unit. --- zaza/openstack/charm_tests/vault/utils.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index cdc68b1..9491bfd 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -159,6 +159,22 @@ def ensure_secret_backend(client): pass +def find_unit_with_creds(): + """Find the unit thats has stored the credentials. + + :returns: unit name + :rtype: str + """ + unit = None + for vault_unit in zaza.model.get_units('vault'): + cmd = 'ls -l ~ubuntu/{}'.format(AUTH_FILE) + resp = zaza.model.run_on_unit(vault_unit.name, cmd) + if resp.get('Code') == '0': + unit = vault_unit.name + break + return unit + + def get_credentails(): """Retrieve vault token and keys from unit. @@ -168,7 +184,7 @@ def get_credentails(): :returns: Tokens and keys for accessing test environment :rtype: dict """ - unit = zaza.model.get_first_unit_name('vault') + unit = find_unit_with_creds() with tempfile.TemporaryDirectory() as tmpdirname: tmp_file = '{}/{}'.format(tmpdirname, AUTH_FILE) zaza.model.scp_from_unit( From 79a149bfc54c271231634c9bb8d941161d17a93b Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 14 Oct 2019 08:53:42 +0000 Subject: [PATCH 142/898] Facade for vault & basic_setup_and_unseal Add a facade for interacting with vault and a function which unseals vault units (useful when vault units have been rebooted). --- zaza/openstack/charm_tests/vault/setup.py | 34 ++++++-------- zaza/openstack/charm_tests/vault/utils.py | 56 +++++++++++++++++++++++ 2 files changed, 71 insertions(+), 19 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 37bd332..ca97bf0 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -33,26 +33,22 @@ def basic_setup(cacert=None, unseal_and_authorize=False): :param unseal_and_authorize: Whether to unseal and authorize vault. :type unseal_and_authorize: bool """ - clients = vault_utils.get_clients(cacert=cacert) - vip_client = vault_utils.get_vip_client(cacert=cacert) - if vip_client: - unseal_client = vip_client - else: - unseal_client = clients[0] - initialized = vault_utils.is_initialized(unseal_client) - # The credentials are written to a file to allow the tests to be re-run - # this is mainly useful for manually working on the tests. - if initialized: - vault_creds = vault_utils.get_credentails() - else: - vault_creds = vault_utils.init_vault(unseal_client) - vault_utils.store_credentails(vault_creds) - - # For use by charms or bundles other than vault + vault_svc = vault_utils.VaultFacade(cacert=cacert) if unseal_and_authorize: - vault_utils.unseal_all(clients, vault_creds['keys'][0]) - vault_utils.auth_all(clients, vault_creds['root_token']) - vault_utils.run_charm_authorize(vault_creds['root_token']) + vault_svc.unseal() + vault_svc.authorize() + + +def basic_setup_and_unseal(cacert=None): + """Initialize (if needed) and unseal vault. + + :param cacert: Path to CA cert used for vaults api cert. + :type cacert: str + """ + vault_svc = vault_utils.VaultFacade(cacert=cacert) + vault_svc.unseal() + for unit in zaza.model.get_units('vault'): + zaza.model.run_on_unit(unit.name, './hooks/update-status') def auto_initialize(cacert=None, validation_application='keystone'): diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 9491bfd..b6f4cf5 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -33,6 +33,62 @@ CharmVaultClient = collections.namedtuple( 'CharmVaultClient', ['addr', 'hvac_client', 'vip_client']) +class VaultFacade: + """Provide a facade for interacting with vault. + + For example to setup new vault deployment:: + + vault_svc = VaultFacade() + vault_svc.unseal() + vault_svc.authorize() + """ + + def __init__(self, cacert=None, initialize=True): + """Create a facade for interacting with vault. + + :param cacert: Path to CA cert used for vaults api cert. + :type cacert: str + :param initialize: Whether to initialize vault. + :type initialize: bool + """ + self.clients = get_clients(cacert=cacert) + self.vip_client = get_vip_client(cacert=cacert) + if self.vip_client: + self.unseal_client = self.vip_client + else: + self.unseal_client = self.clients[0] + self.initialized = is_initialized(self.unseal_client) + if initialize: + self.initialize() + + @property + def is_initialized(self): + """Check if vault is initialized.""" + return self.initialized + + def initialize(self): + """Initialise vault and store resulting credentials.""" + if self.is_initialized: + self.vault_creds = get_credentails() + else: + self.vault_creds = init_vault(self.unseal_client) + store_credentails(self.vault_creds) + self.initialized = is_initialized(self.unseal_client) + + def unseal(self): + """Unseal all the vaults clients.""" + unseal_all(self.clients, self.vault_creds['keys'][0]) + + def authorize(self): + """Authorize charm to perfom certain actions. + + Run vault charm action to authorize the charm to perform a limited + set of calls against the vault API. + """ + auth_all(self.clients, self.vault_creds['root_token']) + run_charm_authorize(self.vault_creds['root_token']) + + def get_unit_api_url(ip): """Return URL for api access. From 6c15605222b6b361581bdfd9f7c4cc410c039889 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 14 Oct 2019 10:39:11 +0000 Subject: [PATCH 143/898] Series upgrade tests run funcs after upgrade. Execute a list of functions after series upgrade if one is supplied. This change is driven by the upgrade testing of vault. Vault needs to be unsealed after it is rebooted. --- .../utilities/test_zaza_utilities_generic.py | 9 +++-- .../charm_tests/series_upgrade/tests.py | 11 +++++- zaza/openstack/utilities/generic.py | 39 +++++++++++++++---- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 4869ae4..9e0f39e 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -287,7 +287,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): mock.call("{}/{}".format(_application, machine_num), machine_num, origin=_origin, from_series=_from_series, to_series=_to_series, - workaround_script=_workaround_script, files=_files), + workaround_script=_workaround_script, files=_files, + post_upgrade_functions=None), ) # Pause primary peers and subordinates @@ -325,7 +326,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): mock.call("{}/{}".format(_application, machine_num), machine_num, origin=_origin, from_series=_from_series, to_series=_to_series, - workaround_script=_workaround_script, files=_files), + workaround_script=_workaround_script, files=_files, + post_upgrade_functions=None), ) # Pause subordinates @@ -356,7 +358,8 @@ class TestGenericUtils(ut_utils.BaseTestCase): mock.call("{}/{}".format(_application, machine_num), machine_num, origin=_origin, from_series=_from_series, to_series=_to_series, - workaround_script=_workaround_script, files=_files), + workaround_script=_workaround_script, files=_files, + post_upgrade_functions=None), ) # No Pausiing diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 6d75fb9..3d378cb 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -52,6 +52,7 @@ class SeriesUpgradeTest(unittest.TestCase): origin = "openstack-origin" pause_non_leader_subordinate = True pause_non_leader_primary = True + post_upgrade_functions = [] # Skip subordinates if applications[application]["subordinate-to"]: continue @@ -78,6 +79,10 @@ class SeriesUpgradeTest(unittest.TestCase): origin = None pause_non_leader_primary = False pause_non_leader_subordinate = False + if "vault" in applications[application]["charm"]: + post_upgrade_functions = [ + ('zaza.openstack.charm_tests.vault.setup.' + 'basic_setup_and_unseal')] if ("mongodb" in applications[application]["charm"] or "vault" in applications[application]["charm"]): # Mongodb and vault need to run series upgrade @@ -86,7 +91,8 @@ class SeriesUpgradeTest(unittest.TestCase): application, from_series=self.from_series, to_series=self.to_series, - completed_machines=completed_machines) + completed_machines=completed_machines, + post_upgrade_functions=post_upgrade_functions) continue # The rest are likley APIs use defaults @@ -100,7 +106,8 @@ class SeriesUpgradeTest(unittest.TestCase): origin=origin, completed_machines=completed_machines, workaround_script=self.workaround_script, - files=self.files) + files=self.files, + post_upgrade_functions=post_upgrade_functions) class OpenStackSeriesUpgrade(SeriesUpgradeTest): diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 7eb175c..f1bb621 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -25,7 +25,7 @@ from zaza import model from zaza.openstack.utilities import juju as juju_utils from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE - +from zaza.charm_lifecycle import utils as cl_utils SUBORDINATE_PAUSE_RESUME_BLACKLIST = [ "cinder-ceph", @@ -177,9 +177,22 @@ def get_yaml_config(config_file): return yaml.safe_load(open(config_file, 'r').read()) +def run_post_upgrade_functions(post_upgrade_functions): + """Execute list supplied functions. + + :param post_upgrade_functions: List of functions + :type post_upgrade_functions: [function, function, ...] + """ + if post_upgrade_functions: + for func in post_upgrade_functions: + logging.info("Running {}".format(func)) + cl_utils.get_class(func)() + + def series_upgrade_non_leaders_first(application, from_series="trusty", to_series="xenial", - completed_machines=[]): + completed_machines=[], + post_upgrade_functions=None): """Series upgrade non leaders first. Wrap all the functionality to handle series upgrade for charms @@ -214,7 +227,9 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", .format(unit)) series_upgrade(unit, machine, from_series=from_series, to_series=to_series, - origin=None) + origin=None, + post_upgrade_functions=post_upgrade_functions) + run_post_upgrade_functions(post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded. " @@ -227,7 +242,8 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", if machine not in completed_machines: series_upgrade(leader, machine, from_series=from_series, to_series=to_series, - origin=None) + origin=None, + post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded." @@ -240,7 +256,8 @@ def series_upgrade_application(application, pause_non_leader_primary=True, from_series="trusty", to_series="xenial", origin='openstack-origin', completed_machines=[], - files=None, workaround_script=None): + files=None, workaround_script=None, + post_upgrade_functions=None): """Series upgrade application. Wrap all the functionality to handle series upgrade for a given @@ -310,7 +327,8 @@ def series_upgrade_application(application, pause_non_leader_primary=True, series_upgrade(leader, machine, from_series=from_series, to_series=to_series, origin=origin, workaround_script=workaround_script, - files=files) + files=files, + post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded." @@ -329,7 +347,8 @@ def series_upgrade_application(application, pause_non_leader_primary=True, series_upgrade(unit, machine, from_series=from_series, to_series=to_series, origin=origin, workaround_script=workaround_script, - files=files) + files=files, + post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded. " @@ -343,7 +362,8 @@ def series_upgrade_application(application, pause_non_leader_primary=True, def series_upgrade(unit_name, machine_num, from_series="trusty", to_series="xenial", origin='openstack-origin', - files=None, workaround_script=None): + files=None, workaround_script=None, + post_upgrade_functions=None): """Perform series upgrade on a unit. :param unit_name: Unit Name @@ -394,6 +414,9 @@ def series_upgrade(unit_name, machine_num, logging.info("Complete series upgrade on {}".format(machine_num)) model.complete_series_upgrade(machine_num) model.block_until_all_units_idle() + logging.info("Running run_post_upgrade_functions {}".format( + post_upgrade_functions)) + run_post_upgrade_functions(post_upgrade_functions) logging.info("Waiting for workload status 'active' on {}" .format(unit_name)) model.block_until_unit_wl_status(unit_name, "active") From 009f37ab0b4e357d16eaa178b71b60deb217f5a5 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Tue, 15 Oct 2019 12:48:21 +1100 Subject: [PATCH 144/898] Add unit test for generic util function: get_unit_hostnames --- .../utilities/test_zaza_utilities_generic.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 4869ae4..a4eed09 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -572,3 +572,33 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.telnet.side_effect = generic_utils.socket.error self.assertFalse(generic_utils.is_port_open(_port, _addr)) + + def test_get_unit_hostnames(self): + self.patch( + "zaza.openstack.utilities.generic.model.run_on_unit", + new_callable=mock.MagicMock(), + name="_run" + ) + + _unit1 = mock.MagicMock() + _unit1.entity_id = "testunit/1" + _unit2 = mock.MagicMock() + _unit2.entity_id = "testunit/2" + + _hostname1 = "host1.domain" + _hostname2 = "host2.domain" + + expected = { + _unit1.entity_id: _hostname1, + _unit2.entity_id: _hostname2, + } + + _units = [_unit1, _unit2] + + self._run.side_effect = [{"Stdout": _hostname1}, + {"Stdout": _hostname2}] + + actual = generic_utils.get_unit_hostnames(_units) + + self.assertEqual(actual, expected) + From 6402d5a0212e2c4a6595b6a3b0e1071e459c5762 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Tue, 15 Oct 2019 12:50:56 +1100 Subject: [PATCH 145/898] Add unit test for generic util function: port_knock_units --- .../utilities/test_zaza_utilities_generic.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index a4eed09..c992e03 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -602,3 +602,29 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.assertEqual(actual, expected) + def test_port_knock_units(self): + self.patch( + "zaza.openstack.utilities.generic.is_port_open", + new_callable=mock.MagicMock(), + name="_is_port_open" + ) + + _units = [ + mock.MagicMock(), + mock.MagicMock(), + ] + + self._is_port_open.side_effect = [True, True] + self.assertIsNone(generic_utils.port_knock_units(_units)) + self.assertEqual(self._is_port_open.call_count, len(_units)) + + self._is_port_open.side_effect = [True, False] + self.assertIsNotNone(generic_utils.port_knock_units(_units)) + + # check when func is expecting failure, i.e. should succeed + self._is_port_open.reset_mock() + self._is_port_open.side_effect = [False, False] + self.assertIsNone(generic_utils.port_knock_units(_units, + expect_success=False)) + self.assertEqual(self._is_port_open.call_count, len(_units)) + From 596c347be4a37e0b0e917e887f5f9f5c13e2b1a9 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Tue, 15 Oct 2019 12:51:12 +1100 Subject: [PATCH 146/898] Add unit test for generic util function: check_commands_on_units --- .../utilities/test_zaza_utilities_generic.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index c992e03..4c09d2b 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -628,3 +628,31 @@ class TestGenericUtils(ut_utils.BaseTestCase): expect_success=False)) self.assertEqual(self._is_port_open.call_count, len(_units)) + def test_check_commands_on_units(self): + self.patch( + "zaza.openstack.utilities.generic.model.run_on_unit", + new_callable=mock.MagicMock(), + name="_run" + ) + + num_units = 2 + _units = [mock.MagicMock() for i in range(num_units)] + + num_cmds = 3 + cmds = ["/usr/bin/fakecmd"] * num_cmds + + # Test success, all calls return 0 + # zero is a string to replicate run_on_unit return data type + _cmd_results = [{"Code": "0"}] * len(_units) * len(cmds) + self._run.side_effect = _cmd_results + + result = generic_utils.check_commands_on_units(cmds, _units) + self.assertIsNone(result) + self.assertEqual(self._run.call_count, len(_units) * len(cmds)) + + # Test failure, some call returns 1 + _cmd_results[2] = {"Code": "1"} + self._run.side_effect = _cmd_results + + result = generic_utils.check_commands_on_units(cmds, _units) + self.assertIsNotNone(result) From 743c4765a4e83b316c05cbbb42b8f59a3d0e673f Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 16 Oct 2019 14:57:53 -0700 Subject: [PATCH 147/898] Allow more time for BGP route test Add more retries to the bgp route test and print out the routing table. --- zaza/openstack/charm_tests/dragent/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/dragent/test.py b/zaza/openstack/charm_tests/dragent/test.py index de4e230..b8c9fd5 100644 --- a/zaza/openstack/charm_tests/dragent/test.py +++ b/zaza/openstack/charm_tests/dragent/test.py @@ -61,14 +61,14 @@ def test_bgp_routes(peer_application_name="quagga", keystone_session=None): # This test may run immediately after configuration. It may take time for # routes to propogate via BGP. Do a binary backoff. @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), - reraise=True, stop=tenacity.stop_after_attempt(8)) + reraise=True, stop=tenacity.stop_after_attempt(10)) def _assert_cidr_in_peer_routing_table(peer_unit, cidr): logging.debug("Checking for {} on BGP peer {}" .format(cidr, peer_unit)) # Run show ip route bgp on BGP peer routes = juju_utils.remote_run( peer_unit, remote_cmd='vtysh -c "show ip route bgp"') - logging.debug(routes) + logging.info(routes) assert cidr in routes, ( "CIDR, {}, not found in BGP peer's routing table" .format(cidr)) From 29a4c7cf4437da606de3398218224d847e511837 Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 16 Oct 2019 15:46:33 -0700 Subject: [PATCH 148/898] Display routes in assertion error --- zaza/openstack/charm_tests/dragent/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/dragent/test.py b/zaza/openstack/charm_tests/dragent/test.py index b8c9fd5..90aacb9 100644 --- a/zaza/openstack/charm_tests/dragent/test.py +++ b/zaza/openstack/charm_tests/dragent/test.py @@ -70,7 +70,8 @@ def test_bgp_routes(peer_application_name="quagga", keystone_session=None): peer_unit, remote_cmd='vtysh -c "show ip route bgp"') logging.info(routes) assert cidr in routes, ( - "CIDR, {}, not found in BGP peer's routing table" .format(cidr)) + "CIDR, {}, not found in BGP peer's routing table: {}" + .format(cidr, routes)) _assert_cidr_in_peer_routing_table(peer_unit, private_cidr) logging.info("Private subnet CIDR, {}, found in routing table" From a82979d1450a5ea0db7a66afab2e8801b450e19f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 30 Sep 2019 15:24:21 +0200 Subject: [PATCH 149/898] vault: Await execution prior to validation step Make it possible for consumers of the ``auto_initialize_no_validation`` function to execute subsequent setup and test code that require vault and the consumers of the ``certificates`` relation to be ready. With the current order of execution, it is not possible to use the configure function in models where ``keystone`` application is not present. --- zaza/openstack/charm_tests/vault/setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index ca97bf0..efb3a8d 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -83,6 +83,11 @@ def auto_initialize(cacert=None, validation_application='keystone'): root_ca=cacertificate, allowed_domains='openstack.local') + zaza.model.wait_for_agent_status() + test_config = lifecycle_utils.get_charm_config() + zaza.model.wait_for_application_states( + states=test_config.get('target_deploy_status', {})) + if validation_application: validate_ca(cacertificate, application=validation_application) # Once validation has completed restart nova-compute to work around @@ -120,9 +125,6 @@ def validate_ca(cacertificate, application="keystone", port=5000): application, zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT, cacertificate.decode().strip()) - test_config = lifecycle_utils.get_charm_config() - zaza.model.wait_for_application_states( - states=test_config.get('target_deploy_status', {})) vip = (zaza.model.get_application_config(application) .get("vip").get("value")) if vip: From f29aac19c4f1d16c8ebfed922d20dd84d757e51b Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 18 Oct 2019 10:14:34 -0700 Subject: [PATCH 150/898] Handle OSCI environment variables Move to preferred TEST_* environment variables, while remaining backward compatible. --- .../utilities/test_zaza_utilities_generic.py | 25 +++++++++++- zaza/openstack/utilities/generic.py | 38 +++++++++++++------ 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 4aa73f6..1d9d686 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -141,7 +141,30 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.assertEqual(generic_utils.get_undercloud_env_vars(), _expected_result) - # Overriding configure.network named variables + # Prefered OSCI TEST_ env vars + _env = {"NET_ID": "netid", + "NAMESERVER": "10.0.0.10", + "GATEWAY": "10.0.0.1", + "CIDR_EXT": "10.0.0.0/24", + "FIP_RANGE": "10.0.200.0:10.0.200.254", + "TEST_NET_ID": "test_netid", + "TEST_NAMESERVER": "10.9.0.10", + "TEST_GATEWAY": "10.9.0.1", + "TEST_CIDR_EXT": "10.9.0.0/24", + "TEST_FIP_RANGE": "10.9.200.0:10.0.200.254"} + _expected_result = {} + _expected_result["net_id"] = _env["TEST_NET_ID"] + _expected_result["external_dns"] = _env["TEST_NAMESERVER"] + _expected_result["default_gateway"] = _env["TEST_GATEWAY"] + _expected_result["external_net_cidr"] = _env["TEST_CIDR_EXT"] + _expected_result["start_floating_ip"] = _env[ + "TEST_FIP_RANGE"].split(":")[0] + _expected_result["end_floating_ip"] = _env[ + "TEST_FIP_RANGE"].split(":")[1] + self.assertEqual(generic_utils.get_undercloud_env_vars(), + _expected_result) + + # Overriding local configure.network named variables _override = {"start_floating_ip": "10.100.50.0", "end_floating_ip": "10.100.50.254", "default_gateway": "10.100.0.1", diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 856e036..2c81b88 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -139,21 +139,35 @@ def get_undercloud_env_vars(): :returns: Network environment variables :rtype: dict """ - # Handle backward compatibile OSCI enviornment variables + # Handle OSCI environment variables + # Over time we have changed the names of prefixes. Try them all in + # ascending order of significance. + # Note: TEST_* is the preferred prefix + _prefixes = ["AMULET_", "MOJO_", "OS_", "", "TEST_"] _vars = {} - _vars['net_id'] = os.environ.get('NET_ID') - _vars['external_dns'] = os.environ.get('NAMESERVER') - _vars['default_gateway'] = os.environ.get('GATEWAY') - _vars['external_net_cidr'] = os.environ.get('CIDR_EXT') + for prefix in _prefixes: + if os.environ.get('{}NET_ID'.format(prefix)) is not None: + _vars['net_id'] = os.environ.get('{}NET_ID'.format(prefix)) + if os.environ.get('{}NAMESERVER'.format(prefix))is not None: + _vars['external_dns'] = os.environ.get( + '{}NAMESERVER'.format(prefix)) + if os.environ.get('{}GATEWAY'.format(prefix)) is not None: + _vars['default_gateway'] = os.environ.get( + '{}GATEWAY'.format(prefix)) + if os.environ.get('{}CIDR_EXT'.format(prefix)) is not None: + _vars['external_net_cidr'] = os.environ.get( + '{}CIDR_EXT'.format(prefix)) - # Take FIP_RANGE and create start and end floating ips - _fip_range = os.environ.get('FIP_RANGE') - if _fip_range and ':' in _fip_range: - _vars['start_floating_ip'] = os.environ.get('FIP_RANGE').split(':')[0] - _vars['end_floating_ip'] = os.environ.get('FIP_RANGE').split(':')[1] + # Take FIP_RANGE and create start and end floating ips + _fip_range = os.environ.get('{}FIP_RANGE'.format(prefix)) + if _fip_range is not None and ':' in _fip_range: + _vars['start_floating_ip'] = os.environ.get( + '{}FIP_RANGE'.format(prefix)).split(':')[0] + _vars['end_floating_ip'] = os.environ.get( + '{}FIP_RANGE'.format(prefix)).split(':')[1] - # Env var naming consistent with zaza.openstack.configure.network - # functions takes priority. Override backward compatible settings. + # zaza.openstack.configure.network functions variables still take priority + # for local testing. Override OSCI settings. _keys = ['default_gateway', 'start_floating_ip', 'end_floating_ip', From 13fa5e9851d48e26557a18d014ea51a440acaa25 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 18 Oct 2019 11:26:37 -0700 Subject: [PATCH 151/898] Only use TEST_ prefix --- .../utilities/test_zaza_utilities_generic.py | 16 --------- zaza/openstack/utilities/generic.py | 35 +++++++------------ 2 files changed, 12 insertions(+), 39 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 1d9d686..f2b3365 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -125,22 +125,6 @@ class TestGenericUtils(ut_utils.BaseTestCase): return _env.get(key) self.get.side_effect = _get_env - # OSCI backward compatible env vars - _env = {"NET_ID": "netid", - "NAMESERVER": "10.0.0.10", - "GATEWAY": "10.0.0.1", - "CIDR_EXT": "10.0.0.0/24", - "FIP_RANGE": "10.0.200.0:10.0.200.254"} - _expected_result = {} - _expected_result["net_id"] = _env["NET_ID"] - _expected_result["external_dns"] = _env["NAMESERVER"] - _expected_result["default_gateway"] = _env["GATEWAY"] - _expected_result["external_net_cidr"] = _env["CIDR_EXT"] - _expected_result["start_floating_ip"] = _env["FIP_RANGE"].split(":")[0] - _expected_result["end_floating_ip"] = _env["FIP_RANGE"].split(":")[1] - self.assertEqual(generic_utils.get_undercloud_env_vars(), - _expected_result) - # Prefered OSCI TEST_ env vars _env = {"NET_ID": "netid", "NAMESERVER": "10.0.0.10", diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 2c81b88..e04ae34 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -140,31 +140,20 @@ def get_undercloud_env_vars(): :rtype: dict """ # Handle OSCI environment variables - # Over time we have changed the names of prefixes. Try them all in - # ascending order of significance. - # Note: TEST_* is the preferred prefix - _prefixes = ["AMULET_", "MOJO_", "OS_", "", "TEST_"] + # Note: TEST_* is the only prefix honored _vars = {} - for prefix in _prefixes: - if os.environ.get('{}NET_ID'.format(prefix)) is not None: - _vars['net_id'] = os.environ.get('{}NET_ID'.format(prefix)) - if os.environ.get('{}NAMESERVER'.format(prefix))is not None: - _vars['external_dns'] = os.environ.get( - '{}NAMESERVER'.format(prefix)) - if os.environ.get('{}GATEWAY'.format(prefix)) is not None: - _vars['default_gateway'] = os.environ.get( - '{}GATEWAY'.format(prefix)) - if os.environ.get('{}CIDR_EXT'.format(prefix)) is not None: - _vars['external_net_cidr'] = os.environ.get( - '{}CIDR_EXT'.format(prefix)) + _vars['net_id'] = os.environ.get('TEST_NET_ID') + _vars['external_dns'] = os.environ.get('TEST_NAMESERVER') + _vars['default_gateway'] = os.environ.get('TEST_GATEWAY') + _vars['external_net_cidr'] = os.environ.get('TEST_CIDR_EXT') - # Take FIP_RANGE and create start and end floating ips - _fip_range = os.environ.get('{}FIP_RANGE'.format(prefix)) - if _fip_range is not None and ':' in _fip_range: - _vars['start_floating_ip'] = os.environ.get( - '{}FIP_RANGE'.format(prefix)).split(':')[0] - _vars['end_floating_ip'] = os.environ.get( - '{}FIP_RANGE'.format(prefix)).split(':')[1] + # Take FIP_RANGE and create start and end floating ips + _fip_range = os.environ.get('TEST_FIP_RANGE') + if _fip_range is not None and ':' in _fip_range: + _vars['start_floating_ip'] = os.environ.get( + 'TEST_FIP_RANGE').split(':')[0] + _vars['end_floating_ip'] = os.environ.get( + 'TEST_FIP_RANGE').split(':')[1] # zaza.openstack.configure.network functions variables still take priority # for local testing. Override OSCI settings. From a207fea389e4bfce62296f13cad0ef2556feb134 Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 21 Oct 2019 16:06:52 -0700 Subject: [PATCH 152/898] Stop setting external DNS Use neutron DNS by not setting a DNS server on the subnet. Also correct TEST_NAME_SERVER for potential future use. --- zaza/openstack/configure/network.py | 4 ---- zaza/openstack/utilities/generic.py | 4 ++-- zaza/openstack/utilities/openstack.py | 8 +++++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 55bcdf3..409141c 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -172,10 +172,6 @@ def setup_sdn(network_config, keystone_session=None): network_config.get("private_net_cidr"), subnetpool=subnetpool, ip_version=ip_version) - openstack_utils.update_subnet_dns( - neutron_client, - project_subnet, - network_config["external_dns"]) openstack_utils.plug_subnet_into_router( neutron_client, network_config["router_name"], diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index e04ae34..7c66793 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -133,7 +133,7 @@ def get_undercloud_env_vars(): export NET_ID="a705dd0f-5571-4818-8c30-4132cc494668" export GATEWAY="172.17.107.1" export CIDR_EXT="172.17.107.0/24" - export NAMESERVER="10.5.0.2" + export NAME_SERVER="10.5.0.2" export FIP_RANGE="172.17.107.200:172.17.107.249" :returns: Network environment variables @@ -143,7 +143,7 @@ def get_undercloud_env_vars(): # Note: TEST_* is the only prefix honored _vars = {} _vars['net_id'] = os.environ.get('TEST_NET_ID') - _vars['external_dns'] = os.environ.get('TEST_NAMESERVER') + _vars['external_dns'] = os.environ.get('TEST_NAME_SERVER') _vars['default_gateway'] = os.environ.get('TEST_GATEWAY') _vars['external_net_cidr'] = os.environ.get('TEST_CIDR_EXT') diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 87b0934..f8ccd1b 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2078,8 +2078,8 @@ def get_ports_from_device_id(neutron_client, device_id): return ports -@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), - reraise=True, stop=tenacity.stop_after_attempt(10)) +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=120), + reraise=True, stop=tenacity.stop_after_attempt(12)) def cloud_init_complete(nova_client, vm_id, bootstring): """Wait for cloud init to complete on the given vm. @@ -2098,7 +2098,9 @@ def cloud_init_complete(nova_client, vm_id, bootstring): instance = nova_client.servers.find(id=vm_id) console_log = instance.get_console_output() if bootstring not in console_log: - raise exceptions.CloudInitIncomplete() + raise exceptions.CloudInitIncomplete( + "'{}' not found in console log: {}" + .format(bootstring, console_log)) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), From ea48e8b3d627ffa9e42731d61d1d7b4d1da347b7 Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 21 Oct 2019 16:19:42 -0700 Subject: [PATCH 153/898] Unit test fix --- unit_tests/utilities/test_zaza_utilities_generic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index f2b3365..3fb26da 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -127,18 +127,18 @@ class TestGenericUtils(ut_utils.BaseTestCase): # Prefered OSCI TEST_ env vars _env = {"NET_ID": "netid", - "NAMESERVER": "10.0.0.10", + "NAME_SERVER": "10.0.0.10", "GATEWAY": "10.0.0.1", "CIDR_EXT": "10.0.0.0/24", "FIP_RANGE": "10.0.200.0:10.0.200.254", "TEST_NET_ID": "test_netid", - "TEST_NAMESERVER": "10.9.0.10", + "TEST_NAME_SERVER": "10.9.0.10", "TEST_GATEWAY": "10.9.0.1", "TEST_CIDR_EXT": "10.9.0.0/24", "TEST_FIP_RANGE": "10.9.200.0:10.0.200.254"} _expected_result = {} _expected_result["net_id"] = _env["TEST_NET_ID"] - _expected_result["external_dns"] = _env["TEST_NAMESERVER"] + _expected_result["external_dns"] = _env["TEST_NAME_SERVER"] _expected_result["default_gateway"] = _env["TEST_GATEWAY"] _expected_result["external_net_cidr"] = _env["TEST_CIDR_EXT"] _expected_result["start_floating_ip"] = _env[ From addee58dd8710b8e20665240852d7e0c261fd9ac Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 22 Oct 2019 16:12:26 -0700 Subject: [PATCH 154/898] Do set DNS on overcloud subnets It turns out we do need to set DNS on the overcloud subnet. It should be the .2 of the CIDR under test and not the _admin_net's .2. There seems there are security group rules in the way. --- zaza/openstack/configure/network.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 409141c..55bcdf3 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -172,6 +172,10 @@ def setup_sdn(network_config, keystone_session=None): network_config.get("private_net_cidr"), subnetpool=subnetpool, ip_version=ip_version) + openstack_utils.update_subnet_dns( + neutron_client, + project_subnet, + network_config["external_dns"]) openstack_utils.plug_subnet_into_router( neutron_client, network_config["router_name"], From 47dc2a8b5f6fc63efbe3d6e9732f324b319e4629 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 23 Oct 2019 15:48:00 +0100 Subject: [PATCH 155/898] Add openstack-dashboard functional tests This PR adds the ported from the amulet tests in the openstack-dashboard charm. This is launchpad bug: 1828424 [1] [1]: https://bugs.launchpad.net/charm-openstack-dashboard/+bug/1828424 --- .../openstack_dashboard/__init__.py | 15 + .../charm_tests/openstack_dashboard/tests.py | 359 ++++++++++++++++++ 2 files changed, 374 insertions(+) create mode 100644 zaza/openstack/charm_tests/openstack_dashboard/__init__.py create mode 100644 zaza/openstack/charm_tests/openstack_dashboard/tests.py diff --git a/zaza/openstack/charm_tests/openstack_dashboard/__init__.py b/zaza/openstack/charm_tests/openstack_dashboard/__init__.py new file mode 100644 index 0000000..36f23e9 --- /dev/null +++ b/zaza/openstack/charm_tests/openstack_dashboard/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and openstack-dasboard.""" diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py new file mode 100644 index 0000000..ee39518 --- /dev/null +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -0,0 +1,359 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate horizon (openstack-dashboard) charm testing.""" + +import logging +import requests +import tenacity +import urllib.request + +import zaza.model as zaza_model + +import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.juju as openstack_juju + + +class OpenStackDashboardTests(test_utils.OpenStackBaseTest): + """Encapsulate openstack dashboard charm tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running openstack dashboard charm tests.""" + super(OpenStackDashboardTests, cls).setUpClass() + cls.application = 'openstack-dashboard' + + def test_050_local_settings_permissions_regression_check_lp1755027(self): + """Assert regression check lp1755027. + + Assert the intended file permissions on openstack-dashboard's + configuration file. Regression coverage for + https://bugs.launchpad.net/bugs/1755027. + + Ported from amulet tests. + """ + file_path = '/etc/openstack-dashboard/local_settings.py' + expected_perms = '640' + unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') + + logging.info('Checking {} permissions...'.format(file_path)) + + # NOTE(beisner): This could be a new test helper, but it needs + # to be a clean backport to stable with high prio, so maybe later. + cmd = 'stat -c %a {}'.format(file_path) + output = zaza_model.run_on_unit(unit_name, cmd) + perms = output['Stdout'].strip() + assert perms == expected_perms, \ + ('{} perms of {} not expected ones of {}' + .format(file_path, perms, expected_perms)) + + def test_100_services(self): + """Verify the expected services are running. + + Ported from amulet tests. + """ + logging.info('Checking openstack-dashboard services...') + + unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') + openstack_services = ['apache2'] + services = {} + services[unit_name] = openstack_services + + for unit_name, unit_services in services.items(): + zaza_model.block_until_service_status( + unit_name=unit_name, + services=unit_services, + target_status='running' + ) + + def test_302_router_settings(self): + """Verify that the horizon router settings are correct. + + Ported from amulet tests. + """ + # note this test is only valid after trusty-icehouse; however, all of + # the zaza tests are after trusty-icehouse + logging.info('Checking dashboard router settings...') + unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') + conf = ('/usr/share/openstack-dashboard/openstack_dashboard/' + 'enabled/_40_router.py') + + cmd = 'cat {}'.format(conf) + output = zaza_model.run_on_unit(unit_name, cmd) + + expected = { + 'DISABLED': "True", + } + mismatches = self.crude_py_parse(output['Stdout'], expected) + assert not mismatches, ("mismatched keys on {} were:\n{}" + .format(conf, ", ".join(mismatches))) + + def crude_py_parse(self, file_contents, expected): + """Parse a python file looking for key = value assignements.""" + mismatches = [] + for line in file_contents.split('\n'): + if '=' in line: + args = line.split('=') + if len(args) <= 1: + continue + key = args[0].strip() + value = args[1].strip() + if key in expected.keys(): + if expected[key] != value: + msg = "Mismatch %s != %s" % (expected[key], value) + mismatches.append(msg) + return mismatches + + def test_400_connection(self): + """Test that dashboard responds to http request. + + Ported from amulet tests. + """ + logging.info('Checking dashboard http response...') + + unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') + keystone_unit = zaza_model.get_lead_unit_name('keystone') + dashboard_relation = openstack_juju.get_relation_from_unit( + keystone_unit, unit_name, 'identity-service') + dashboard_ip = dashboard_relation['private-address'] + logging.debug("... dashboard_ip is:{}".format(dashboard_ip)) + + # NOTE(fnordahl) there is a eluding issue that currently makes the + # first request to the OpenStack Dashboard error out + # with 500 Internal Server Error in CI. Temporarilly + # add retry logic to unwedge the gate. This issue + # should be revisited and root caused properly when time + # allows. + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, + min=5, max=10), + reraise=True) + def do_request(): + logging.info("... trying to fetch the page") + try: + response = urllib.request.urlopen('http://{}/horizon' + .format(dashboard_ip)) + logging.info("... fetched page") + except Exception as e: + logging.info("... exception raised was {}".format(str(e))) + raise + return response.read().decode('utf-8') + html = do_request() + self.assertIn('OpenStack Dashboard', html, + "Dashboard frontpage check failed") + + class AuthExceptions(Exception): + """Exception base class for the 401 test.""" + + pass + + class FailedAuth(AuthExceptions): + """Failed exception for the 401 test.""" + + pass + + class PassedAuth(AuthExceptions): + """Passed exception for the 401 test.""" + + pass + + def test_401_authenticate(self): + """Validate that authentication succeeds for client log in. + + Ported from amulet tests. + """ + logging.info('Checking authentication through dashboard...') + + unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') + keystone_unit = zaza_model.get_lead_unit_name('keystone') + dashboard_relation = openstack_juju.get_relation_from_unit( + keystone_unit, unit_name, 'identity-service') + dashboard_ip = dashboard_relation['private-address'] + logging.debug("... dashboard_ip is:{}".format(dashboard_ip)) + + url = 'http://{}/horizon/auth/login/'.format(dashboard_ip) + + overcloud_auth = openstack_utils.get_overcloud_auth() + if overcloud_auth['OS_AUTH_URL'].endswith("v2.0"): + api_version = 2 + else: + api_version = 3 + keystone_client = openstack_utils.get_keystone_client( + overcloud_auth) + catalog = keystone_client.service_catalog.get_endpoints() + logging.info(catalog) + if api_version == 2: + region = catalog['identity'][0]['publicURL'] + else: + region = [i['url'] + for i in catalog['identity'] + if i['interface'] == 'public'][0] + + # NOTE(ajkavanagh) there used to be a trusty/icehouse test in the + # amulet test, but as the zaza tests only test from trusty/mitaka + # onwards, the test has been dropped + if (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('bionic_stein')): + expect = "Sign Out" + # update the in dashboard seems to require region to be default in + # this test configuration + region = 'default' + else: + expect = 'Projects - OpenStack Dashboard' + + # NOTE(thedac) Similar to the connection test above we get occasional + # intermittent authentication fails. Wrap in a retry loop. + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, + min=5, max=10), + retry=tenacity.retry_unless_exception_type( + self.AuthExceptions), + reraise=True) + def _do_auth_check(expect): + # start session, get csrftoken + client = requests.session() + client.get(url) + + if 'csrftoken' in client.cookies: + csrftoken = client.cookies['csrftoken'] + else: + raise Exception("Missing csrftoken") + + # build and send post request + auth = { + 'domain': 'admin_domain', + 'username': 'admin', + 'password': overcloud_auth['OS_PASSWORD'], + 'csrfmiddlewaretoken': csrftoken, + 'next': '/horizon/', + 'region': region, + } + + # In the minimal test deployment /horizon/project/ is unauthorized, + # this does not occur in a full deployment and is probably due to + # services/information missing that horizon wants to display data + # for. + # Redirect to /horizon/identity/ instead. + if (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('xenial_queens')): + auth['next'] = '/horizon/identity/' + + if (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('bionic_stein')): + auth['region'] = 'default' + + if api_version == 2: + del auth['domain'] + + logging.info('POST data: "{}"'.format(auth)) + response = client.post(url, data=auth, headers={'Referer': url}) + + if expect not in response.text: + msg = 'FAILURE code={} text="{}"'.format(response, + response.text) + # NOTE(thedac) amulet.raise_status exits on exception. + # Raise a custom exception. + logging.info("Yeah, wen't wrong: {}".format(msg)) + raise self.FailedAuth(msg) + raise self.PassedAuth() + + try: + _do_auth_check(expect) + except self.FailedAuth as e: + assert False, str(e) + except self.PassedAuth: + pass + + def test_404_connection(self): + """Verify the apache status module gets disabled when hardening apache. + + Ported from amulet tests. + """ + logging.info('Checking apache mod_status gets disabled.') + + unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') + keystone_unit = zaza_model.get_lead_unit_name('keystone') + dashboard_relation = openstack_juju.get_relation_from_unit( + keystone_unit, unit_name, 'identity-service') + dashboard_ip = dashboard_relation['private-address'] + logging.debug("... dashboard_ip is:{}".format(dashboard_ip)) + + logging.debug('Maybe enabling hardening for apache...') + _app_config = zaza_model.get_application_config(self.application_name) + logging.info(_app_config['harden']) + with self.config_change( + {'harden': _app_config['harden'].get('value', '')}, + {'harden': 'apache'}): + try: + urllib.request.urlopen('http://{}/server-status' + .format(dashboard_ip)) + except urllib.request.HTTPError as e: + if e.code == 404: + return + # test failed if it didn't return 404 + msg = "Apache mod_status check failed." + assert False, msg + + def test_501_security_checklist_action(self): + """Verify expected result on a default install. + + Ported from amulet tests. + """ + logging.info("Testing security-checklist") + unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') + action = zaza_model.run_action(unit_name, 'security-checklist') + assert action.data.get(u"status") == "failed", \ + "Security check is expected to not pass by default" + + def test_900_restart_on_config_change(self): + """Verify that the specified services are restarted on config changed. + + Ported from amulet tests. + """ + logging.info("Testing restart on config changed.") + + # Expected default and alternate values + current_value = zaza_model.get_application_config( + self.application_name)['use-syslog']['value'] + new_value = str(not bool(current_value)).title() + current_value = str(current_value).title() + + # Expected default and alternate values + set_default = {'use-syslog': current_value} + set_alternate = {'use-syslog': new_value} + + # Services which are expected to restart upon config change, + # and corresponding config files affected by the change + services = ['apache2', 'memcached'] + conf_file = '/etc/openstack-dashboard/local_settings.py' + + # Make config change, check for service restarts + logging.info('Setting use-syslog on openstack-dashboard {}' + .format(set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + None, None, + services) + + def test_910_pause_and_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + + Ported from amulet tests. + """ + with self.pause_resume(['apache2']): + logging.info("Testing pause resume") From d4ea94298fad209087f3556f4aa2319d7e683c3c Mon Sep 17 00:00:00 2001 From: Jose Delarosa Date: Fri, 25 Oct 2019 15:54:03 -0500 Subject: [PATCH 156/898] Add heat functional tests --- setup.py | 1 + zaza/openstack/charm_tests/heat/__init__.py | 17 + zaza/openstack/charm_tests/heat/tests.py | 363 ++++++++++++++++++ .../tests/files/icehouse/hot_hello_world.yaml | 66 ++++ .../tests/files/queens/hot_hello_world.yaml | 69 ++++ zaza/openstack/utilities/openstack.py | 13 + 6 files changed, 529 insertions(+) create mode 100644 zaza/openstack/charm_tests/heat/__init__.py create mode 100644 zaza/openstack/charm_tests/heat/tests.py create mode 100644 zaza/openstack/charm_tests/heat/tests/files/icehouse/hot_hello_world.yaml create mode 100644 zaza/openstack/charm_tests/heat/tests/files/queens/hot_hello_world.yaml diff --git a/setup.py b/setup.py index 7fe8b81..71c965b 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ install_require = [ 'tenacity', 'oslo.config', 'aodhclient', + 'python-heatclient', 'python-glanceclient', 'python-keystoneclient', 'python-novaclient', diff --git a/zaza/openstack/charm_tests/heat/__init__.py b/zaza/openstack/charm_tests/heat/__init__.py new file mode 100644 index 0000000..276455e --- /dev/null +++ b/zaza/openstack/charm_tests/heat/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing heat.""" diff --git a/zaza/openstack/charm_tests/heat/tests.py b/zaza/openstack/charm_tests/heat/tests.py new file mode 100644 index 0000000..f627da0 --- /dev/null +++ b/zaza/openstack/charm_tests/heat/tests.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 Canonical Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate heat testing.""" +import logging +import json +import os +import subprocess +from urllib import parse as urlparse +from heatclient.common import template_utils +from novaclient import exceptions + +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + +# Resource and name constants +IMAGE_NAME = 'cirros-image-1' +KEYPAIR_NAME = 'testkey' +STACK_NAME = 'hello_world' +RESOURCE_TYPE = 'server' +TEMPLATES_PATH = 'tests/files' +FLAVOR_NAME = 'm1.tiny' + + +class HeatBasicDeployment(test_utils.OpenStackBaseTest): + """Encapsulate Heat tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Heat tests.""" + super(HeatBasicDeployment, cls).setUpClass() + cls.application = 'heat' + cls.keystone_session = openstack_utils.get_overcloud_keystone_session() + cls.heat_client = openstack_utils.get_heat_session_client( + cls.keystone_session) + cls.glance_client = openstack_utils.get_glance_session_client( + cls.keystone_session) + cls.nova_client = openstack_utils.get_nova_session_client( + cls.keystone_session) + + @property + def services(self): + """Return a list services for OpenStack release.""" + services = ['heat-api', 'heat-api-cfn', 'heat-engine'] + return services + + def _image_create(self): + """Create an image to be used by the heat template, verify it exists""" + logging.info('Creating glance image ({})...'.format(IMAGE_NAME)) + + # Create a new image + image_url = openstack_utils.find_cirros_image(arch='x86_64') + image_new = openstack_utils.create_image( + self.glance_client, + image_url, + IMAGE_NAME) + + # Confirm image is created and has status of 'active' + if not image_new: + message = 'glance image create failed' + logging.error(message) + + # Verify new image name + images_list = list(self.glance_client.images.list()) + if images_list[0].name != IMAGE_NAME: + message = ('glance image create failed or unexpected ' + 'image name {}'.format(images_list[0].name)) + logging.error(message) + + def _keypair_create(self): + """Create a keypair to be used by the heat template, + or get a keypair if it exists.""" + + logging.info('Creating keypair {} if none exists'.format(KEYPAIR_NAME)) + if not openstack_utils.valid_key_exists(self.nova_client, + KEYPAIR_NAME): + key = openstack_utils.create_ssh_key( + self.nova_client, + KEYPAIR_NAME, + replace=True) + openstack_utils.write_private_key( + KEYPAIR_NAME, + key.private_key) + logging.info('Keypair created') + else: + logging.info('Keypair not created') + + def _stack_create(self): + """Create a heat stack from a basic heat template, verify its status""" + logging.info('Creating heat stack...') + + t_name = 'hot_hello_world.yaml' + if (openstack_utils.get_os_release() < + openstack_utils.get_os_release('xenial_queens')): + os_release = 'icehouse' + else: + os_release = 'queens' + + file_rel_path = os.path.join(TEMPLATES_PATH, os_release, t_name) + file_abs_path = os.path.abspath(file_rel_path) + t_url = urlparse.urlparse(file_abs_path, scheme='file').geturl() + logging.info('template url: {}'.format(t_url)) + + # Create flavor + try: + self.nova_client.flavors.find(name=FLAVOR_NAME) + except (exceptions.NotFound, exceptions.NoUniqueMatch): + logging.info('Creating flavor ({})'.format(FLAVOR_NAME)) + self.nova_client.flavors.create(FLAVOR_NAME, ram=512, vcpus=1, + disk=1, flavorid=1) + + r_req = self.heat_client.http_client + + t_files, template = template_utils.get_template_contents(t_url, r_req) + env_files, env = template_utils.process_environment_and_files( + env_path=None) + + fields = { + 'stack_name': STACK_NAME, + 'timeout_mins': '15', + 'disable_rollback': False, + 'parameters': { + 'admin_pass': 'Ubuntu', + 'key_name': KEYPAIR_NAME, + 'image': IMAGE_NAME + }, + 'template': template, + 'files': dict(list(t_files.items()) + list(env_files.items())), + 'environment': env + } + + # Create the stack. + try: + _stack = self.heat_client.stacks.create(**fields) + logging.info('Stack data: {}'.format(_stack)) + _stack_id = _stack['stack']['id'] + logging.info('Creating new stack, ID: {}'.format(_stack_id)) + except Exception as e: + # Generally, an api or cloud config error if this is hit. + msg = 'Failed to create heat stack: {}'.format(e) + logging.error(msg) + raise + + # Confirm stack reaches COMPLETE status. + # /!\ Heat stacks reach a COMPLETE status even when nova cannot + # find resources (a valid hypervisor) to fit the instance, in + # which case the heat stack self-deletes! Confirm anyway... + openstack_utils.resource_reaches_status(self.heat_client.stacks, + _stack_id, + expected_status="COMPLETE", + msg="Stack status wait") + _stacks = list(self.heat_client.stacks.list()) + logging.info('All stacks: {}'.format(_stacks)) + + # Confirm stack still exists. + try: + _stack = self.heat_client.stacks.get(STACK_NAME) + except Exception as e: + # Generally, a resource availability issue if this is hit. + msg = 'Failed to get heat stack: {}'.format(e) + logging.error(msg) + + # Confirm stack name. + logging.info('Expected, actual stack name: {}, ' + '{}'.format(STACK_NAME, _stack.stack_name)) + if STACK_NAME != _stack.stack_name: + msg = 'Stack name mismatch, {} != {}'.format(STACK_NAME, + _stack.stack_name) + logging.error(msg) + + def _stack_resource_compute(self): + """Confirm that the stack has created a subsequent nova + compute resource, and confirm its status.""" + logging.info('Confirming heat stack resource status...') + + # Confirm existence of a heat-generated nova compute resource. + _resource = self.heat_client.resources.get(STACK_NAME, RESOURCE_TYPE) + _server_id = _resource.physical_resource_id + if _server_id: + logging.debug('Heat template spawned nova instance, ' + 'ID: {}'.format(_server_id)) + else: + msg = 'Stack failed to spawn a nova compute resource (instance).' + logging.error(msg) + + # Confirm nova instance reaches ACTIVE status. + openstack_utils.resource_reaches_status(self.nova_client.servers, + _server_id, + expected_status="ACTIVE", + msg="nova instance") + logging.info('Nova instance reached ACTIVE status') + + def _stack_delete(self): + """Delete a heat stack, verify.""" + logging.info('Deleting heat stack...') + openstack_utils.delete_resource(self.heat_client.stacks, + STACK_NAME, msg="heat stack") + + def _image_delete(self): + """Delete that image.""" + logging.info('Deleting glance image...') + image = self.nova_client.glance.find_image(IMAGE_NAME) + openstack_utils.delete_resource(self.glance_client.images, + image.id, msg="glance image") + + def _keypair_delete(self): + """Delete that keypair.""" + logging.info('Deleting keypair...') + openstack_utils.delete_resource(self.nova_client.keypairs, + KEYPAIR_NAME, msg="nova keypair") + + def test_100_domain_setup(self): + # Action is REQUIRED to run for a functioning heat deployment + logging.info('Running domain-setup action on heat unit...') + unit = zaza.model.get_units(self.application_name)[0] + assert unit.workload_status == "active" + zaza.model.run_action(unit.entity_id, "domain-setup") + zaza.model.block_until_unit_wl_status(unit.entity_id, "active") + unit = zaza.model.get_unit_from_name(unit.entity_id) + assert unit.workload_status == "active" + + def test_400_heat_resource_types_list(self): + """Check default heat resource list behavior, also confirm + heat functionality.""" + logging.info('Checking default heat resource list...') + try: + types = list(self.heat_client.resource_types.list()) + if type(types) is list: + logging.info('Resource type list check is ok.') + else: + msg = 'Resource type list is not a list!' + logging.error('{}'.format(msg)) + raise + if len(types) > 0: + logging.info('Resource type list is populated ' + '({}, ok).'.format(len(types))) + else: + msg = 'Resource type list length is zero!' + logging.error(msg) + raise + except Exception as e: + msg = 'Resource type list failed: {}'.format(e) + logging.error(msg) + raise + + def test_402_heat_stack_list(self): + """Check default heat stack list behavior, also confirm + heat functionality.""" + logging.info('Checking default heat stack list...') + try: + stacks = list(self.heat_client.stacks.list()) + if type(stacks) is list: + logging.info("Stack list check is ok.") + else: + msg = 'Stack list returned something other than a list.' + logging.error(msg) + raise + except Exception as e: + msg = 'Heat stack list failed: {}'.format(e) + logging.error(msg) + raise + + def test_410_heat_stack_create_delete(self): + """Create a heat stack from template, confirm that a corresponding + nova compute resource is spawned, delete stack.""" + logging.info('Creating, deleting heat stack (compute)...') + self._image_create() + self._keypair_create() + self._stack_create() + self._stack_resource_compute() + self._stack_delete() + self._image_delete() + self._keypair_delete() + + def test_500_auth_encryption_key_same_on_units(self): + """Test that the auth_encryption_key in heat.conf is the same on all of + the units. + """ + logging.info("Checking the 'auth_encryption_key' is the same on " + "all units.") + output, ret = self._run_arbitrary( + "--application heat " + "--format json " + "grep auth_encryption_key /etc/heat/heat.conf") + if ret: + msg = "juju run returned error: ({}) -> {}".format(ret, output) + logging.error("Error: {}".format(msg)) + output = json.loads(output) + keys = {} + for r in output: + k = r['Stdout'].split('=')[1].strip() + keys[r['UnitId']] = k + # see if keys are different. + ks = list(keys.values()) + if any(((k != ks[0]) for k in ks[1:])): + msg = ("'auth_encryption_key' is not identical on every unit: {}" + .format("{}={}".format(k, v) for k, v in keys.items())) + logging.error("Error: {}".format(msg)) + + @staticmethod + def _run_arbitrary(command, timeout=300): + """Run an arbitrary command (as root), but not necessarily on a unit. + + (Otherwise the self.run(...) command could have been used for the unit + + :param str command: The command to run. + :param int timeout: Seconds to wait before timing out. + :return: A 2-tuple containing the output of the command and the exit + code of the command. + """ + cmd = ['juju', 'run', '--timeout', "{}s".format(timeout), + ] + command.split() + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + output = stdout if p.returncode == 0 else stderr + return output.decode('utf8').strip(), p.returncode + + def test_900_heat_restart_on_config_change(self): + """Verify that the specified services are restarted when the config + is changed.""" + logging.info('Testing restart on configuration change') + + # Expected default and alternate values + set_default = {'use-syslog': 'False'} + set_alternate = {'use-syslog': 'True'} + + # Config file affected by juju set config change + conf_file = '/etc/heat/heat.conf' + + # Make config change, check for service restarts + # In Amulet we waited 30 seconds...do we still need to? + logging.info('Making configuration change') + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + None, + None, + self.services) + + def test_910_pause_and_resume(self): + """Run services pause and resume tests.""" + logging.info('Checking pause and resume actions...') + + with self.pause_resume(self.services): + logging.info("Testing pause resume") diff --git a/zaza/openstack/charm_tests/heat/tests/files/icehouse/hot_hello_world.yaml b/zaza/openstack/charm_tests/heat/tests/files/icehouse/hot_hello_world.yaml new file mode 100644 index 0000000..f799a2a --- /dev/null +++ b/zaza/openstack/charm_tests/heat/tests/files/icehouse/hot_hello_world.yaml @@ -0,0 +1,66 @@ +# +# This is a hello world HOT template just defining a single compute +# server. +# +heat_template_version: 2013-05-23 + +description: > + Hello world HOT template that just defines a single server. + Contains just base features to verify base HOT support. + +parameters: + key_name: + type: string + description: Name of an existing key pair to use for the server + constraints: + - custom_constraint: nova.keypair + flavor: + type: string + description: Flavor for the server to be created + default: m1.tiny + constraints: + - custom_constraint: nova.flavor + image: + type: string + description: Image ID or image name to use for the server + constraints: + - custom_constraint: glance.image + admin_pass: + type: string + description: Admin password + hidden: true + constraints: + - length: { min: 6, max: 8 } + description: Password length must be between 6 and 8 characters + - allowed_pattern: "[a-zA-Z0-9]+" + description: Password must consist of characters and numbers only + - allowed_pattern: "[A-Z]+[a-zA-Z0-9]*" + description: Password must start with an uppercase character + db_port: + type: number + description: Database port number + default: 50000 + constraints: + - range: { min: 40000, max: 60000 } + description: Port number must be between 40000 and 60000 + +resources: + server: + type: OS::Nova::Server + properties: + key_name: { get_param: key_name } + image: { get_param: image } + flavor: { get_param: flavor } + admin_pass: { get_param: admin_pass } + user_data: + str_replace: + template: | + #!/bin/bash + echo db_port + params: + db_port: { get_param: db_port } + +outputs: + server_networks: + description: The networks of the deployed server + value: { get_attr: [server, networks] } diff --git a/zaza/openstack/charm_tests/heat/tests/files/queens/hot_hello_world.yaml b/zaza/openstack/charm_tests/heat/tests/files/queens/hot_hello_world.yaml new file mode 100644 index 0000000..53302c1 --- /dev/null +++ b/zaza/openstack/charm_tests/heat/tests/files/queens/hot_hello_world.yaml @@ -0,0 +1,69 @@ +# +# This is a hello world HOT template just defining a single compute +# server. +# +heat_template_version: 2013-05-23 + +description: > + Hello world HOT template that just defines a single server. + Contains just base features to verify base HOT support. + +parameters: + key_name: + type: string + description: Name of an existing key pair to use for the server + constraints: + - custom_constraint: nova.keypair + flavor: + type: string + description: Flavor for the server to be created + default: m1.tiny + constraints: + - custom_constraint: nova.flavor + image: + type: string + description: Image ID or image name to use for the server + constraints: + - custom_constraint: glance.image + admin_pass: + type: string + description: Admin password + hidden: true + constraints: + - length: { min: 6, max: 8 } + description: Password length must be between 6 and 8 characters + - allowed_pattern: "[a-zA-Z0-9]+" + description: Password must consist of characters and numbers only + - allowed_pattern: "[A-Z]+[a-zA-Z0-9]*" + description: Password must start with an uppercase character + db_port: + type: number + description: Database port number + default: 50000 + constraints: + - range: { min: 40000, max: 60000 } + description: Port number must be between 40000 and 60000 + +resources: + server: + type: OS::Nova::Server + properties: + key_name: { get_param: key_name } + image: { get_param: image } + flavor: { get_param: flavor } + admin_pass: { get_param: admin_pass } + # See https://docs.openstack.org/heat/queens/template_guide/contrib.html#OS::Nova::Server-prop-networks + networks: + - allocate_network: none + user_data: + str_replace: + template: | + #!/bin/bash + echo db_port + params: + db_port: { get_param: db_port } + +outputs: + server_networks: + description: The networks of the deployed server + value: { get_attr: [server, networks] } diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index f8ccd1b..4cc764c 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -27,6 +27,7 @@ from openstack import connection from aodhclient.v2 import client as aodh_client from cinderclient import client as cinderclient +from heatclient import client as heatclient from glanceclient import Client as GlanceClient from keystoneclient.v2_0 import client as keystoneclient_v2 @@ -258,6 +259,18 @@ def get_octavia_session_client(session, service_type='load-balancer', endpoint=endpoint.url) +def get_heat_session_client(session, version=1): + """Return heatclient authenticated by keystone session. + + :param session: Keystone session object + :type session: keystoneauth1.session.Session object + :param version: Heat API version + :type version: int + :returns: Authenticated cinderclient + :rtype: heatclient.Client object + """ + return heatclient.Client(session=session, version=version) + def get_cinder_session_client(session, version=2): """Return cinderclient authenticated by keystone session. From 08b5b7085408e41f96e146798e9e5c2cfc5113d3 Mon Sep 17 00:00:00 2001 From: Jose Delarosa Date: Fri, 25 Oct 2019 16:28:27 -0500 Subject: [PATCH 157/898] Fix pep8 failures --- zaza/openstack/charm_tests/heat/tests.py | 28 +++++++++--------------- zaza/openstack/utilities/openstack.py | 1 + 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/zaza/openstack/charm_tests/heat/tests.py b/zaza/openstack/charm_tests/heat/tests.py index f627da0..8ea741b 100644 --- a/zaza/openstack/charm_tests/heat/tests.py +++ b/zaza/openstack/charm_tests/heat/tests.py @@ -59,7 +59,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): return services def _image_create(self): - """Create an image to be used by the heat template, verify it exists""" + """Create an image for use by the heat template, verify it exists.""" logging.info('Creating glance image ({})...'.format(IMAGE_NAME)) # Create a new image @@ -82,9 +82,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): logging.error(message) def _keypair_create(self): - """Create a keypair to be used by the heat template, - or get a keypair if it exists.""" - + """Create a keypair or get a keypair if it exists.""" logging.info('Creating keypair {} if none exists'.format(KEYPAIR_NAME)) if not openstack_utils.valid_key_exists(self.nova_client, KEYPAIR_NAME): @@ -100,7 +98,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): logging.info('Keypair not created') def _stack_create(self): - """Create a heat stack from a basic heat template, verify its status""" + """Create a heat stack from a heat template, verify its status.""" logging.info('Creating heat stack...') t_name = 'hot_hello_world.yaml' @@ -183,8 +181,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): logging.error(msg) def _stack_resource_compute(self): - """Confirm that the stack has created a subsequent nova - compute resource, and confirm its status.""" + """Confirm the stack has created a nova resource and check status.""" logging.info('Confirming heat stack resource status...') # Confirm existence of a heat-generated nova compute resource. @@ -224,6 +221,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): KEYPAIR_NAME, msg="nova keypair") def test_100_domain_setup(self): + """Run required action for a working Heat unit.""" # Action is REQUIRED to run for a functioning heat deployment logging.info('Running domain-setup action on heat unit...') unit = zaza.model.get_units(self.application_name)[0] @@ -234,8 +232,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): assert unit.workload_status == "active" def test_400_heat_resource_types_list(self): - """Check default heat resource list behavior, also confirm - heat functionality.""" + """Check default resource list behavior and confirm functionality.""" logging.info('Checking default heat resource list...') try: types = list(self.heat_client.resource_types.list()) @@ -258,8 +255,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): raise def test_402_heat_stack_list(self): - """Check default heat stack list behavior, also confirm - heat functionality.""" + """Check default heat stack list behavior, confirm functionality.""" logging.info('Checking default heat stack list...') try: stacks = list(self.heat_client.stacks.list()) @@ -275,8 +271,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): raise def test_410_heat_stack_create_delete(self): - """Create a heat stack from template, confirm that a corresponding - nova compute resource is spawned, delete stack.""" + """Create stack, confirm nova compute resource, delete stack.""" logging.info('Creating, deleting heat stack (compute)...') self._image_create() self._keypair_create() @@ -287,9 +282,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): self._keypair_delete() def test_500_auth_encryption_key_same_on_units(self): - """Test that the auth_encryption_key in heat.conf is the same on all of - the units. - """ + """Test the auth_encryption_key in heat.conf is same on all units.""" logging.info("Checking the 'auth_encryption_key' is the same on " "all units.") output, ret = self._run_arbitrary( @@ -333,8 +326,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): return output.decode('utf8').strip(), p.returncode def test_900_heat_restart_on_config_change(self): - """Verify that the specified services are restarted when the config - is changed.""" + """Verify the specified services are restarted when config changes.""" logging.info('Testing restart on configuration change') # Expected default and alternate values diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 4cc764c..442c878 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -271,6 +271,7 @@ def get_heat_session_client(session, version=1): """ return heatclient.Client(session=session, version=version) + def get_cinder_session_client(session, version=2): """Return cinderclient authenticated by keystone session. From 80d1f1086d0bd9ff7aad7a41fa133805c40ec244 Mon Sep 17 00:00:00 2001 From: Jose Delarosa Date: Mon, 28 Oct 2019 23:30:51 -0500 Subject: [PATCH 158/898] Code improvements and fixes * Removed heat templates * Improved Docstring content * Added code to assert errors when failing * Removed private variable names * Removed test_402_heat_stack_list function --- zaza/openstack/charm_tests/heat/tests.py | 107 ++++++++---------- .../tests/files/icehouse/hot_hello_world.yaml | 66 ----------- .../tests/files/queens/hot_hello_world.yaml | 69 ----------- 3 files changed, 50 insertions(+), 192 deletions(-) delete mode 100644 zaza/openstack/charm_tests/heat/tests/files/icehouse/hot_hello_world.yaml delete mode 100644 zaza/openstack/charm_tests/heat/tests/files/queens/hot_hello_world.yaml diff --git a/zaza/openstack/charm_tests/heat/tests.py b/zaza/openstack/charm_tests/heat/tests.py index 8ea741b..878b108 100644 --- a/zaza/openstack/charm_tests/heat/tests.py +++ b/zaza/openstack/charm_tests/heat/tests.py @@ -26,13 +26,14 @@ from novaclient import exceptions import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils +import zaza.charm_lifecycle.utils as charm_lifecycle_utils # Resource and name constants IMAGE_NAME = 'cirros-image-1' KEYPAIR_NAME = 'testkey' STACK_NAME = 'hello_world' RESOURCE_TYPE = 'server' -TEMPLATES_PATH = 'tests/files' +TEMPLATES_PATH = 'files' FLAVOR_NAME = 'm1.tiny' @@ -54,7 +55,11 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list services for OpenStack release.""" + """Return a list services for OpenStack release. + + :returns: List of services + :rtype: [str] + """ services = ['heat-api', 'heat-api-cfn', 'heat-engine'] return services @@ -71,15 +76,14 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): # Confirm image is created and has status of 'active' if not image_new: - message = 'glance image create failed' - logging.error(message) + assert False, 'glance image create failed' # Verify new image name images_list = list(self.glance_client.images.list()) if images_list[0].name != IMAGE_NAME: message = ('glance image create failed or unexpected ' 'image name {}'.format(images_list[0].name)) - logging.error(message) + assert False, message def _keypair_create(self): """Create a keypair or get a keypair if it exists.""" @@ -95,7 +99,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): key.private_key) logging.info('Keypair created') else: - logging.info('Keypair not created') + assert False, 'Keypair not created' def _stack_create(self): """Create a heat stack from a heat template, verify its status.""" @@ -108,7 +112,13 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): else: os_release = 'queens' - file_rel_path = os.path.join(TEMPLATES_PATH, os_release, t_name) + # Get location of template files in charm-heat + bundle_path = charm_lifecycle_utils.BUNDLE_DIR + if bundle_path[-1:] == "/": + bundle_path = bundle_path[0:-1] + + file_rel_path = os.path.join(os.path.dirname(bundle_path), + TEMPLATES_PATH, os_release, t_name) file_abs_path = os.path.abspath(file_rel_path) t_url = urlparse.urlparse(file_abs_path, scheme='file').geturl() logging.info('template url: {}'.format(t_url)) @@ -143,10 +153,10 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): # Create the stack. try: - _stack = self.heat_client.stacks.create(**fields) - logging.info('Stack data: {}'.format(_stack)) - _stack_id = _stack['stack']['id'] - logging.info('Creating new stack, ID: {}'.format(_stack_id)) + stack = self.heat_client.stacks.create(**fields) + logging.info('Stack data: {}'.format(stack)) + stack_id = stack['stack']['id'] + logging.info('Creating new stack, ID: {}'.format(stack_id)) except Exception as e: # Generally, an api or cloud config error if this is hit. msg = 'Failed to create heat stack: {}'.format(e) @@ -158,45 +168,46 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): # find resources (a valid hypervisor) to fit the instance, in # which case the heat stack self-deletes! Confirm anyway... openstack_utils.resource_reaches_status(self.heat_client.stacks, - _stack_id, + stack_id, expected_status="COMPLETE", msg="Stack status wait") - _stacks = list(self.heat_client.stacks.list()) - logging.info('All stacks: {}'.format(_stacks)) + # List stack + stacks = list(self.heat_client.stacks.list()) + logging.info('All stacks: {}'.format(stacks)) - # Confirm stack still exists. + # Get stack information try: - _stack = self.heat_client.stacks.get(STACK_NAME) + stack = self.heat_client.stacks.get(STACK_NAME) except Exception as e: # Generally, a resource availability issue if this is hit. msg = 'Failed to get heat stack: {}'.format(e) - logging.error(msg) + assert False, msg # Confirm stack name. logging.info('Expected, actual stack name: {}, ' - '{}'.format(STACK_NAME, _stack.stack_name)) - if STACK_NAME != _stack.stack_name: + '{}'.format(STACK_NAME, stack.stack_name)) + if STACK_NAME != stack.stack_name: msg = 'Stack name mismatch, {} != {}'.format(STACK_NAME, - _stack.stack_name) - logging.error(msg) + stack.stack_name) + assert False, msg def _stack_resource_compute(self): """Confirm the stack has created a nova resource and check status.""" logging.info('Confirming heat stack resource status...') # Confirm existence of a heat-generated nova compute resource. - _resource = self.heat_client.resources.get(STACK_NAME, RESOURCE_TYPE) - _server_id = _resource.physical_resource_id - if _server_id: + resource = self.heat_client.resources.get(STACK_NAME, RESOURCE_TYPE) + server_id = resource.physical_resource_id + if server_id: logging.debug('Heat template spawned nova instance, ' - 'ID: {}'.format(_server_id)) + 'ID: {}'.format(server_id)) else: msg = 'Stack failed to spawn a nova compute resource (instance).' logging.error(msg) # Confirm nova instance reaches ACTIVE status. openstack_utils.resource_reaches_status(self.nova_client.servers, - _server_id, + server_id, expected_status="ACTIVE", msg="nova instance") logging.info('Nova instance reached ACTIVE status') @@ -228,47 +239,26 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): assert unit.workload_status == "active" zaza.model.run_action(unit.entity_id, "domain-setup") zaza.model.block_until_unit_wl_status(unit.entity_id, "active") - unit = zaza.model.get_unit_from_name(unit.entity_id) - assert unit.workload_status == "active" def test_400_heat_resource_types_list(self): """Check default resource list behavior and confirm functionality.""" logging.info('Checking default heat resource list...') try: - types = list(self.heat_client.resource_types.list()) + types = self.heat_client.resource_types.list() if type(types) is list: logging.info('Resource type list check is ok.') else: msg = 'Resource type list is not a list!' - logging.error('{}'.format(msg)) - raise + assert False, msg if len(types) > 0: logging.info('Resource type list is populated ' '({}, ok).'.format(len(types))) else: msg = 'Resource type list length is zero!' - logging.error(msg) - raise + assert False, msg except Exception as e: msg = 'Resource type list failed: {}'.format(e) - logging.error(msg) - raise - - def test_402_heat_stack_list(self): - """Check default heat stack list behavior, confirm functionality.""" - logging.info('Checking default heat stack list...') - try: - stacks = list(self.heat_client.stacks.list()) - if type(stacks) is list: - logging.info("Stack list check is ok.") - else: - msg = 'Stack list returned something other than a list.' - logging.error(msg) - raise - except Exception as e: - msg = 'Heat stack list failed: {}'.format(e) - logging.error(msg) - raise + assert False, msg def test_410_heat_stack_create_delete(self): """Create stack, confirm nova compute resource, delete stack.""" @@ -290,8 +280,8 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): "--format json " "grep auth_encryption_key /etc/heat/heat.conf") if ret: - msg = "juju run returned error: ({}) -> {}".format(ret, output) - logging.error("Error: {}".format(msg)) + msg = "juju run error: ret: {}, output: {}".format(ret, output) + self.assertEqual(ret, 0, msg) output = json.loads(output) keys = {} for r in output: @@ -310,10 +300,13 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): (Otherwise the self.run(...) command could have been used for the unit - :param str command: The command to run. - :param int timeout: Seconds to wait before timing out. - :return: A 2-tuple containing the output of the command and the exit - code of the command. + :param command: The command to run. + :type command: str + :param timeout: Seconds to wait before timing out. + :type timeout: int + :raises: subprocess.CalledProcessError. + :returns: A pair containing the output of the command and exit value + :rtype: (str, int) """ cmd = ['juju', 'run', '--timeout', "{}s".format(timeout), ] + command.split() diff --git a/zaza/openstack/charm_tests/heat/tests/files/icehouse/hot_hello_world.yaml b/zaza/openstack/charm_tests/heat/tests/files/icehouse/hot_hello_world.yaml deleted file mode 100644 index f799a2a..0000000 --- a/zaza/openstack/charm_tests/heat/tests/files/icehouse/hot_hello_world.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# -# This is a hello world HOT template just defining a single compute -# server. -# -heat_template_version: 2013-05-23 - -description: > - Hello world HOT template that just defines a single server. - Contains just base features to verify base HOT support. - -parameters: - key_name: - type: string - description: Name of an existing key pair to use for the server - constraints: - - custom_constraint: nova.keypair - flavor: - type: string - description: Flavor for the server to be created - default: m1.tiny - constraints: - - custom_constraint: nova.flavor - image: - type: string - description: Image ID or image name to use for the server - constraints: - - custom_constraint: glance.image - admin_pass: - type: string - description: Admin password - hidden: true - constraints: - - length: { min: 6, max: 8 } - description: Password length must be between 6 and 8 characters - - allowed_pattern: "[a-zA-Z0-9]+" - description: Password must consist of characters and numbers only - - allowed_pattern: "[A-Z]+[a-zA-Z0-9]*" - description: Password must start with an uppercase character - db_port: - type: number - description: Database port number - default: 50000 - constraints: - - range: { min: 40000, max: 60000 } - description: Port number must be between 40000 and 60000 - -resources: - server: - type: OS::Nova::Server - properties: - key_name: { get_param: key_name } - image: { get_param: image } - flavor: { get_param: flavor } - admin_pass: { get_param: admin_pass } - user_data: - str_replace: - template: | - #!/bin/bash - echo db_port - params: - db_port: { get_param: db_port } - -outputs: - server_networks: - description: The networks of the deployed server - value: { get_attr: [server, networks] } diff --git a/zaza/openstack/charm_tests/heat/tests/files/queens/hot_hello_world.yaml b/zaza/openstack/charm_tests/heat/tests/files/queens/hot_hello_world.yaml deleted file mode 100644 index 53302c1..0000000 --- a/zaza/openstack/charm_tests/heat/tests/files/queens/hot_hello_world.yaml +++ /dev/null @@ -1,69 +0,0 @@ -# -# This is a hello world HOT template just defining a single compute -# server. -# -heat_template_version: 2013-05-23 - -description: > - Hello world HOT template that just defines a single server. - Contains just base features to verify base HOT support. - -parameters: - key_name: - type: string - description: Name of an existing key pair to use for the server - constraints: - - custom_constraint: nova.keypair - flavor: - type: string - description: Flavor for the server to be created - default: m1.tiny - constraints: - - custom_constraint: nova.flavor - image: - type: string - description: Image ID or image name to use for the server - constraints: - - custom_constraint: glance.image - admin_pass: - type: string - description: Admin password - hidden: true - constraints: - - length: { min: 6, max: 8 } - description: Password length must be between 6 and 8 characters - - allowed_pattern: "[a-zA-Z0-9]+" - description: Password must consist of characters and numbers only - - allowed_pattern: "[A-Z]+[a-zA-Z0-9]*" - description: Password must start with an uppercase character - db_port: - type: number - description: Database port number - default: 50000 - constraints: - - range: { min: 40000, max: 60000 } - description: Port number must be between 40000 and 60000 - -resources: - server: - type: OS::Nova::Server - properties: - key_name: { get_param: key_name } - image: { get_param: image } - flavor: { get_param: flavor } - admin_pass: { get_param: admin_pass } - # See https://docs.openstack.org/heat/queens/template_guide/contrib.html#OS::Nova::Server-prop-networks - networks: - - allocate_network: none - user_data: - str_replace: - template: | - #!/bin/bash - echo db_port - params: - db_port: { get_param: db_port } - -outputs: - server_networks: - description: The networks of the deployed server - value: { get_attr: [server, networks] } From b2141a1c6264e23b22f01cfc7eb7f90e6867d376 Mon Sep 17 00:00:00 2001 From: Jose Delarosa Date: Tue, 29 Oct 2019 23:28:07 -0500 Subject: [PATCH 159/898] heat: more enhancements * Move code into test_410_heat_stack...() instead of calling functions * Use glance and nova helper functions as available * Simplify method for detecting duplicate encryption keys --- zaza/openstack/charm_tests/heat/tests.py | 146 +++++++++-------------- 1 file changed, 55 insertions(+), 91 deletions(-) diff --git a/zaza/openstack/charm_tests/heat/tests.py b/zaza/openstack/charm_tests/heat/tests.py index 878b108..ebb491b 100644 --- a/zaza/openstack/charm_tests/heat/tests.py +++ b/zaza/openstack/charm_tests/heat/tests.py @@ -24,13 +24,15 @@ from heatclient.common import template_utils from novaclient import exceptions import zaza.model +import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.charm_tests.nova.setup as nova_setup +import zaza.openstack.charm_tests.nova.utils as nova_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.charm_lifecycle.utils as charm_lifecycle_utils # Resource and name constants -IMAGE_NAME = 'cirros-image-1' -KEYPAIR_NAME = 'testkey' +IMAGE_NAME = 'cirros' STACK_NAME = 'hello_world' RESOURCE_TYPE = 'server' TEMPLATES_PATH = 'files' @@ -63,20 +65,41 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): services = ['heat-api', 'heat-api-cfn', 'heat-engine'] return services - def _image_create(self): - """Create an image for use by the heat template, verify it exists.""" + def test_100_domain_setup(self): + """Run required action for a working Heat unit.""" + # Action is REQUIRED to run for a functioning heat deployment + logging.info('Running domain-setup action on heat unit...') + unit = zaza.model.get_units(self.application_name)[0] + zaza.model.block_until_unit_wl_status(unit.entity_id, "active") + zaza.model.run_action(unit.entity_id, "domain-setup") + zaza.model.block_until_unit_wl_status(unit.entity_id, "active") + + def test_400_heat_resource_types_list(self): + """Check default resource list behavior and confirm functionality.""" + logging.info('Checking default heat resource list...') + try: + types = self.heat_client.resource_types.list() + if type(types) is list: + logging.info('Resource type list check is ok.') + else: + msg = 'Resource type list is not a list!' + assert False, msg + if len(types) > 0: + logging.info('Resource type list is populated ' + '({}, ok).'.format(len(types))) + else: + msg = 'Resource type list length is zero!' + assert False, msg + except Exception as e: + msg = 'Resource type list failed: {}'.format(e) + assert False, msg + + def test_410_heat_stack_create_delete(self): + """Create stack, confirm nova compute resource, delete stack.""" + + # Create an image for use by the heat template logging.info('Creating glance image ({})...'.format(IMAGE_NAME)) - - # Create a new image - image_url = openstack_utils.find_cirros_image(arch='x86_64') - image_new = openstack_utils.create_image( - self.glance_client, - image_url, - IMAGE_NAME) - - # Confirm image is created and has status of 'active' - if not image_new: - assert False, 'glance image create failed' + glance_setup.add_cirros_image(image_name=IMAGE_NAME) # Verify new image name images_list = list(self.glance_client.images.list()) @@ -85,24 +108,11 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): 'image name {}'.format(images_list[0].name)) assert False, message - def _keypair_create(self): - """Create a keypair or get a keypair if it exists.""" - logging.info('Creating keypair {} if none exists'.format(KEYPAIR_NAME)) - if not openstack_utils.valid_key_exists(self.nova_client, - KEYPAIR_NAME): - key = openstack_utils.create_ssh_key( - self.nova_client, - KEYPAIR_NAME, - replace=True) - openstack_utils.write_private_key( - KEYPAIR_NAME, - key.private_key) - logging.info('Keypair created') - else: - assert False, 'Keypair not created' + # Create a keypair + logging.info('Creating keypair {}'.format(nova_utils.KEYPAIR_NAME)) + nova_setup.manage_ssh_key(self.nova_client) - def _stack_create(self): - """Create a heat stack from a heat template, verify its status.""" + # Create a heat stack from a heat template, verify its status logging.info('Creating heat stack...') t_name = 'hot_hello_world.yaml' @@ -132,7 +142,6 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): disk=1, flavorid=1) r_req = self.heat_client.http_client - t_files, template = template_utils.get_template_contents(t_url, r_req) env_files, env = template_utils.process_environment_and_files( env_path=None) @@ -143,7 +152,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): 'disable_rollback': False, 'parameters': { 'admin_pass': 'Ubuntu', - 'key_name': KEYPAIR_NAME, + 'key_name': nova_utils.KEYPAIR_NAME, 'image': IMAGE_NAME }, 'template': template, @@ -151,7 +160,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): 'environment': env } - # Create the stack. + # Create the stack try: stack = self.heat_client.stacks.create(**fields) logging.info('Stack data: {}'.format(stack)) @@ -191,11 +200,8 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): stack.stack_name) assert False, msg - def _stack_resource_compute(self): - """Confirm the stack has created a nova resource and check status.""" + # Confirm existence of a heat-generated nova compute resource logging.info('Confirming heat stack resource status...') - - # Confirm existence of a heat-generated nova compute resource. resource = self.heat_client.resources.get(STACK_NAME, RESOURCE_TYPE) server_id = resource.physical_resource_id if server_id: @@ -205,71 +211,29 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): msg = 'Stack failed to spawn a nova compute resource (instance).' logging.error(msg) - # Confirm nova instance reaches ACTIVE status. + # Confirm nova instance reaches ACTIVE status openstack_utils.resource_reaches_status(self.nova_client.servers, server_id, expected_status="ACTIVE", msg="nova instance") logging.info('Nova instance reached ACTIVE status') - def _stack_delete(self): - """Delete a heat stack, verify.""" + # Delete stack logging.info('Deleting heat stack...') openstack_utils.delete_resource(self.heat_client.stacks, STACK_NAME, msg="heat stack") - def _image_delete(self): - """Delete that image.""" + # Delete image logging.info('Deleting glance image...') image = self.nova_client.glance.find_image(IMAGE_NAME) openstack_utils.delete_resource(self.glance_client.images, image.id, msg="glance image") - def _keypair_delete(self): - """Delete that keypair.""" + # Delete keypair logging.info('Deleting keypair...') openstack_utils.delete_resource(self.nova_client.keypairs, - KEYPAIR_NAME, msg="nova keypair") - - def test_100_domain_setup(self): - """Run required action for a working Heat unit.""" - # Action is REQUIRED to run for a functioning heat deployment - logging.info('Running domain-setup action on heat unit...') - unit = zaza.model.get_units(self.application_name)[0] - assert unit.workload_status == "active" - zaza.model.run_action(unit.entity_id, "domain-setup") - zaza.model.block_until_unit_wl_status(unit.entity_id, "active") - - def test_400_heat_resource_types_list(self): - """Check default resource list behavior and confirm functionality.""" - logging.info('Checking default heat resource list...') - try: - types = self.heat_client.resource_types.list() - if type(types) is list: - logging.info('Resource type list check is ok.') - else: - msg = 'Resource type list is not a list!' - assert False, msg - if len(types) > 0: - logging.info('Resource type list is populated ' - '({}, ok).'.format(len(types))) - else: - msg = 'Resource type list length is zero!' - assert False, msg - except Exception as e: - msg = 'Resource type list failed: {}'.format(e) - assert False, msg - - def test_410_heat_stack_create_delete(self): - """Create stack, confirm nova compute resource, delete stack.""" - logging.info('Creating, deleting heat stack (compute)...') - self._image_create() - self._keypair_create() - self._stack_create() - self._stack_resource_compute() - self._stack_delete() - self._image_delete() - self._keypair_delete() + nova_utils.KEYPAIR_NAME, + msg="nova keypair") def test_500_auth_encryption_key_same_on_units(self): """Test the auth_encryption_key in heat.conf is same on all units.""" @@ -287,12 +251,12 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): for r in output: k = r['Stdout'].split('=')[1].strip() keys[r['UnitId']] = k - # see if keys are different. - ks = list(keys.values()) - if any(((k != ks[0]) for k in ks[1:])): + # see if keys are different + ks = set(keys.values()) + if len(ks) != 1: msg = ("'auth_encryption_key' is not identical on every unit: {}" .format("{}={}".format(k, v) for k, v in keys.items())) - logging.error("Error: {}".format(msg)) + assert False, msg @staticmethod def _run_arbitrary(command, timeout=300): From 806e6afd8704ce6b9afa4d9d39ba648917573f27 Mon Sep 17 00:00:00 2001 From: Jose Delarosa Date: Tue, 29 Oct 2019 23:46:52 -0500 Subject: [PATCH 160/898] Removed blank line after docstring for pep8 compliance --- zaza/openstack/charm_tests/heat/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zaza/openstack/charm_tests/heat/tests.py b/zaza/openstack/charm_tests/heat/tests.py index ebb491b..fba3577 100644 --- a/zaza/openstack/charm_tests/heat/tests.py +++ b/zaza/openstack/charm_tests/heat/tests.py @@ -96,7 +96,6 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): def test_410_heat_stack_create_delete(self): """Create stack, confirm nova compute resource, delete stack.""" - # Create an image for use by the heat template logging.info('Creating glance image ({})...'.format(IMAGE_NAME)) glance_setup.add_cirros_image(image_name=IMAGE_NAME) From 337c83b68dbd86eb7466977f2843b9513f045970 Mon Sep 17 00:00:00 2001 From: Jose Delarosa Date: Wed, 30 Oct 2019 11:32:54 -0500 Subject: [PATCH 161/898] heat: Reduce number of asserts and use helpers when appropriate --- zaza/openstack/charm_tests/heat/tests.py | 79 +++++------------------- 1 file changed, 14 insertions(+), 65 deletions(-) diff --git a/zaza/openstack/charm_tests/heat/tests.py b/zaza/openstack/charm_tests/heat/tests.py index fba3577..040aa02 100644 --- a/zaza/openstack/charm_tests/heat/tests.py +++ b/zaza/openstack/charm_tests/heat/tests.py @@ -21,11 +21,8 @@ import os import subprocess from urllib import parse as urlparse from heatclient.common import template_utils -from novaclient import exceptions import zaza.model -import zaza.openstack.charm_tests.glance.setup as glance_setup -import zaza.openstack.charm_tests.nova.setup as nova_setup import zaza.openstack.charm_tests.nova.utils as nova_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -79,41 +76,21 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): logging.info('Checking default heat resource list...') try: types = self.heat_client.resource_types.list() - if type(types) is list: - logging.info('Resource type list check is ok.') - else: - msg = 'Resource type list is not a list!' - assert False, msg - if len(types) > 0: - logging.info('Resource type list is populated ' - '({}, ok).'.format(len(types))) - else: - msg = 'Resource type list length is zero!' - assert False, msg + self.assertIsInstance(types, list, "Resource type is not a list!") + self.assertGreater(len(types), 0, "Resource type list len is zero") except Exception as e: msg = 'Resource type list failed: {}'.format(e) - assert False, msg + self.fail(msg) def test_410_heat_stack_create_delete(self): """Create stack, confirm nova compute resource, delete stack.""" - # Create an image for use by the heat template - logging.info('Creating glance image ({})...'.format(IMAGE_NAME)) - glance_setup.add_cirros_image(image_name=IMAGE_NAME) - # Verify new image name images_list = list(self.glance_client.images.list()) - if images_list[0].name != IMAGE_NAME: - message = ('glance image create failed or unexpected ' - 'image name {}'.format(images_list[0].name)) - assert False, message - - # Create a keypair - logging.info('Creating keypair {}'.format(nova_utils.KEYPAIR_NAME)) - nova_setup.manage_ssh_key(self.nova_client) + self.assertEqual(images_list[0].name, IMAGE_NAME, + "glance image create failed or unexpected") # Create a heat stack from a heat template, verify its status logging.info('Creating heat stack...') - t_name = 'hot_hello_world.yaml' if (openstack_utils.get_os_release() < openstack_utils.get_os_release('xenial_queens')): @@ -132,14 +109,6 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): t_url = urlparse.urlparse(file_abs_path, scheme='file').geturl() logging.info('template url: {}'.format(t_url)) - # Create flavor - try: - self.nova_client.flavors.find(name=FLAVOR_NAME) - except (exceptions.NotFound, exceptions.NoUniqueMatch): - logging.info('Creating flavor ({})'.format(FLAVOR_NAME)) - self.nova_client.flavors.create(FLAVOR_NAME, ram=512, vcpus=1, - disk=1, flavorid=1) - r_req = self.heat_client.http_client t_files, template = template_utils.get_template_contents(t_url, r_req) env_files, env = template_utils.process_environment_and_files( @@ -168,8 +137,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): except Exception as e: # Generally, an api or cloud config error if this is hit. msg = 'Failed to create heat stack: {}'.format(e) - logging.error(msg) - raise + self.fail(msg) # Confirm stack reaches COMPLETE status. # /!\ Heat stacks reach a COMPLETE status even when nova cannot @@ -189,26 +157,20 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): except Exception as e: # Generally, a resource availability issue if this is hit. msg = 'Failed to get heat stack: {}'.format(e) - assert False, msg + self.fail(msg) # Confirm stack name. logging.info('Expected, actual stack name: {}, ' '{}'.format(STACK_NAME, stack.stack_name)) - if STACK_NAME != stack.stack_name: - msg = 'Stack name mismatch, {} != {}'.format(STACK_NAME, - stack.stack_name) - assert False, msg + self.assertEqual(stack.stack_name, STACK_NAME, + 'Stack name mismatch, ' + '{} != {}'.format(STACK_NAME, stack.stack_name)) # Confirm existence of a heat-generated nova compute resource logging.info('Confirming heat stack resource status...') resource = self.heat_client.resources.get(STACK_NAME, RESOURCE_TYPE) server_id = resource.physical_resource_id - if server_id: - logging.debug('Heat template spawned nova instance, ' - 'ID: {}'.format(server_id)) - else: - msg = 'Stack failed to spawn a nova compute resource (instance).' - logging.error(msg) + self.assertTrue(server_id, "Stack failed to spawn a compute resource.") # Confirm nova instance reaches ACTIVE status openstack_utils.resource_reaches_status(self.nova_client.servers, @@ -222,18 +184,6 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): openstack_utils.delete_resource(self.heat_client.stacks, STACK_NAME, msg="heat stack") - # Delete image - logging.info('Deleting glance image...') - image = self.nova_client.glance.find_image(IMAGE_NAME) - openstack_utils.delete_resource(self.glance_client.images, - image.id, msg="glance image") - - # Delete keypair - logging.info('Deleting keypair...') - openstack_utils.delete_resource(self.nova_client.keypairs, - nova_utils.KEYPAIR_NAME, - msg="nova keypair") - def test_500_auth_encryption_key_same_on_units(self): """Test the auth_encryption_key in heat.conf is same on all units.""" logging.info("Checking the 'auth_encryption_key' is the same on " @@ -252,10 +202,9 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): keys[r['UnitId']] = k # see if keys are different ks = set(keys.values()) - if len(ks) != 1: - msg = ("'auth_encryption_key' is not identical on every unit: {}" - .format("{}={}".format(k, v) for k, v in keys.items())) - assert False, msg + self.assertEqual(len(ks), 1, "'auth_encryption_key' is not identical " + "on every unit: {}".format("{}={}".format(k, v) + for k, v in keys.items())) @staticmethod def _run_arbitrary(command, timeout=300): From 8178607ba934790e758aa4587994e3effe63c272 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 31 Oct 2019 15:37:56 +0000 Subject: [PATCH 162/898] Refactor swift tests and add storage The SwiftImageCreateTest was running a pause/resume test for swift proxy which does not make sense. So, break the proxy specific tests into their own class and add a class for storage. A subsequent change to the swift-proxy charm will be needed to add the new proxy test class to the tests.yaml. --- zaza/openstack/charm_tests/swift/tests.py | 26 +++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index dd2e4e1..0a814ba 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -62,6 +62,8 @@ class SwiftImageCreateTest(test_utils.OpenStackBaseTest): self.assertEqual(image['size'], total_bytes) openstack_utils.delete_image(self.glance_client, image['id']) +class SwiftProxyTests(test_utils.OpenStackBaseTest): + def test_901_pause_resume(self): """Run pause and resume tests. @@ -80,3 +82,27 @@ class SwiftImageCreateTest(test_utils.OpenStackBaseTest): 'diskusage', action_params={}) self.assertEqual(action.status, "completed") + +class SwiftStorageTests(test_utils.OpenStackBaseTest): + + def test_901_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + services = ['swift-account-server', + 'swift-account-auditor', + 'swift-account-reaper', + 'swift-account-replicator', + 'swift-container-server', + 'swift-container-auditor', + 'swift-container-replicator', + 'swift-container-updater', + 'swift-object-server', + 'swift-object-auditor', + 'swift-object-replicator', + 'swift-object-updater', + 'swift-container-sync'] + with self.pause_resume(services): + logging.info("Testing pause resume") From 99b53a2a760dea4a4ba7dc96a11ccf736383f3af Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 31 Oct 2019 15:52:37 +0000 Subject: [PATCH 163/898] Lint fixes --- zaza/openstack/charm_tests/swift/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index 0a814ba..ad2c0e5 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -62,7 +62,9 @@ class SwiftImageCreateTest(test_utils.OpenStackBaseTest): self.assertEqual(image['size'], total_bytes) openstack_utils.delete_image(self.glance_client, image['id']) + class SwiftProxyTests(test_utils.OpenStackBaseTest): + """Tests specific to swift proxy.""" def test_901_pause_resume(self): """Run pause and resume tests. @@ -83,7 +85,9 @@ class SwiftProxyTests(test_utils.OpenStackBaseTest): action_params={}) self.assertEqual(action.status, "completed") + class SwiftStorageTests(test_utils.OpenStackBaseTest): + """Tests specific to swift storage.""" def test_901_pause_resume(self): """Run pause and resume tests. From 85927e20c68c3ad97bf23206cc07ec57150f532c Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 30 Oct 2019 14:14:42 +0100 Subject: [PATCH 164/898] Add helper to get uuids of machines w/OVN chassis Refactor get_machines_for_application as generator function instead of building and passing on static List. --- .../utilities/test_zaza_utilities_juju.py | 15 +++++---- .../test_zaza_utilities_openstack.py | 33 +++++++++++++++++-- zaza/openstack/utilities/juju.py | 19 ++++------- zaza/openstack/utilities/openstack.py | 26 +++++++++++---- 4 files changed, 65 insertions(+), 28 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_juju.py b/unit_tests/utilities/test_zaza_utilities_juju.py index e70bef8..d62534a 100644 --- a/unit_tests/utilities/test_zaza_utilities_juju.py +++ b/unit_tests/utilities/test_zaza_utilities_juju.py @@ -101,8 +101,8 @@ class TestJujuUtils(ut_utils.BaseTestCase): # Machine data self.assertEqual( - juju_utils.get_machines_for_application(self.application), - [self.machine]) + next(juju_utils.get_machines_for_application(self.application)), + self.machine) self.get_application_status.assert_called_once() # Subordinate application has no units @@ -115,9 +115,9 @@ class TestJujuUtils(ut_utils.BaseTestCase): self.get_application_status.side_effect = _get_application_status self.assertEqual( - juju_utils.get_machines_for_application( - self.subordinate_application), - [self.machine]) + next(juju_utils.get_machines_for_application( + self.subordinate_application)), + self.machine) def test_get_unit_name_from_host_name(self): unit_mock1 = mock.MagicMock() @@ -151,8 +151,9 @@ class TestJujuUtils(ut_utils.BaseTestCase): self.get_machines_for_application.return_value = [self.machine] self.assertEqual( - juju_utils.get_machine_uuids_for_application(self.application), - [self.machine_data.get("instance-id")]) + next(juju_utils.get_machine_uuids_for_application( + self.application)), + self.machine_data.get("instance-id")) self.get_machines_for_application.assert_called_once_with( self.application) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 9aa3cde..e27a2c2 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -15,6 +15,7 @@ import copy import datetime import io +import itertools import mock import tenacity @@ -814,12 +815,13 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): ) # No machine returned - self._get_machines.return_value = [] + self._get_machines.side_effect = StopIteration with self.assertRaises(exceptions.ApplicationNotFound): openstack_utils.get_current_os_release_pair() + self._get_machines.side_effect = None # No series returned - self._get_machines.return_value = ['6'] + self._get_machines.return_value = itertools.repeat('6') self._get_machine_series.return_value = None with self.assertRaises(exceptions.SeriesNotFound): openstack_utils.get_current_os_release_pair() @@ -1165,3 +1167,30 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'OS_PROJECT_NAME': 'services'}, scope='PROJECT', verify=None) + + def test_get_gateway_uuids(self): + self.patch_object(openstack_utils.juju_utils, + 'get_machine_uuids_for_application') + self.get_machine_uuids_for_application.return_value = 'ret' + self.assertEquals(openstack_utils.get_gateway_uuids(), 'ret') + self.get_machine_uuids_for_application.assert_called_once_with( + 'neutron-gateway') + + def test_get_ovs_uuids(self): + self.patch_object(openstack_utils.juju_utils, + 'get_machine_uuids_for_application') + self.get_machine_uuids_for_application.return_value = 'ret' + self.assertEquals(openstack_utils.get_ovs_uuids(), 'ret') + self.get_machine_uuids_for_application.assert_called_once_with( + 'neutron-openvswitch') + + def test_get_ovn_uuids(self): + self.patch_object(openstack_utils.juju_utils, + 'get_machine_uuids_for_application') + self.get_machine_uuids_for_application.return_value = ['ret'] + self.assertEquals(list(openstack_utils.get_ovn_uuids()), + ['ret', 'ret']) + self.get_machine_uuids_for_application.assert_has_calls([ + mock.call('ovn-chassis'), + mock.call('ovn-dedicated-chassis'), + ]) diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index 0c46b7e..f6dfa42 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -81,21 +81,18 @@ def get_machines_for_application(application): :param application: Application name :type application: string - :returns: List of machines for an application - :rtype: list + :returns: machines for an application + :rtype: Iterator[str] """ status = get_application_status(application) # libjuju juju status no longer has units for subordinate charms # Use the application it is subordinate-to to find machines if status.get("units") is None and status.get("subordinate-to"): - return get_machines_for_application(status.get("subordinate-to")[0]) + status = get_application_status(status.get("subordinate-to")[0]) - machines = [] for unit in status.get("units").keys(): - machines.append( - status.get("units").get(unit).get("machine")) - return machines + yield status.get("units").get(unit).get("machine") def get_unit_name_from_host_name(host_name, application): @@ -151,13 +148,11 @@ def get_machine_uuids_for_application(application): :param application: Application name :type application: string - :returns: List of machine uuuids for an application - :rtype: list + :returns: machine uuuids for an application + :rtype: Iterator[str] """ - uuids = [] for machine in get_machines_for_application(application): - uuids.append(get_machine_status(machine, key="instance-id")) - return uuids + yield get_machine_status(machine, key="instance-id") def get_provider_type(): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index f8ccd1b..02db487 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -45,6 +45,7 @@ from swiftclient import client as swiftclient import datetime import io +import itertools import juju_wait import logging import os @@ -55,8 +56,8 @@ import subprocess import sys import tempfile import tenacity -import urllib import textwrap +import urllib from zaza import model from zaza.openstack.utilities import ( @@ -434,7 +435,7 @@ def get_gateway_uuids(): """Return machine uuids for neutron-gateway(s). :returns: List of uuids - :rtype: list + :rtype: Iterator[str] """ return juju_utils.get_machine_uuids_for_application('neutron-gateway') @@ -443,12 +444,24 @@ def get_ovs_uuids(): """Return machine uuids for neutron-openvswitch(s). :returns: List of uuids - :rtype: list + :rtype: Iterator[str] """ return (juju_utils .get_machine_uuids_for_application('neutron-openvswitch')) +def get_ovn_uuids(): + """Provide machine uuids for OVN Chassis. + + :returns: List of uuids + :rtype: Iterator[str] + """ + return itertools.chain( + juju_utils.get_machine_uuids_for_application('ovn-chassis'), + juju_utils.get_machine_uuids_for_application('ovn-dedicated-chassis'), + ) + + BRIDGE_MAPPINGS = 'bridge-mappings' NEW_STYLE_NETWORKING = 'physnet1:br-ex' @@ -1358,10 +1371,9 @@ def get_current_os_release_pair(application='keystone'): :raises: exceptions.SeriesNotFound :raises: exceptions.OSVersionNotFound """ - machines = juju_utils.get_machines_for_application(application) - if len(machines) >= 1: - machine = machines[0] - else: + try: + machine = next(juju_utils.get_machines_for_application(application)) + except StopIteration: raise exceptions.ApplicationNotFound(application) series = juju_utils.get_machine_series(machine) From 7d1d04a7423ab1650b5124311ad21d2ca9662544 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 30 Oct 2019 15:19:20 +0100 Subject: [PATCH 165/898] Deduce whether DVR is enabled from neutron-api config Add helper to detect presence of OVN in deployment --- .../test_zaza_utilities_openstack.py | 13 +++- zaza/openstack/charm_tests/neutron/setup.py | 5 -- zaza/openstack/configure/network.py | 2 - zaza/openstack/utilities/openstack.py | 61 +++++++++++-------- 4 files changed, 48 insertions(+), 33 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index e27a2c2..6e00ff0 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -160,7 +160,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): # Already exists network = openstack_utils.create_external_network( - self.neutronclient, self.project_id, False) + self.neutronclient, self.project_id) self.assertEqual(network, self.network["network"]) self.neutronclient.create_network.assert_not_called() @@ -170,7 +170,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): network_msg = copy.deepcopy(self.network) network_msg["network"].pop("id") network = openstack_utils.create_external_network( - self.neutronclient, self.project_id, False) + self.neutronclient, self.project_id) self.assertEqual(network, self.network["network"]) self.neutronclient.create_network.assert_called_once_with( network_msg) @@ -1194,3 +1194,12 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): mock.call('ovn-chassis'), mock.call('ovn-dedicated-chassis'), ]) + + def test_dvr_enabled(self): + self.patch_object(openstack_utils, 'get_application_config_option') + openstack_utils.dvr_enabled() + self.get_application_config_option.assert_called_once_with( + 'neutron-api', 'enable-dvr') + + def test_ovn_present(self): + pass diff --git a/zaza/openstack/charm_tests/neutron/setup.py b/zaza/openstack/charm_tests/neutron/setup.py index edacd22..444ce8b 100644 --- a/zaza/openstack/charm_tests/neutron/setup.py +++ b/zaza/openstack/charm_tests/neutron/setup.py @@ -25,7 +25,6 @@ from zaza.openstack.utilities import ( juju as juju_utils, openstack as openstack_utils, ) -import zaza.model as model # The overcloud network configuration settings are declared. @@ -74,10 +73,6 @@ def basic_overcloud_network(): network_config.update(DEFAULT_UNDERCLOUD_NETWORK_CONFIG) # Environment specific settings network_config.update(generic_utils.get_undercloud_env_vars()) - # Deployed model settings - if (model.get_application_config('neutron-api') - .get('enable-dvr').get('value')): - network_config.update({"dvr_enabled": True}) # Get keystone session keystone_session = openstack_utils.get_overcloud_keystone_session() diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 55bcdf3..7b8f0a9 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -129,7 +129,6 @@ def setup_sdn(network_config, keystone_session=None): ext_network = openstack_utils.create_external_network( neutron_client, project_id, - network_config.get("dvr_enabled", False), network_config["external_net_name"]) openstack_utils.create_external_subnet( neutron_client, @@ -226,7 +225,6 @@ def setup_gateway_ext_port(network_config, keystone_session=None): openstack_utils.configure_gateway_ext_port( nova_client, neutron_client, - dvr_mode=network_config.get("dvr_enabled", False), net_id=net_id, add_dataport_to_netplan=add_dataport_to_netplan) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 02db487..3774493 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -462,20 +462,44 @@ def get_ovn_uuids(): ) +def dvr_enabled(): + """Check whether DVR is enabled in deployment. + + :returns: True when DVR is enabled, False otherwise + :rtype: bool + """ + return get_application_config_option('neutron-api', 'enable-dvr') + + +def ovn_present(): + """Check whether OVN is present in deployment. + + :returns: True when OVN is present, False otherwise + :rtype: bool + """ + try: + for name in ('ovn-chassis', 'ovn-dedicated-chassis'): + model.get_application(name) + return True + except KeyError: + return False + else: + raise RuntimeError('Unable to determine whether or not OVN is present ' + 'in deployment') + + BRIDGE_MAPPINGS = 'bridge-mappings' NEW_STYLE_NETWORKING = 'physnet1:br-ex' -def deprecated_external_networking(dvr_mode=False): +def deprecated_external_networking(): """Determine whether deprecated external network mode is in use. - :param dvr_mode: Using DVR mode or not - :type dvr_mode: boolean :returns: True or False :rtype: boolean """ bridge_mappings = None - if dvr_mode: + if dvr_enabled(): bridge_mappings = get_application_config_option('neutron-openvswitch', BRIDGE_MAPPINGS) else: @@ -514,17 +538,15 @@ def get_admin_net(neutron_client): return net -def add_interface_to_netplan(server_name, mac_address, dvr_mode=None): +def add_interface_to_netplan(server_name, mac_address): """In guest server_name, add nic with mac_address to netplan. :param server_name: Hostname of instance :type server_name: string :param mac_address: mac address of nic to be added to netplan :type mac_address: string - :param dvr_mode: Using DVR mode or not - :type dvr_mode: boolean """ - if dvr_mode: + if dvr_enabled(): application_name = 'neutron-openvswitch' else: application_name = 'neutron-gateway' @@ -572,8 +594,7 @@ def add_interface_to_netplan(server_name, mac_address, dvr_mode=None): model.run_on_unit(unit_name, "sudo netplan apply") -def configure_gateway_ext_port(novaclient, neutronclient, - dvr_mode=None, net_id=None, +def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, add_dataport_to_netplan=False): """Configure the neturong-gateway external port. @@ -581,20 +602,20 @@ def configure_gateway_ext_port(novaclient, neutronclient, :type novaclient: novaclient.Client object :param neutronclient: Authenticated neutronclient :type neutronclient: neutronclient.Client object - :param dvr_mode: Using DVR mode or not - :type dvr_mode: boolean :param net_id: Network ID :type net_id: string """ - if dvr_mode: + if dvr_enabled(): uuids = get_ovs_uuids() # If dvr, do not attempt to persist nic in netplan # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78 add_dataport_to_netplan = False + application_name = 'neutron-openvswitch' else: uuids = get_gateway_uuids() + application_name = 'neutron-gateway' - deprecated_extnet_mode = deprecated_external_networking(dvr_mode) + deprecated_extnet_mode = deprecated_external_networking() config_key = 'data-port' if deprecated_extnet_mode: @@ -627,8 +648,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, if add_dataport_to_netplan: mac_address = get_mac_from_port(port, neutronclient) add_interface_to_netplan(server.name, - mac_address=mac_address, - dvr_mode=dvr_mode) + mac_address=mac_address) ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: if 'ext-port' in port['name']: @@ -638,10 +658,6 @@ def configure_gateway_ext_port(novaclient, neutronclient, ext_br_macs.append('br-ex:{}'.format(port['mac_address'])) ext_br_macs.sort() ext_br_macs_str = ' '.join(ext_br_macs) - if dvr_mode: - application_name = 'neutron-openvswitch' - else: - application_name = 'neutron-gateway' if ext_br_macs: logging.info('Setting {} on {} external port to {}'.format( @@ -715,16 +731,13 @@ def create_project_network(neutron_client, project_id, net_name='private', return network -def create_external_network(neutron_client, project_id, dvr_mode, - net_name='ext_net'): +def create_external_network(neutron_client, project_id, net_name='ext_net'): """Create the external network. :param neutron_client: Authenticated neutronclient :type neutron_client: neutronclient.Client object :param project_id: Project ID :type project_id: string - :param dvr_mode: Using DVR mode or not - :type dvr_mode: boolean :param net_name: Network name :type net_name: string :returns: Network object From 0163f79ec216857e9b9a4afe006d7447abdfe8a4 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 31 Oct 2019 15:17:10 +0100 Subject: [PATCH 166/898] Teach ``configure_gateway_ext_port()`` about OVN Also stop using ``juju_wait``. --- zaza/openstack/utilities/juju.py | 2 + zaza/openstack/utilities/openstack.py | 64 ++++++++++++++++++--------- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index f6dfa42..a5edba2 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -85,6 +85,8 @@ def get_machines_for_application(application): :rtype: Iterator[str] """ status = get_application_status(application) + if not status: + raise StopIteration # libjuju juju status no longer has units for subordinate charms # Use the application it is subordinate-to to find machines diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 3774493..22a2f27 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -46,7 +46,6 @@ from swiftclient import client as swiftclient import datetime import io import itertools -import juju_wait import logging import os import paramiko @@ -59,6 +58,7 @@ import tenacity import textwrap import urllib +import zaza from zaza import model from zaza.openstack.utilities import ( exceptions, @@ -502,6 +502,8 @@ def deprecated_external_networking(): if dvr_enabled(): bridge_mappings = get_application_config_option('neutron-openvswitch', BRIDGE_MAPPINGS) + elif ovn_present(): + return False else: bridge_mappings = get_application_config_option('neutron-gateway', BRIDGE_MAPPINGS) @@ -605,21 +607,35 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, :param net_id: Network ID :type net_id: string """ + deprecated_extnet_mode = deprecated_external_networking() + + port_config_key = 'data-port' + if deprecated_extnet_mode: + port_config_key = 'ext-port' + + config = {} if dvr_enabled(): uuids = get_ovs_uuids() # If dvr, do not attempt to persist nic in netplan # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78 add_dataport_to_netplan = False - application_name = 'neutron-openvswitch' + application_names = ['neutron-openvswitch'] + elif ovn_present(): + uuids = get_ovn_uuids() + application_names = ['ovn-chassis'] + try: + ovn_dc_name = 'ovn-dedicated-chassis' + next(juju_utils.get_machine_uuids_for_application(ovn_dc_name)) + application_names.append(ovn_dc_name) + except StopIteration: + # ovn-dedicated-chassis not in deployment + pass + port_config_key = 'interface-bridge-mappings' + config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) + add_dataport_to_netplan = False else: uuids = get_gateway_uuids() - application_name = 'neutron-gateway' - - deprecated_extnet_mode = deprecated_external_networking() - - config_key = 'data-port' - if deprecated_extnet_mode: - config_key = 'ext-port' + application_names = ['neutron-gateway'] if not net_id: net_id = get_admin_net(neutronclient)['id'] @@ -654,23 +670,31 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, if 'ext-port' in port['name']: if deprecated_extnet_mode: ext_br_macs.append(port['mac_address']) + elif ovn_present(): + ext_br_macs.append('{}:br-ex'.format(port['mac_address'])) else: ext_br_macs.append('br-ex:{}'.format(port['mac_address'])) ext_br_macs.sort() ext_br_macs_str = ' '.join(ext_br_macs) if ext_br_macs: - logging.info('Setting {} on {} external port to {}'.format( - config_key, application_name, ext_br_macs_str)) - current_data_port = get_application_config_option(application_name, - config_key) - if current_data_port == ext_br_macs_str: - logging.info('Config already set to value') - return - model.set_application_config( - application_name, - configuration={config_key: ext_br_macs_str}) - juju_wait.wait(wait_for_workload=True) + config.update({port_config_key: ext_br_macs_str}) + for application_name in application_names: + logging.info('Setting {} on {}'.format( + config, application_name)) + current_data_port = get_application_config_option(application_name, + port_config_key) + if current_data_port == ext_br_macs_str: + logging.info('Config already set to value') + return + + model.set_application_config( + application_name, + configuration=config) + zaza.model.wait_for_agent_status() + test_config = zaza.charm_lifecycle.utils.get_charm_config() + zaza.model.wait_for_application_states( + states=test_config.get('target_deploy_status', {})) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), From 8e8544bab7cea7c401a6144071bb94cd13c46f4d Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sat, 2 Nov 2019 10:31:30 +0100 Subject: [PATCH 167/898] Fix ovn_present for ovn-dedicated-chassis Add missing unit test. --- .../utilities/test_zaza_utilities_openstack.py | 8 +++++++- zaza/openstack/utilities/openstack.py | 15 +++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 6e00ff0..4576564 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1202,4 +1202,10 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'neutron-api', 'enable-dvr') def test_ovn_present(self): - pass + self.patch_object(openstack_utils.model, 'get_application') + self.get_application.side_effect = [None, KeyError] + self.assertTrue(openstack_utils.ovn_present()) + self.get_application.side_effect = [KeyError, None] + self.assertTrue(openstack_utils.ovn_present()) + self.get_application.side_effect = [KeyError, KeyError] + self.assertFalse(openstack_utils.ovn_present()) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 22a2f27..959cb09 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -477,15 +477,14 @@ def ovn_present(): :returns: True when OVN is present, False otherwise :rtype: bool """ - try: - for name in ('ovn-chassis', 'ovn-dedicated-chassis'): + app_presence = [] + for name in ('ovn-chassis', 'ovn-dedicated-chassis'): + try: model.get_application(name) - return True - except KeyError: - return False - else: - raise RuntimeError('Unable to determine whether or not OVN is present ' - 'in deployment') + app_presence.append(True) + except KeyError: + app_presence.append(False) + return any(app_presence) BRIDGE_MAPPINGS = 'bridge-mappings' From 675b109d54d20062e4590b1f6f2293f06d6d8b50 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sat, 2 Nov 2019 17:16:39 +0100 Subject: [PATCH 168/898] Optionally limit number of units to get port attached This is useful for validating deployments with OVN where it is not required to have external networking attached to every chassis. Any chassis that does not have external networking directly attached will forward traffic destined for the external network through a tunnel to a chassis that does. --- zaza/openstack/charm_tests/neutron/setup.py | 21 +++++++++++++++++++-- zaza/openstack/configure/network.py | 8 ++++++-- zaza/openstack/utilities/openstack.py | 11 +++++++---- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/setup.py b/zaza/openstack/charm_tests/neutron/setup.py index 444ce8b..e6f19a5 100644 --- a/zaza/openstack/charm_tests/neutron/setup.py +++ b/zaza/openstack/charm_tests/neutron/setup.py @@ -16,6 +16,8 @@ """Setup for Neutron deployments.""" +import functools + from zaza.openstack.configure import ( network, ) @@ -56,12 +58,14 @@ DEFAULT_UNDERCLOUD_NETWORK_CONFIG = { } -def basic_overcloud_network(): +def basic_overcloud_network(limit_gws=None): """Run setup for neutron networking. Configure the following: The overcloud network using subnet pools + :param limit_gws: Limit the number of gateways that get a port attached + :type limit_gws: int """ cli_utils.setup_logging() @@ -81,7 +85,20 @@ def basic_overcloud_network(): if juju_utils.get_provider_type() == "openstack": undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session() network.setup_gateway_ext_port(network_config, - keystone_session=undercloud_ks_sess) + keystone_session=undercloud_ks_sess, + limit_gws=None) # Confugre the overcloud network network.setup_sdn(network_config, keystone_session=keystone_session) + + +# Configure function to get one gateway with external network +overcloud_network_one_gw = functools.partial( + basic_overcloud_network, + limit_gws=1) + + +# Configure function to get two gateways with external network +overcloud_network_two_gws = functools.partial( + basic_overcloud_network, + limit_gws=2) diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 7b8f0a9..12b8d9e 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -183,7 +183,8 @@ def setup_sdn(network_config, keystone_session=None): openstack_utils.add_neutron_secgroup_rules(neutron_client, project_id) -def setup_gateway_ext_port(network_config, keystone_session=None): +def setup_gateway_ext_port(network_config, keystone_session=None, + limit_gws=None): """Perform setup external port on Neutron Gateway. For OpenStack on OpenStack scenarios. @@ -192,6 +193,8 @@ def setup_gateway_ext_port(network_config, keystone_session=None): :type network_config: dict :param keystone_session: Keystone session object for undercloud :type keystone_session: keystoneauth1.session.Session object + :param limit_gws: Limit the number of gateways that get a port attached + :type limit_gws: Optional[int] :returns: None :rtype: None """ @@ -226,7 +229,8 @@ def setup_gateway_ext_port(network_config, keystone_session=None): nova_client, neutron_client, net_id=net_id, - add_dataport_to_netplan=add_dataport_to_netplan) + add_dataport_to_netplan=add_dataport_to_netplan, + limit_gws=limit_gws) def run_from_cli(**kwargs): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 959cb09..7869443 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -596,7 +596,8 @@ def add_interface_to_netplan(server_name, mac_address): def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, - add_dataport_to_netplan=False): + add_dataport_to_netplan=False, + limit_gws=None): """Configure the neturong-gateway external port. :param novaclient: Authenticated novaclient @@ -605,6 +606,8 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, :type neutronclient: neutronclient.Client object :param net_id: Network ID :type net_id: string + :param limit_gws: Limit the number of gateways that get a port attached + :type limit_gws: Optional[int] """ deprecated_extnet_mode = deprecated_external_networking() @@ -614,13 +617,13 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, config = {} if dvr_enabled(): - uuids = get_ovs_uuids() + uuids = itertools.islice(get_ovs_uuids(), limit_gws) # If dvr, do not attempt to persist nic in netplan # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78 add_dataport_to_netplan = False application_names = ['neutron-openvswitch'] elif ovn_present(): - uuids = get_ovn_uuids() + uuids = itertools.islice(get_ovn_uuids(), limit_gws) application_names = ['ovn-chassis'] try: ovn_dc_name = 'ovn-dedicated-chassis' @@ -633,7 +636,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) add_dataport_to_netplan = False else: - uuids = get_gateway_uuids() + uuids = itertools.islice(get_gateway_uuids(), limit_gws) application_names = ['neutron-gateway'] if not net_id: From 63fb045f024f11f1b6518674ec7e2c1dc9ab60f2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 4 Nov 2019 11:35:26 +0000 Subject: [PATCH 169/898] Teach zaza about designate version --- zaza/openstack/utilities/openstack.py | 5 +++++ zaza/openstack/utilities/os_versions.py | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 442c878..3b25539 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -102,6 +102,10 @@ CHARM_TYPES = { 'pkg': 'ceilometer-common', 'origin_setting': 'openstack-origin' }, + 'designate': { + 'pkg': 'designate-common', + 'origin_setting': 'openstack-origin' + }, } UPGRADE_SERVICES = [ {'name': 'keystone', 'type': CHARM_TYPES['keystone']}, @@ -114,6 +118,7 @@ UPGRADE_SERVICES = [ {'name': 'openstack-dashboard', 'type': CHARM_TYPES['openstack-dashboard']}, {'name': 'ceilometer', 'type': CHARM_TYPES['ceilometer']}, + {'name': 'designate', 'type': CHARM_TYPES['designate']}, ] diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index cd81a84..eb5093c 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -205,4 +205,15 @@ PACKAGE_CODENAMES = { ('15', 'stein'), ('16', 'train'), ]), + 'designate-common': OrderedDict([ + ('1', 'liberty'), + ('2', 'mitaka'), + ('3', 'newton'), + ('4', 'ocata'), + ('5', 'pike'), + ('6', 'queens'), + ('7', 'rocky'), + ('8', 'stein'), + ('9', 'train'), + ]), } From 8d676bf9fa4446ddd6eaa9ae30a9845bdf775dfe Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 5 Nov 2019 15:21:21 +0100 Subject: [PATCH 170/898] Make non-existent tests/tests.yaml non-fatal where possible Not all test environments or runners are equal, don't crash on non-existent test config. --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- zaza/openstack/charm_tests/test_utils.py | 2 +- zaza/openstack/charm_tests/vault/setup.py | 2 +- zaza/openstack/utilities/openstack.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 1e11845..04b511e 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -344,7 +344,7 @@ class PerconaClusterColdStartTest(PerconaClusterTest): logging.debug("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") - test_config = lifecycle_utils.get_charm_config() + test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index e054038..eff6b41 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -98,7 +98,7 @@ class OpenStackBaseTest(unittest.TestCase): """Run setup for test class to create common resourcea.""" cls.keystone_session = openstack_utils.get_overcloud_keystone_session() cls.model_name = model.get_juju_model() - cls.test_config = lifecycle_utils.get_charm_config() + cls.test_config = lifecycle_utils.get_charm_config(fatal=False) if application_name: cls.application_name = application_name else: diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index efb3a8d..03b8f65 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -84,7 +84,7 @@ def auto_initialize(cacert=None, validation_application='keystone'): allowed_domains='openstack.local') zaza.model.wait_for_agent_status() - test_config = lifecycle_utils.get_charm_config() + test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index bff8da7..116bfed 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -713,7 +713,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, application_name, configuration=config) zaza.model.wait_for_agent_status() - test_config = zaza.charm_lifecycle.utils.get_charm_config() + test_config = zaza.charm_lifecycle.utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) From b95f6e6e7ddd552822e92b503a7c140b51d754ab Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 6 Nov 2019 11:40:20 +0000 Subject: [PATCH 171/898] Add heat policy.d functional tests --- zaza/openstack/charm_tests/policyd/tests.py | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 8779149..c359695 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -41,6 +41,7 @@ import zipfile import keystoneauth1 import glanceclient.common.exceptions import cinderclient.exceptions +import heatclient.exc import zaza.model as zaza_model @@ -559,3 +560,35 @@ class CinderTests(BasePolicydSpecialization): cinder_client.volumes.list() except cinderclient.exceptions.Forbidden: raise PolicydOperationFailedException() + + +class HeatTests(BasePolicydSpecialization): + """Test the policyd override using the heat client.""" + + _rule = "{'stacks:index': '!'}" + + @classmethod + def setUpClass(cls, application_name=None): + """Run class setup for running HeatTests charm operation tests.""" + super(HeatTests, cls).setUpClass(application_name="heat") + cls.application_name = "heat" + + def get_client_and_attempt_operation(self, ip): + """Attempt to list the heat stacks as a policyd override. + + This operation should pass normally, and fail when + the rule has been overriden (see the `rule` class variable). + + :param ip: the IP address to get the session against. + :type ip: str + :raises: PolicydOperationFailedException if operation fails. + """ + heat_client = openstack_utils.get_heat_session_client( + self.get_keystone_session_admin_user(ip)) + try: + # stacks.list() returns a generator (as opposed to a list), so to + # force the client to actually connect, the generator has to be + # iterated. + list(heat_client.stacks.list()) + except heatclient.exc.HTTPForbidden: + raise PolicydOperationFailedException() From 8aced1c5b24cd5a1f06362146ef58151a8f4ee3d Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 6 Nov 2019 12:59:31 +0100 Subject: [PATCH 172/898] net: Revert switch to wait_for_application_states We are stuck with juju_wait until we figure out how to deal with all the non ['active', 'idle', 'Unit is ready.'] workload/agent states and msgs that our mojo specs are exposed to. --- zaza/openstack/utilities/openstack.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 116bfed..8608e97 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -47,6 +47,7 @@ from swiftclient import client as swiftclient import datetime import io import itertools +import juju_wait import logging import os import paramiko @@ -59,7 +60,6 @@ import tenacity import textwrap import urllib -import zaza from zaza import model from zaza.openstack.utilities import ( exceptions, @@ -712,10 +712,10 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, model.set_application_config( application_name, configuration=config) - zaza.model.wait_for_agent_status() - test_config = zaza.charm_lifecycle.utils.get_charm_config(fatal=False) - zaza.model.wait_for_application_states( - states=test_config.get('target_deploy_status', {})) + # NOTE(fnordahl): We are stuck with juju_wait until we figure out how + # to deal with all the non ['active', 'idle', 'Unit is ready.'] + # workload/agent states and msgs that our mojo specs are exposed to. + juju_wait.wait(wait_for_workload=True) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), From 3437f8bc026c918d5e60c62624f09a5ce5375576 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 Nov 2019 14:58:39 -0500 Subject: [PATCH 173/898] tmp --- zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py index fa664c3..ed7c8e1 100644 --- a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py @@ -205,7 +205,7 @@ class CephRBDMirrorTest(CephRBDMirrorBase): # # We do not use tenacity here as it will interfere with tenacity used # in ``resource_reaches_status`` - def create_volume_from_image(cinder, image, retry=5): + def create_volume_from_image(cinder, image, retry=20): if retry < 1: return volume = cinder.volumes.create(8, name='zaza', imageRef=image.id) From 25b3932ce907166bd65c0a43b348c95fd294539f Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 1 Nov 2019 13:27:07 -0700 Subject: [PATCH 174/898] MySQL 8 zaza tests * Re-arrange the MySQL, Percona-cluster classes to avoid code duplication * cluster-status action --- zaza/openstack/charm_tests/mysql/tests.py | 196 ++++++++++++---------- 1 file changed, 110 insertions(+), 86 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 1e11845..cad31f4 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -14,6 +14,7 @@ """MySQL/Percona Cluster Testing.""" +import json import logging import os import re @@ -27,31 +28,19 @@ import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.utilities.generic as generic_utils -class MySQLTest(test_utils.OpenStackBaseTest): +class MySQLBaseTest(test_utils.OpenStackBaseTest): """Base for mysql charm tests.""" @classmethod def setUpClass(cls): """Run class setup for running mysql tests.""" - super(MySQLTest, cls).setUpClass() + super(MySQLBaseTest, cls).setUpClass() cls.application = "mysql" cls.services = ["mysqld"] - - -class PerconaClusterTest(test_utils.OpenStackBaseTest): - """Base for percona-cluster charm tests.""" - - @classmethod - def setUpClass(cls): - """Run class setup for running percona-cluster tests.""" - super(PerconaClusterTest, cls).setUpClass() - cls.application = "percona-cluster" - # This is the service pidof will attempt to find - # rather than what systemctl uses - cls.services = ["mysqld"] - cls.vip = os.environ.get("OS_VIP00") cls.leader = None cls.non_leaders = [] + # Config file affected by juju set config change + cls.conf_file = "/etc/mysql/mysql.conf.d/mysqld.cnf" def get_root_password(self): """Get the MySQL root password. @@ -63,6 +52,75 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): self.application, "leader-get root-password")["Stdout"].strip() + def update_leaders_and_non_leaders(self): + """Get leader node and non-leader nodes of percona. + + Update and set on the object the leader node and list of non-leader + nodes. + + :returns: None + :rtype: None + """ + status = zaza.model.get_status().applications[self.application] + # Reset + self.leader = None + self.non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + self.leader = unit + else: + self.non_leaders.append(unit) + + +class MySQLCommonTests(MySQLBaseTest): + """Common mysql charm tests.""" + + def test_910_restart_on_config_change(self): + """Checking restart happens on config change. + + Change disk format and assert then change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + set_default = {"max-connections": "600"} + set_alternate = {"max-connections": "1000"} + + # Make config change, check for service restarts + logging.debug("Setting peer timeout ...") + self.restart_on_changed( + self.conf_file, + set_default, + set_alternate, + {}, {}, + self.services) + logging.info("Passed restart on changed test.") + + def test_920_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + with self.pause_resume(self.services): + logging.info("Testing pause resume") + logging.info("Passed pause and resume test.") + + +class PerconaClusterBaseTest(MySQLBaseTest): + """Base for percona-cluster charm tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running percona-cluster tests.""" + super().setUpClass() + cls.application = "percona-cluster" + # This is the service pidof will attempt to find + # rather than what systemctl uses + cls.services = ["mysqld"] + cls.vip = os.environ.get("TEST_VIP00") + # Config file affected by juju set config change + cls.conf_file = "/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf" + def get_wsrep_value(self, attr): """Get wsrrep value from the DB. @@ -123,39 +181,13 @@ class PerconaClusterTest(test_utils.OpenStackBaseTest): ) return unit.entity_id - def update_leaders_and_non_leaders(self): - """Get leader node and non-leader nodes of percona. - Update and set on the object the leader node and list of non-leader - nodes. - - :returns: None - :rtype: None - """ - status = zaza.model.get_status().applications[self.application] - # Reset - self.leader = None - self.non_leaders = [] - for unit in status["units"]: - if status["units"][unit].get("leader"): - self.leader = unit - else: - self.non_leaders.append(unit) - - -class PerconaClusterCharmTests(PerconaClusterTest): - """Base for percona-cluster charm tests. +class PerconaClusterCharmTests(MySQLCommonTests, PerconaClusterBaseTest): + """Percona-cluster charm tests. .. note:: these have tests have been ported from amulet tests """ - @classmethod - def setUpClass(cls): - """Run class setup for running percona-cluster tests.""" - super(PerconaClusterTest, cls).setUpClass() - cls.application = "percona-cluster" - cls.services = ["mysqld"] - def test_100_bootstrapped_and_clustered(self): """Ensure PXC is bootstrapped and that peer units are clustered.""" self.units = zaza.model.get_application_config( @@ -170,38 +202,6 @@ class PerconaClusterCharmTests(PerconaClusterTest): " (wanted=%s, cluster_size=%s)" % (self.units, cluster_size)) assert cluster_size >= self.units, msg - def test_110_restart_on_config_change(self): - """Checking restart happens on config change. - - Change disk format and assert then change propagates to the correct - file and that services are restarted as a result - """ - # Expected default and alternate values - set_default = {"peer-timeout": "PT3S"} - set_alternate = {"peer-timeout": "PT15S"} - - # Config file affected by juju set config change - conf_file = "/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf" - - # Make config change, check for service restarts - logging.debug("Setting peer timeout ...") - self.restart_on_changed( - conf_file, - set_default, - set_alternate, - {}, {}, - self.services) - logging.info("Passed restart on changed") - - def test_120_pause_resume(self): - """Run pause and resume tests. - - Pause service and check services are stopped then resume and check - they are started - """ - with self.pause_resume(self.services): - logging.info("Testing pause resume") - def test_130_change_root_password(self): """Change root password. @@ -232,7 +232,7 @@ class PerconaClusterCharmTests(PerconaClusterTest): assert code == "0", output -class PerconaClusterColdStartTest(PerconaClusterTest): +class PerconaClusterColdStartTest(PerconaClusterBaseTest): """Percona Cluster cold start tests.""" @classmethod @@ -349,17 +349,9 @@ class PerconaClusterColdStartTest(PerconaClusterTest): states=test_config.get("target_deploy_status", {})) -class PerconaClusterScaleTests(PerconaClusterTest): +class PerconaClusterScaleTests(PerconaClusterBaseTest): """Percona Cluster scale tests.""" - @classmethod - def setUpClass(cls): - """Run class setup for running percona scale tests. - - .. note:: these have tests have been ported from amulet tests - """ - super(PerconaClusterScaleTests, cls).setUpClass() - def test_100_kill_crm_master(self): """Ensure VIP failover. @@ -398,3 +390,35 @@ class PerconaClusterScaleTests(PerconaClusterTest): # always true. assert generic_utils.is_port_open("3306", self.vip), \ "Cannot connect to vip" + + +class MySQLInnoDBClusterTests(MySQLCommonTests): + """Mysql-innodb-cluster charm tests. + + Note: The restart on changed and pause/resume tests also validate the + changing of the R/W primary. On each mysqld shutodown a new R/W primary is + elected automatically by MySQL. + """ + + @classmethod + def setUpClass(cls): + """Run class setup for running mysql-innodb-cluster tests.""" + super().setUpClass() + cls.application = "mysql-innodb-cluster" + + def test_100_cluster_status(self): + """Checking cluster status. + + Run the cluster-status action. + """ + # Update which node is the leader and which are not + self.update_leaders_and_non_leaders() + logging.info("Execute cluster-status action") + action = zaza.model.run_action( + self.non_leaders[0], + "cluster-status", + action_params={}) + cluster_status = json.loads(action.data["results"]["cluster-status"]) + assert "OK" in cluster_status["defaultReplicaSet"]["status"], ( + "Cluster status action failed.") + logging.info("Passed cluster-status action test.") From 22614a3099e5bff0b8270b45c126b8aa81fc2879 Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 6 Nov 2019 15:35:56 -0800 Subject: [PATCH 175/898] Get leaders and non-leaders --- zaza/openstack/charm_tests/mysql/tests.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index cad31f4..dc2366a 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -37,8 +37,6 @@ class MySQLBaseTest(test_utils.OpenStackBaseTest): super(MySQLBaseTest, cls).setUpClass() cls.application = "mysql" cls.services = ["mysqld"] - cls.leader = None - cls.non_leaders = [] # Config file affected by juju set config change cls.conf_file = "/etc/mysql/mysql.conf.d/mysqld.cnf" @@ -52,7 +50,7 @@ class MySQLBaseTest(test_utils.OpenStackBaseTest): self.application, "leader-get root-password")["Stdout"].strip() - def update_leaders_and_non_leaders(self): + def get_leaders_and_non_leaders(self): """Get leader node and non-leader nodes of percona. Update and set on the object the leader node and list of non-leader @@ -70,6 +68,7 @@ class MySQLBaseTest(test_utils.OpenStackBaseTest): self.leader = unit else: self.non_leaders.append(unit) + return self.leader, self.non_leaders class MySQLCommonTests(MySQLBaseTest): @@ -317,14 +316,14 @@ class PerconaClusterColdStartTest(PerconaClusterBaseTest): zaza.model.wait_for_application_states(states=states) # Update which node is the leader and which are not - self.update_leaders_and_non_leaders() + _leader, _non_leaders = self.get_leaders_and_non_leaders() # We want to test the worst possible scenario which is the # non-leader with the highest sequence number. We will use the leader # for the notify-bootstrapped after. They just need to be different # units. logging.info("Execute bootstrap-pxc action after cold boot ...") zaza.model.run_action( - self.non_leaders[0], + _non_leaders[0], "bootstrap-pxc", action_params={}) logging.debug("Wait for application states ...") @@ -412,10 +411,10 @@ class MySQLInnoDBClusterTests(MySQLCommonTests): Run the cluster-status action. """ # Update which node is the leader and which are not - self.update_leaders_and_non_leaders() + _leaders, _non_leaders = self.get_leaders_and_non_leaders() logging.info("Execute cluster-status action") action = zaza.model.run_action( - self.non_leaders[0], + _non_leaders[0], "cluster-status", action_params={}) cluster_status = json.loads(action.data["results"]["cluster-status"]) From 6d53e6d758a71da8a32863e4269c8ed77bc0c51d Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 Nov 2019 20:34:51 -0500 Subject: [PATCH 176/898] Increase wait attempts for volume availability Increase wait attempts for availability of volume created from image in CephRBDMirrorTest. This is taking longer as of Nautilus due to switch to using juju storage backed by undercloud cinder taking longer than prior method of directory-backed OSD devices. --- zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py index ed7c8e1..16ebfc0 100644 --- a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py @@ -210,8 +210,13 @@ class CephRBDMirrorTest(CephRBDMirrorBase): return volume = cinder.volumes.create(8, name='zaza', imageRef=image.id) try: + # Note(coreycb): stop_after_attempt is increased because using + # juju storage for ceph-osd backed by cinder on undercloud + # takes longer than the prior method of directory-backed OSD + # devices. openstack.resource_reaches_status( - cinder.volumes, volume.id, msg='volume') + cinder.volumes, volume.id, msg='volume', + stop_after_attempt=20) return volume except AssertionError: logging.info('retrying') From ce0eb7ab12c259849948205344d10d2af77d26b4 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 7 Nov 2019 09:40:01 +0800 Subject: [PATCH 177/898] Add functional tests for Manila with Ganesha Add a functional gate where we mount a share on two VMs in the cloud under test to validate that manila + ganesha shares work correctly. --- requirements.txt | 1 + .../charm_tests/manila_ganesha/__init__.py | 18 +++ .../charm_tests/manila_ganesha/setup.py | 43 +++++++ .../charm_tests/manila_ganesha/tests.py | 118 ++++++++++++++++++ 4 files changed, 180 insertions(+) create mode 100644 zaza/openstack/charm_tests/manila_ganesha/__init__.py create mode 100644 zaza/openstack/charm_tests/manila_ganesha/setup.py create mode 100644 zaza/openstack/charm_tests/manila_ganesha/tests.py diff --git a/requirements.txt b/requirements.txt index 1ee936a..5d27b15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,6 +27,7 @@ python-cinderclient python-glanceclient python-heatclient python-keystoneclient +python-manilaclient python-neutronclient python-novaclient python-octaviaclient diff --git a/zaza/openstack/charm_tests/manila_ganesha/__init__.py b/zaza/openstack/charm_tests/manila_ganesha/__init__.py new file mode 100644 index 0000000..d9238d6 --- /dev/null +++ b/zaza/openstack/charm_tests/manila_ganesha/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Encapsulate Manila Ganesha setup and testing.""" diff --git a/zaza/openstack/charm_tests/manila_ganesha/setup.py b/zaza/openstack/charm_tests/manila_ganesha/setup.py new file mode 100644 index 0000000..d2b694e --- /dev/null +++ b/zaza/openstack/charm_tests/manila_ganesha/setup.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Encapsulate Manila Ganesha setup.""" + + +import zaza.openstack.utilities.openstack as openstack_utils + +from manilaclient import client as manilaclient + + +def setup_ganesha_share_type(manila_client=None): + """Create a share type for manila with Ganesha. + + :param manila_client: Authenticated manilaclient + :type manila_client: manilaclient.Client + """ + if manila_client is None: + keystone_session = openstack_utils.get_overcloud_keystone_session() + manila_client = manilaclient.Client( + session=keystone_session, client_version='2') + + manila_client.share_types.create( + name="cephfsnfstype", spec_driver_handles_share_servers=False, + extra_specs={ + 'vendor_name': 'Ceph', + 'storage_protocol': 'NFS', + 'snapshot_support': False, + }) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py new file mode 100644 index 0000000..f7b5ade --- /dev/null +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila Ganesha testing.""" + + +from manilaclient import client as manilaclient + +import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.charm_tests.neutron.tests as neutron_tests +import zaza.openstack.charm_tests.nova.utils as nova_utils +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.configure.guest as guest +import zaza.openstack.utilities.openstack as openstack_utils + + +class ManilaGaneshaTests(test_utils.OpenStackBaseTest): + """Encapsulate Manila Ganesha tests.""" + + RESOURCE_PREFIX = 'zaza-manilatests' + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaGaneshaTests, cls).setUpClass() + cls.nova_client = ( + openstack_utils.get_nova_session_client(cls.keystone_session)) + cls.manila_client = manilaclient.Client( + session=cls.keystone_session, client_version='2') + + def test_manila_share(self): + """Test that Manila + Ganesha shares can be accessed on two instances. + + 1. create a share + 2. Spawn two servers + 3. mount it on both + 4. write a file on one + 5. read it on the other + 6. profit + """ + # Create a share + share = self.manila_client.shares.create( + share_type='cephfsnfstype', name='cephnfsshare1', + share_proto="nfs", size=1) + + # Spawn Servers + guest.launch_instance( + glance_setup.LTS_IMAGE_NAME, + vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX)) + guest.launch_instance( + glance_setup.LTS_IMAGE_NAME, + vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX)) + + instance_1 = self.nova_client.servers.find( + name='{}-ins-1'.format(self.RESOURCE_PREFIX)) + fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] + instance_2 = self.nova_client.servers.find( + name='{}-ins-2'.format(self.RESOURCE_PREFIX)) + fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] + + share.allow(access_type='ip', access=fip_1, access_level='rw') + share.allow(access_type='ip', access=fip_2, access_level='rw') + + # Mount Share + username = guest.boot_tests['bionic']['username'] + password = guest.boot_tests['bionic'].get('password') + privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) + mount_path = share.export_locations[0] + + # Write a file on instance_1 + def verify_setup(stdin, stdout, stderr): + status = stdout.channel.recv_exit_status() + self.assertEqual(status, 0) + + openstack_utils.ssh_command( + username, fip_1, 'instance-1', + 'sudo apt install -yq nfs-common && ' + 'sudo mkdir -p /mnt/ceph && ' + 'sudo mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph && ' + 'echo "test" | sudo tee /mnt/ceph/test'.format( + mount_path), + password=password, privkey=privkey, verify=verify_setup) + + # Read that file on instance_2 + openstack_utils.ssh_command( + username, fip_2, 'instance-2', + 'sudo apt install -yq nfs-common && ' + 'sudo /bin/mkdir -p /mnt/ceph && ' + 'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph' + .format(mount_path), + password=password, privkey=privkey, verify=verify_setup) + + def verify(stdin, stdout, stderr): + status = stdout.channel.recv_exit_status() + out = "" + print("[{}] Stdout:".format(status)) + for line in iter(stdout.readline, ""): + out += line + self.assertEqual(out, "test\n") + + openstack_utils.ssh_command( + username, fip_2, 'instance-2', + 'sudo cat /mnt/ceph/test'.format( + mount_path), + password=password, privkey=privkey, verify=verify) From a5b618d08bde4e3178ad3a815e25b1d0a1236072 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 7 Nov 2019 15:13:06 +0800 Subject: [PATCH 178/898] refactor tests a bit --- .../charm_tests/manila_ganesha/tests.py | 32 +++++++++---------- zaza/openstack/configure/guest.py | 3 ++ 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index f7b5ade..3353517 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -31,6 +31,14 @@ class ManilaGaneshaTests(test_utils.OpenStackBaseTest): """Encapsulate Manila Ganesha tests.""" RESOURCE_PREFIX = 'zaza-manilatests' + INSTANCE_USERDATA = """#cloud-config +packages: +- nfs-common +write_files: +- path: "/mnt/ceph" + permissions: '0600' + owner: root:root +""" @classmethod def setUpClass(cls): @@ -55,20 +63,19 @@ class ManilaGaneshaTests(test_utils.OpenStackBaseTest): share = self.manila_client.shares.create( share_type='cephfsnfstype', name='cephnfsshare1', share_proto="nfs", size=1) + mount_path = share.export_locations[0] # Spawn Servers - guest.launch_instance( + instance_1 = guest.launch_instance( glance_setup.LTS_IMAGE_NAME, - vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX)) - guest.launch_instance( + vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX), + userdata=self.INSTANCE_USERDATA) + instance_2 = guest.launch_instance( glance_setup.LTS_IMAGE_NAME, - vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX)) + vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX), + userdata=self.INSTANCE_USERDATA) - instance_1 = self.nova_client.servers.find( - name='{}-ins-1'.format(self.RESOURCE_PREFIX)) fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] - instance_2 = self.nova_client.servers.find( - name='{}-ins-2'.format(self.RESOURCE_PREFIX)) fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] share.allow(access_type='ip', access=fip_1, access_level='rw') @@ -78,7 +85,6 @@ class ManilaGaneshaTests(test_utils.OpenStackBaseTest): username = guest.boot_tests['bionic']['username'] password = guest.boot_tests['bionic'].get('password') privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) - mount_path = share.export_locations[0] # Write a file on instance_1 def verify_setup(stdin, stdout, stderr): @@ -86,9 +92,6 @@ class ManilaGaneshaTests(test_utils.OpenStackBaseTest): self.assertEqual(status, 0) openstack_utils.ssh_command( - username, fip_1, 'instance-1', - 'sudo apt install -yq nfs-common && ' - 'sudo mkdir -p /mnt/ceph && ' 'sudo mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph && ' 'echo "test" | sudo tee /mnt/ceph/test'.format( mount_path), @@ -96,17 +99,14 @@ class ManilaGaneshaTests(test_utils.OpenStackBaseTest): # Read that file on instance_2 openstack_utils.ssh_command( - username, fip_2, 'instance-2', - 'sudo apt install -yq nfs-common && ' - 'sudo /bin/mkdir -p /mnt/ceph && ' 'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph' .format(mount_path), password=password, privkey=privkey, verify=verify_setup) def verify(stdin, stdout, stderr): status = stdout.channel.recv_exit_status() + self.assertEqual(status, 0) out = "" - print("[{}] Stdout:".format(status)) for line in iter(stdout.readline, ""): out += line self.assertEqual(out, "test\n") diff --git a/zaza/openstack/configure/guest.py b/zaza/openstack/configure/guest.py index 9418cb3..920b539 100644 --- a/zaza/openstack/configure/guest.py +++ b/zaza/openstack/configure/guest.py @@ -62,6 +62,8 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, :type meta: dict :param userdata: Configuration to use upon launch, used by cloud-init. :type userdata: str + :returns: the created instance + :rtype: novaclient.Server """ keystone_session = openstack_utils.get_overcloud_keystone_session() nova_client = openstack_utils.get_nova_session_client(keystone_session) @@ -140,3 +142,4 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, vm_name=vm_name, password=boot_tests[instance_key].get('password'), privkey=openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME)) + return instance From d016777c77584a8589e60d10d116ea935a773713 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 7 Nov 2019 20:15:22 +0800 Subject: [PATCH 179/898] ensure python-manilaclient is in the setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 71c965b..7f2d6a4 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ install_require = [ 'python-heatclient', 'python-glanceclient', 'python-keystoneclient', + 'python-manilaclient', 'python-novaclient', 'python-neutronclient', 'python-octaviaclient', From 6c817e5942eb85b822bda76737f48a70533f3977 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 7 Nov 2019 20:42:41 +0800 Subject: [PATCH 180/898] move getting share locations to later --- zaza/openstack/charm_tests/manila_ganesha/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 3353517..9b9287f 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -63,7 +63,6 @@ write_files: share = self.manila_client.shares.create( share_type='cephfsnfstype', name='cephnfsshare1', share_proto="nfs", size=1) - mount_path = share.export_locations[0] # Spawn Servers instance_1 = guest.launch_instance( @@ -91,6 +90,8 @@ write_files: status = stdout.channel.recv_exit_status() self.assertEqual(status, 0) + mount_path = share.export_locations[0] + openstack_utils.ssh_command( 'sudo mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph && ' 'echo "test" | sudo tee /mnt/ceph/test'.format( From 55c2e60b8ec498be3f6846b0dd8ac51a77188827 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 7 Nov 2019 15:55:35 -0800 Subject: [PATCH 181/898] Fix generator sort failure --- zaza/openstack/charm_tests/mysql/tests.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 50ac8de..d695187 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -242,8 +242,6 @@ class PerconaClusterColdStartTest(PerconaClusterBaseTest): openstack_utils.get_undercloud_keystone_session()) cls.nova_client = openstack_utils.get_nova_session_client( cls.overcloud_keystone_session) - cls.machines = ( - juju_utils.get_machine_uuids_for_application(cls.application)) def resolve_update_status_errors(self): """Resolve update-status hooks error. @@ -267,25 +265,27 @@ class PerconaClusterColdStartTest(PerconaClusterBaseTest): After bootstrapping a non-leader node, notify bootstrapped on the leader node. """ + _machines = list( + juju_utils.get_machine_uuids_for_application(self.application)) # Stop Nodes - self.machines.sort() + _machines.sort() # Avoid hitting an update-status hook logging.debug("Wait till model is idle ...") zaza.model.block_until_all_units_idle() - logging.info("Stopping instances: {}".format(self.machines)) - for uuid in self.machines: + logging.info("Stopping instances: {}".format(_machines)) + for uuid in _machines: self.nova_client.servers.stop(uuid) logging.debug("Wait till all machines are shutoff ...") - for uuid in self.machines: + for uuid in _machines: openstack_utils.resource_reaches_status(self.nova_client.servers, uuid, expected_status='SHUTOFF', stop_after_attempt=16) # Start nodes - self.machines.sort(reverse=True) - logging.info("Starting instances: {}".format(self.machines)) - for uuid in self.machines: + _machines.sort(reverse=True) + logging.info("Starting instances: {}".format(_machines)) + for uuid in _machines: self.nova_client.servers.start(uuid) for unit in zaza.model.get_units(self.application): From d97b6a77c4e105aeb3dac1e3ab01f8fa47eaf2fe Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 8 Nov 2019 08:59:40 +0800 Subject: [PATCH 182/898] fix ssh command --- zaza/openstack/charm_tests/manila_ganesha/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 9b9287f..9745f4b 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -93,6 +93,7 @@ write_files: mount_path = share.export_locations[0] openstack_utils.ssh_command( + username, fip_1, 'instance-1', 'sudo mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph && ' 'echo "test" | sudo tee /mnt/ceph/test'.format( mount_path), @@ -100,6 +101,7 @@ write_files: # Read that file on instance_2 openstack_utils.ssh_command( + username, fip_2, 'instance-2', 'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph' .format(mount_path), password=password, privkey=privkey, verify=verify_setup) From 46fad02251ed5bb71b5e183bd6173f79006705cb Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 8 Nov 2019 11:50:51 +0800 Subject: [PATCH 183/898] apparently cloud-init doesn't actually create a directory like that --- zaza/openstack/charm_tests/manila_ganesha/tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 9745f4b..eb4f2be 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -34,10 +34,6 @@ class ManilaGaneshaTests(test_utils.OpenStackBaseTest): INSTANCE_USERDATA = """#cloud-config packages: - nfs-common -write_files: -- path: "/mnt/ceph" - permissions: '0600' - owner: root:root """ @classmethod @@ -94,6 +90,7 @@ write_files: openstack_utils.ssh_command( username, fip_1, 'instance-1', + 'sudo mkdir -p /mnt/ceph && ' 'sudo mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph && ' 'echo "test" | sudo tee /mnt/ceph/test'.format( mount_path), @@ -102,6 +99,7 @@ write_files: # Read that file on instance_2 openstack_utils.ssh_command( username, fip_2, 'instance-2', + 'sudo mkdir -p /mnt/ceph && ' 'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph' .format(mount_path), password=password, privkey=privkey, verify=verify_setup) From 9391758fbade137b577b903adf65ab90f1de32f2 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 11 Nov 2019 10:04:30 +0100 Subject: [PATCH 184/898] Increase number of attempts for ping_response Some configurations require more time to settle before we get the expected response. (LP: #1851710) --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 116bfed..e029c3c 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2174,7 +2174,7 @@ def cloud_init_complete(nova_client, vm_id, bootstring): @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), - reraise=True, stop=tenacity.stop_after_attempt(8)) + reraise=True, stop=tenacity.stop_after_attempt(16)) def ping_response(ip): """Wait for ping to respond on the given IP. From 1689a4452ed7088ce99860a8dc830356388720b8 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 12 Nov 2019 15:22:53 +0100 Subject: [PATCH 185/898] net: Add support for DVR deployments with NGW --- zaza/openstack/utilities/openstack.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 545e7aa..9a755e8 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -465,8 +465,7 @@ def get_ovs_uuids(): :returns: List of uuids :rtype: Iterator[str] """ - return (juju_utils - .get_machine_uuids_for_application('neutron-openvswitch')) + return juju_utils.get_machine_uuids_for_application('neutron-openvswitch') def get_ovn_uuids(): @@ -636,11 +635,20 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, config = {} if dvr_enabled(): - uuids = itertools.islice(get_ovs_uuids(), limit_gws) + uuids = itertools.islice(itertools.chain(get_ovs_uuids(), + get_gateway_uuids()), + limit_gws) # If dvr, do not attempt to persist nic in netplan # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78 add_dataport_to_netplan = False application_names = ['neutron-openvswitch'] + try: + ngw = 'neutron-gateway' + next(juju_utils.get_machine_uuids_for_application(ngw)) + application_names.append(ngw) + except StopIteration: + # neutron-gateway not in deployment + pass elif ovn_present(): uuids = itertools.islice(get_ovn_uuids(), limit_gws) application_names = ['ovn-chassis'] From 43322b01523f471396ee15fb0204aad6cbc49688 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 7 Nov 2019 09:00:41 +0100 Subject: [PATCH 186/898] neutron: Deduce expected security check results on TLS presence If 'neutron-api' has a certificates relation to vault we have different expectations of the security checklist result. --- zaza/openstack/charm_tests/neutron/tests.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 8657e4b..be20009 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -90,19 +90,22 @@ class SecurityTest(test_utils.OpenStackBaseTest): def test_security_checklist(self): """Verify expected state with security-checklist.""" - # Changes fixing the below expected failures will be made following - # this initial work to get validation in. There will be bugs targeted - # to each one and resolved independently where possible. - - expected_failures = [ - 'validate-enables-tls', + tls_checks = [ 'validate-uses-tls-for-keystone', ] + expected_failures = [ + 'validate-enables-tls', # LP: #1851610 + ] expected_passes = [ 'validate-file-ownership', 'validate-file-permissions', 'validate-uses-keystone', ] + if zaza.model.get_relation_id( + 'neutron-api', 'vault', remote_interface_name='certificates'): + expected_passes.extend(tls_checks) + else: + expected_failures.extend(tls_checks) for unit in zaza.model.get_units('neutron-api', model_name=self.model_name): From 9108059f71ae4dee8e97f23bb52ecc1da24bf683 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 7 Nov 2019 09:28:11 +0100 Subject: [PATCH 187/898] vault: Extend workaround for LP: #1826382 We also need to restart consumers of the placement API on the 'nova-cloud-controller' unit. --- zaza/openstack/charm_tests/vault/setup.py | 26 ++++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 03b8f65..952bbcd 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -92,16 +92,22 @@ def auto_initialize(cacert=None, validation_application='keystone'): validate_ca(cacertificate, application=validation_application) # Once validation has completed restart nova-compute to work around # bug #1826382 - try: - cmd = 'systemctl restart nova-compute' - for unit in zaza.model.get_units('nova-compute'): - result = zaza.model.run_on_unit(unit.entity_id, cmd) - assert int(result['Code']) == 0, ( - 'Restart of nova-compute on {} failed'.format( - unit.entity_id)) - except KeyError: - # Nothing todo if there are no nova-compute units - pass + cmd_map = { + 'nova-cloud-controller': ('systemctl restart ' + 'nova-scheduler nova-conductor'), + 'nova-compute': 'systemctl restart nova-compute', + } + for app in ('nova-compute', 'nova-cloud-controller',): + try: + for unit in zaza.model.get_units(app): + result = zaza.model.run_on_unit( + unit.entity_id, cmd_map[app]) + assert int(result['Code']) == 0, ( + 'Restart of services on {} failed'.format( + unit.entity_id)) + except KeyError: + # Nothing todo if there are no app units + pass auto_initialize_no_validation = functools.partial( From 9cd26e9e4c78d18750cae514d526e18346a200c1 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 12 Nov 2019 14:00:48 -0800 Subject: [PATCH 188/898] Add mysqldump action test --- zaza/openstack/charm_tests/mysql/tests.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index d695187..5cbc42d 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -421,3 +421,22 @@ class MySQLInnoDBClusterTests(MySQLCommonTests): assert "OK" in cluster_status["defaultReplicaSet"]["status"], ( "Cluster status action failed.") logging.info("Passed cluster-status action test.") + + def test_110_mysqldump(self): + """Backup mysql. + + Run the mysqldump action. + """ + _db = "keystone" + _file_key = "mysqldump-file" + # Update which node is the leader and which are not + _leaders, _non_leaders = self.get_leaders_and_non_leaders() + logging.info("Execute mysqldump action") + action = zaza.model.run_action( + _non_leaders[0], + "mysqldump", + action_params={"databases": _db}) + _results = action.data["results"] + assert _db in _results[_file_key], ( + "Mysqldump action failed.") + logging.info("Passed mysqldump action test.") From e51b3c481dc05a99af4e398926d89d396d4a4200 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 13 Nov 2019 11:58:14 +0100 Subject: [PATCH 189/898] neutron: Add tenacity to retry ping It would be awesome if the first iteration of a ARP lookup, ICMP echo and ICMP reply on a newly created instance was always successful. Sadly this is not the reality. --- zaza/openstack/charm_tests/neutron/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index be20009..7433793 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -18,6 +18,7 @@ import logging +import tenacity import unittest import zaza @@ -175,6 +176,8 @@ class NeutronNetworkingTest(unittest.TestCase): self.validate_instance_can_reach_router(instance_1, verify) self.validate_instance_can_reach_router(instance_2, verify) + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), + reraise=True, stop=tenacity.stop_after_attempt(8)) def validate_instance_can_reach_other(self, instance_1, instance_2, @@ -206,6 +209,8 @@ class NeutronNetworkingTest(unittest.TestCase): 'ping -c 1 {}'.format(floating_2), password=password, privkey=privkey, verify=verify) + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), + reraise=True, stop=tenacity.stop_after_attempt(8)) def validate_instance_can_reach_router(self, instance, verify): """ Validate that an instance can reach it's primary gateway. From 4298be05d261f091ef9f0d90147395686112c2ee Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 13 Nov 2019 14:22:33 +0100 Subject: [PATCH 190/898] policyd: await agent status executing after config change The current implementation may miss waiting for config change to actually happen as it in some circumstances jumps to waiting for a idle state prior to the model executing anything. Add a wait for agent status `executing` immediatelly after config change to avoid this. Fixes #123 --- zaza/openstack/charm_tests/policyd/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index c359695..8b96d03 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -86,6 +86,7 @@ class PolicydTest(object): config = {"use-policyd-override": s} logging.info("Setting config to {}".format(config)) zaza_model.set_application_config(self.application_name, config) + zaza_model.wait_for_agent_status() def _make_zip_file_from(self, name, files): """Make a zip file from a dictionary of filename: string. From f0be9b5136dd24f6deeb209d909df1ccd9eb87fb Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 15 Nov 2019 15:49:14 +0100 Subject: [PATCH 191/898] Add manila API test --- zaza/openstack/charm_tests/manila/__init__.py | 18 +++++++ zaza/openstack/charm_tests/manila/tests.py | 48 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 zaza/openstack/charm_tests/manila/__init__.py create mode 100644 zaza/openstack/charm_tests/manila/tests.py diff --git a/zaza/openstack/charm_tests/manila/__init__.py b/zaza/openstack/charm_tests/manila/__init__.py new file mode 100644 index 0000000..064c54c --- /dev/null +++ b/zaza/openstack/charm_tests/manila/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Encapsulate Manila setup and testing.""" diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py new file mode 100644 index 0000000..e2ae24b --- /dev/null +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila testing.""" + + +from manilaclient import client as manilaclient + +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +class ManilaTests(test_utils.OpenStackBaseTest): + """Encapsulate Manila tests.""" + + RESOURCE_PREFIX = 'zaza-manilatests' + INSTANCE_USERDATA = """#cloud-config +packages: +- nfs-common +""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaTests, cls).setUpClass() + cls.nova_client = ( + openstack_utils.get_nova_session_client(cls.keystone_session)) + cls.manila_client = manilaclient.Client( + session=cls.keystone_session, client_version='2') + + def test_manila_api(self): + """Test that the Manila API is working.""" + # now just try a list the shares + # NOTE(AJK) the 'search_opts={}' is to work around Bug#1707303 + self.manila.shares.list(search_opts={}) From 9c75b89273485bbba414fda61dfef6d4a4479ad4 Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 13 Nov 2019 16:05:52 -0800 Subject: [PATCH 192/898] Reboot cluster from complete outage tests Testing for cold start scenario for MySQL InnoDB Cluster. --- zaza/openstack/charm_tests/mysql/tests.py | 101 ++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 5cbc42d..27f5194 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -440,3 +440,104 @@ class MySQLInnoDBClusterTests(MySQLCommonTests): assert _db in _results[_file_key], ( "Mysqldump action failed.") logging.info("Passed mysqldump action test.") + + +class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): + """Percona Cluster cold start tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running percona-cluster cold start tests.""" + super().setUpClass() + cls.application = "mysql-innodb-cluster" + cls.overcloud_keystone_session = ( + openstack_utils.get_undercloud_keystone_session()) + cls.nova_client = openstack_utils.get_nova_session_client( + cls.overcloud_keystone_session) + + def resolve_update_status_errors(self): + """Resolve update-status hooks error. + + This should *only* be used after an instance hard reboot to handle the + situation where a update-status hook was running when the unit was + rebooted. + """ + zaza.model.resolve_units( + application_name=self.application, + erred_hook='update-status', + wait=True) + + def test_100_cold_start_bootstrap(self): + """Bootstrap a non-leader node. + + After bootstrapping a non-leader node, notify bootstrapped on the + leader node. + """ + _machines = list( + juju_utils.get_machine_uuids_for_application(self.application)) + # Stop Nodes + _machines.sort() + # Avoid hitting an update-status hook + logging.debug("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + logging.info("Stopping instances: {}".format(_machines)) + for uuid in _machines: + self.nova_client.servers.stop(uuid) + logging.debug("Wait till all machines are shutoff ...") + for uuid in _machines: + openstack_utils.resource_reaches_status(self.nova_client.servers, + uuid, + expected_status='SHUTOFF', + stop_after_attempt=16) + + # Start nodes + _machines.sort(reverse=True) + logging.info("Starting instances: {}".format(_machines)) + for uuid in _machines: + self.nova_client.servers.start(uuid) + + for unit in zaza.model.get_units(self.application): + zaza.model.block_until_unit_wl_status( + unit.entity_id, + 'unknown', + negate_match=True) + + logging.debug("Wait till model is idle ...") + # XXX If a hook was executing on a unit when it was powered off + # it comes back in an error state. + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + + logging.debug("Wait for application states ...") + for unit in zaza.model.get_units(self.application): + try: + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + states = {self.application: { + "workload-status": "blocked", + "workload-status-message": + "MySQL InnoDB Cluster not healthy: None"}} + zaza.model.wait_for_application_states(states=states) + + # Update which node is the leader and which are not + _leader, _non_leaders = self.get_leaders_and_non_leaders() + logging.info("Execute reboot-cluster-from-complete-outage " + "action after cold boot ...") + action = zaza.model.run_action( + _non_leaders[0], + "reboot-cluster-from-complete-outage", + action_params={}) + assert "completed" in action.data["status"], ( + "Reboot Cluster From Complete Outage action failed: {}" + .format(action.data)) + logging.debug("Wait for application states ...") + for unit in zaza.model.get_units(self.application): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {})) From af64a75fd70ce3ca561500cdd4376fea19c02bc8 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 19 Nov 2019 11:22:00 -0800 Subject: [PATCH 193/898] Use model.run_action_on_leader Address reveiw requests Use model.run_action_on_leader Log action.data on action failure --- zaza/openstack/charm_tests/mysql/tests.py | 53 +++++++++++++---------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 27f5194..7603392 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -410,16 +410,15 @@ class MySQLInnoDBClusterTests(MySQLCommonTests): Run the cluster-status action. """ - # Update which node is the leader and which are not - _leaders, _non_leaders = self.get_leaders_and_non_leaders() logging.info("Execute cluster-status action") - action = zaza.model.run_action( - _non_leaders[0], + action = zaza.model.run_action_on_leader( + self.application, "cluster-status", action_params={}) cluster_status = json.loads(action.data["results"]["cluster-status"]) assert "OK" in cluster_status["defaultReplicaSet"]["status"], ( - "Cluster status action failed.") + "Cluster status action failed: {}" + .format(action.data)) logging.info("Passed cluster-status action test.") def test_110_mysqldump(self): @@ -429,18 +428,33 @@ class MySQLInnoDBClusterTests(MySQLCommonTests): """ _db = "keystone" _file_key = "mysqldump-file" - # Update which node is the leader and which are not - _leaders, _non_leaders = self.get_leaders_and_non_leaders() logging.info("Execute mysqldump action") - action = zaza.model.run_action( - _non_leaders[0], + action = zaza.model.run_action_on_leader( + self.application, "mysqldump", action_params={"databases": _db}) _results = action.data["results"] assert _db in _results[_file_key], ( - "Mysqldump action failed.") + "Mysqldump action failed: {}".format(action.data)) logging.info("Passed mysqldump action test.") + def test_120_set_cluster_option(self): + """Set cluster option. + + Run the set-cluster-option action. + """ + _key = "autoRejoinTries" + _value = "500" + logging.info("Set cluster option {}={}".format(_key, _value)) + action = zaza.model.run_action_on_leader( + self.application, + "set-cluster-option", + action_params={"key": _key, "value": _value}) + assert "Success" in action.data["results"]["outcome"], ( + "Set cluster option {}={} action failed: {}" + .format(_key, _value, action.data)) + logging.info("Passed set cluster option action test.") + class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): """Percona Cluster cold start tests.""" @@ -467,11 +481,10 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): erred_hook='update-status', wait=True) - def test_100_cold_start_bootstrap(self): - """Bootstrap a non-leader node. + def test_100_reboot_cluster_from_complete_outage(self): + """Reboot cluster from complete outage. - After bootstrapping a non-leader node, notify bootstrapped on the - leader node. + After a cold start, reboot cluster from complete outage. """ _machines = list( juju_utils.get_machine_uuids_for_application(self.application)) @@ -503,8 +516,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): negate_match=True) logging.debug("Wait till model is idle ...") - # XXX If a hook was executing on a unit when it was powered off - # it comes back in an error state. try: zaza.model.block_until_all_units_idle() except zaza.model.UnitError: @@ -524,16 +535,14 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): "MySQL InnoDB Cluster not healthy: None"}} zaza.model.wait_for_application_states(states=states) - # Update which node is the leader and which are not - _leader, _non_leaders = self.get_leaders_and_non_leaders() logging.info("Execute reboot-cluster-from-complete-outage " "action after cold boot ...") - action = zaza.model.run_action( - _non_leaders[0], + action = zaza.model.run_action_on_leader( + self.application, "reboot-cluster-from-complete-outage", action_params={}) - assert "completed" in action.data["status"], ( - "Reboot Cluster From Complete Outage action failed: {}" + assert "Success" in action.data["results"]["outcome"], ( + "Reboot cluster from complete outage action failed: {}" .format(action.data)) logging.debug("Wait for application states ...") for unit in zaza.model.get_units(self.application): From c0db2def3bb1049b02ee132bf5aea7051015d0e6 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 15 Nov 2019 15:41:16 +0000 Subject: [PATCH 194/898] Modified policyd tests to support openstack-dashboard * Addition of the (incomplete) openstack-dashboard test. * Modification of other charm policyd override tests to support multi-file policy overrides resource ZIP. --- tox.ini | 2 +- .../charm_tests/openstack_dashboard/tests.py | 305 +++++++++++------- zaza/openstack/charm_tests/policyd/tests.py | 26 +- 3 files changed, 203 insertions(+), 130 deletions(-) diff --git a/tox.ini b/tox.ini index 6a37cfe..f5be2fa 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ deps = -r{toxinidir}/requirements.txt commands = /bin/true [flake8] -ignore = E402,E226,W504 +ignore = E402,E226,W503 per-file-ignores = unit_tests/**: D diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index ee39518..dd37080 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -18,12 +18,141 @@ import logging import requests import tenacity import urllib.request +import yaml import zaza.model as zaza_model import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.juju as openstack_juju +import zaza.openstack.charm_tests.policyd.tests as policyd + + +class AuthExceptions(Exception): + """Exception base class for the 401 test.""" + + pass + + +class FailedAuth(AuthExceptions): + """Failed exception for the 401 test.""" + + pass + + +def _get_dashboard_ip(): + """Get the IP of the dashboard. + + :returns: The IP of the dashboard + :rtype: str + """ + unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') + keystone_unit = zaza_model.get_lead_unit_name('keystone') + dashboard_relation = openstack_juju.get_relation_from_unit( + keystone_unit, unit_name, 'identity-service') + dashboard_ip = dashboard_relation['private-address'] + logging.debug("dashboard_ip is: {}".format(dashboard_ip)) + return dashboard_ip + + +# NOTE: intermittent authentication fails. Wrap in a retry loop. +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, + min=5, max=10), + reraise=True) +def _login(dashboard_ip, domain, username, password): + """Login to the website to get a session. + + :param dashboard_ip: The IP address of the dashboard to log in to. + :type dashboard_ip: str + :param domain: the domain to login into + :type domain: str + :param username: the username to login as + :type username: str + :param password: the password to use to login + :type password: str + :returns: tuple of (client, response) where response is the page after + logging in. + :rtype: (requests.sessions.Session, requests.models.Response) + :raises: FailedAuth if the authorisation doesn't work + """ + auth_url = 'http://{}/horizon/auth/login/'.format(dashboard_ip) + + # start session, get csrftoken + client = requests.session() + client.get(auth_url) + + if 'csrftoken' in client.cookies: + csrftoken = client.cookies['csrftoken'] + else: + raise Exception("Missing csrftoken") + + # build and send post request + overcloud_auth = openstack_utils.get_overcloud_auth() + + if overcloud_auth['OS_AUTH_URL'].endswith("v2.0"): + api_version = 2 + else: + api_version = 3 + keystone_client = openstack_utils.get_keystone_client( + overcloud_auth) + catalog = keystone_client.service_catalog.get_endpoints() + logging.info(catalog) + if api_version == 2: + region = catalog['identity'][0]['publicURL'] + else: + region = [i['url'] + for i in catalog['identity'] + if i['interface'] == 'public'][0] + + auth = { + 'domain': domain, + 'username': username, + 'password': password, + 'csrfmiddlewaretoken': csrftoken, + 'next': '/horizon/', + 'region': region, + } + + # In the minimal test deployment /horizon/project/ is unauthorized, + # this does not occur in a full deployment and is probably due to + # services/information missing that horizon wants to display data + # for. + # Redirect to /horizon/identity/ instead. + if (openstack_utils.get_os_release() + >= openstack_utils.get_os_release('xenial_queens')): + auth['next'] = '/horizon/identity/' + + if (openstack_utils.get_os_release() + >= openstack_utils.get_os_release('bionic_stein')): + auth['region'] = 'default' + + if api_version == 2: + del auth['domain'] + + logging.info('POST data: "{}"'.format(auth)) + response = client.post(auth_url, data=auth, headers={'Referer': auth_url}) + + # NOTE(ajkavanagh) there used to be a trusty/icehouse test in the + # amulet test, but as the zaza tests only test from trusty/mitaka + # onwards, the test has been dropped + if (openstack_utils.get_os_release() + >= openstack_utils.get_os_release('bionic_stein')): + expect = "Sign Out" + # update the in dashboard seems to require region to be default in + # this test configuration + region = 'default' + else: + expect = 'Projects - OpenStack Dashboard' + + if expect not in response.text: + msg = 'FAILURE code={} text="{}"'.format(response, + response.text) + # NOTE(thedac) amulet.raise_status exits on exception. + # Raise a custom exception. + logging.info("Yeah, wen't wrong: {}".format(msg)) + raise FailedAuth(msg) + logging.info("Logged into okay") + return client, response class OpenStackDashboardTests(test_utils.OpenStackBaseTest): @@ -153,126 +282,21 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): self.assertIn('OpenStack Dashboard', html, "Dashboard frontpage check failed") - class AuthExceptions(Exception): - """Exception base class for the 401 test.""" - - pass - - class FailedAuth(AuthExceptions): - """Failed exception for the 401 test.""" - - pass - - class PassedAuth(AuthExceptions): - """Passed exception for the 401 test.""" - - pass - def test_401_authenticate(self): - """Validate that authentication succeeds for client log in. - - Ported from amulet tests. - """ + """Validate that authentication succeeds for client log in.""" logging.info('Checking authentication through dashboard...') - unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') - keystone_unit = zaza_model.get_lead_unit_name('keystone') - dashboard_relation = openstack_juju.get_relation_from_unit( - keystone_unit, unit_name, 'identity-service') - dashboard_ip = dashboard_relation['private-address'] - logging.debug("... dashboard_ip is:{}".format(dashboard_ip)) - - url = 'http://{}/horizon/auth/login/'.format(dashboard_ip) - + dashboard_ip = _get_dashboard_ip() overcloud_auth = openstack_utils.get_overcloud_auth() - if overcloud_auth['OS_AUTH_URL'].endswith("v2.0"): - api_version = 2 - else: - api_version = 3 - keystone_client = openstack_utils.get_keystone_client( - overcloud_auth) - catalog = keystone_client.service_catalog.get_endpoints() - logging.info(catalog) - if api_version == 2: - region = catalog['identity'][0]['publicURL'] - else: - region = [i['url'] - for i in catalog['identity'] - if i['interface'] == 'public'][0] - - # NOTE(ajkavanagh) there used to be a trusty/icehouse test in the - # amulet test, but as the zaza tests only test from trusty/mitaka - # onwards, the test has been dropped - if (openstack_utils.get_os_release() >= - openstack_utils.get_os_release('bionic_stein')): - expect = "Sign Out" - # update the in dashboard seems to require region to be default in - # this test configuration - region = 'default' - else: - expect = 'Projects - OpenStack Dashboard' - - # NOTE(thedac) Similar to the connection test above we get occasional - # intermittent authentication fails. Wrap in a retry loop. - @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, - min=5, max=10), - retry=tenacity.retry_unless_exception_type( - self.AuthExceptions), - reraise=True) - def _do_auth_check(expect): - # start session, get csrftoken - client = requests.session() - client.get(url) - - if 'csrftoken' in client.cookies: - csrftoken = client.cookies['csrftoken'] - else: - raise Exception("Missing csrftoken") - - # build and send post request - auth = { - 'domain': 'admin_domain', - 'username': 'admin', - 'password': overcloud_auth['OS_PASSWORD'], - 'csrfmiddlewaretoken': csrftoken, - 'next': '/horizon/', - 'region': region, - } - - # In the minimal test deployment /horizon/project/ is unauthorized, - # this does not occur in a full deployment and is probably due to - # services/information missing that horizon wants to display data - # for. - # Redirect to /horizon/identity/ instead. - if (openstack_utils.get_os_release() >= - openstack_utils.get_os_release('xenial_queens')): - auth['next'] = '/horizon/identity/' - - if (openstack_utils.get_os_release() >= - openstack_utils.get_os_release('bionic_stein')): - auth['region'] = 'default' - - if api_version == 2: - del auth['domain'] - - logging.info('POST data: "{}"'.format(auth)) - response = client.post(url, data=auth, headers={'Referer': url}) - - if expect not in response.text: - msg = 'FAILURE code={} text="{}"'.format(response, - response.text) - # NOTE(thedac) amulet.raise_status exits on exception. - # Raise a custom exception. - logging.info("Yeah, wen't wrong: {}".format(msg)) - raise self.FailedAuth(msg) - raise self.PassedAuth() - - try: - _do_auth_check(expect) - except self.FailedAuth as e: - assert False, str(e) - except self.PassedAuth: - pass + password = overcloud_auth['OS_PASSWORD'], + logging.info("admin password is {}".format(password)) + # try to get the url which will either pass or fail with a 403 + overcloud_auth = openstack_utils.get_overcloud_auth() + domain = 'admin_domain', + username = 'admin', + password = overcloud_auth['OS_PASSWORD'], + _login(dashboard_ip, domain, username, password) + logging.info('OK') def test_404_connection(self): """Verify the apache status module gets disabled when hardening apache. @@ -357,3 +381,52 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): """ with self.pause_resume(['apache2']): logging.info("Testing pause resume") + + +class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): + """Test the policyd override using the dashboard.""" + + _rule = {'identity/rule.yaml': yaml.dump({ + 'identity:list_domains': '!', + 'identity:get_domain': '!', + 'identity:update_domain': '!', + 'identity:list_domains_for_user': '!', + })} + + # url associated with rule above that will return HTTP 403 + url = "http://{}/horizon/identity/domains" + + @classmethod + def setUpClass(cls, application_name=None): + """Run class setup for running horizon charm operation tests.""" + super(OpenStackDashboardPolicydTests, cls).setUpClass( + application_name="openstack-dashboard") + cls.application_name = "openstack-dashboard" + + def get_client_and_attempt_operation(self, ip): + """Attempt to list users on the openstack-dashboard service. + + This is slightly complicated in that the client is actually a web-site. + Thus, the test has to login first and then attempt the operation. This + makes the test a little more complicated. + + :param ip: the IP address to get the session against. + :type ip: str + :raises: PolicydOperationFailedException if operation fails. + """ + dashboard_ip = _get_dashboard_ip() + logging.info("Dashboard is at {}".format(dashboard_ip)) + overcloud_auth = openstack_utils.get_overcloud_auth() + password = overcloud_auth['OS_PASSWORD'], + logging.info("admin password is {}".format(password)) + # try to get the url which will either pass or fail with a 403 + overcloud_auth = openstack_utils.get_overcloud_auth() + domain = 'admin_domain', + username = 'admin', + password = overcloud_auth['OS_PASSWORD'], + client, response = _login(dashboard_ip, domain, username, password) + # now attempt to get the domains page + _url = self.url.format(dashboard_ip) + result = client.get(_url) + if result.status_code == 403: + raise policyd.PolicydOperationFailedException("Not authenticated") diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 8b96d03..e469458 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -104,8 +104,8 @@ class PolicydTest(object): zfp.writestr(name, contents) return path - def _set_policy_with(self, rules): - rules_zip_path = self._make_zip_file_from('rules.zip', rules) + def _set_policy_with(self, rules, filename='rules.zip'): + rules_zip_path = self._make_zip_file_from(filename, rules) zaza_model.attach_resource(self.application_name, 'policyd-override', rules_zip_path) @@ -198,8 +198,8 @@ class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): def setUpClass(cls, application_name=None): """Run class setup for running KeystonePolicydTest tests.""" super(GenericPolicydTest, cls).setUpClass(application_name) - if (openstack_utils.get_os_release() < - openstack_utils.get_os_release('xenial_queens')): + if (openstack_utils.get_os_release() + < openstack_utils.get_os_release('xenial_queens')): raise unittest.SkipTest( "zaza.openstack.charm_tests.policyd.tests.GenericPolicydTest " "not valid before xenial_queens") @@ -242,7 +242,7 @@ class BasePolicydSpecialization(PolicydTest, class KeystonePolicydTest(BasePolicydSpecialization): - _rule = "{'identity:list_services': '!'}" + _rule = {'rule.yaml': "{'identity:list_services': '!'}"} def get_client_and_attempt_operation(self, keystone_session): ... etc. @@ -260,8 +260,8 @@ class BasePolicydSpecialization(PolicydTest, def setUpClass(cls, application_name=None): """Run class setup for running KeystonePolicydTest tests.""" super(BasePolicydSpecialization, cls).setUpClass(application_name) - if (openstack_utils.get_os_release() < - openstack_utils.get_os_release('xenial_queens')): + if (openstack_utils.get_os_release() + < openstack_utils.get_os_release('xenial_queens')): raise unittest.SkipTest( "zaza.openstack.charm_tests.policyd.tests.* " "not valid before xenial_queens") @@ -386,7 +386,7 @@ class BasePolicydSpecialization(PolicydTest, # now do the policyd override. logging.info("Doing policyd override with: {}".format(self._rule)) - self._set_policy_with({'rule.yaml': self._rule}) + self._set_policy_with(self._rule) zaza_model.block_until_all_units_idle() # now make sure the operation fails @@ -439,7 +439,7 @@ class BasePolicydSpecialization(PolicydTest, class KeystoneTests(BasePolicydSpecialization): """Test the policyd override using the keystone client.""" - _rule = "{'identity:list_services': '!'}" + _rule = {'rule.yaml': "{'identity:list_services': '!'}"} @classmethod def setUpClass(cls, application_name=None): @@ -468,7 +468,7 @@ class KeystoneTests(BasePolicydSpecialization): class NeutronApiTests(BasePolicydSpecialization): """Test the policyd override using the neutron client.""" - _rule = "{'get_network': '!'}" + _rule = {'rule.yaml': "{'get_network': '!'}"} @classmethod def setUpClass(cls, application_name=None): @@ -503,7 +503,7 @@ class NeutronApiTests(BasePolicydSpecialization): class GlanceTests(BasePolicydSpecialization): """Test the policyd override using the glance client.""" - _rule = "{'get_images': '!'}" + _rule = {'rule.yaml': "{'get_images': '!'}"} @classmethod def setUpClass(cls, application_name=None): @@ -537,7 +537,7 @@ class GlanceTests(BasePolicydSpecialization): class CinderTests(BasePolicydSpecialization): """Test the policyd override using the cinder client.""" - _rule = "{'volume:get_all': '!'}" + _rule = {'rule.yaml': "{'volume:get_all': '!'}"} @classmethod def setUpClass(cls, application_name=None): @@ -566,7 +566,7 @@ class CinderTests(BasePolicydSpecialization): class HeatTests(BasePolicydSpecialization): """Test the policyd override using the heat client.""" - _rule = "{'stacks:index': '!'}" + _rule = {'rule.yaml': "{'stacks:index': '!'}"} @classmethod def setUpClass(cls, application_name=None): From 28db85e06ce6b0080f3ddf80df6d442633256f1a Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 20 Nov 2019 17:21:57 +0000 Subject: [PATCH 195/898] Fix more policyd tests so that they can work o-s dashboard The difference in how openstack-dashboard and every other service uses policies continues to create special cases. This set of fixes deals with some more of those differences. --- .../charm_tests/openstack_dashboard/tests.py | 33 +++++++++++++++---- zaza/openstack/charm_tests/policyd/tests.py | 33 +++++++++++++------ 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index dd37080..991733f 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -14,6 +14,7 @@ """Encapsulate horizon (openstack-dashboard) charm testing.""" +import http.client import logging import requests import tenacity @@ -315,18 +316,29 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): logging.debug('Maybe enabling hardening for apache...') _app_config = zaza_model.get_application_config(self.application_name) logging.info(_app_config['harden']) + + # NOTE(ajkavanagh): it seems that apache2 doesn't start quickly enough + # for the test, and so it gets reset errors; repeat until either that + # stops or there is a failure + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, + min=5, max=10), + retry=tenacity.retry_if_exception_type( + http.client.RemoteDisconnected), + reraise=True) + def _do_request(): + return urllib.request.urlopen('http://{}/server-status' + .format(dashboard_ip)) + with self.config_change( {'harden': _app_config['harden'].get('value', '')}, {'harden': 'apache'}): try: - urllib.request.urlopen('http://{}/server-status' - .format(dashboard_ip)) + _do_request() except urllib.request.HTTPError as e: - if e.code == 404: - return - # test failed if it didn't return 404 - msg = "Apache mod_status check failed." - assert False, msg + # test failed if it didn't return 404 + msg = "Apache mod_status check failed." + self.assertEqual(e.code, 404, msg) + logging.info('OK') def test_501_security_checklist_action(self): """Verify expected result on a default install. @@ -386,6 +398,13 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): """Test the policyd override using the dashboard.""" + good = { + "identity/file1.yaml": "{'rule1': '!'}" + } + bad = { + "identity/file2.yaml": "{'rule': '!}" + } + path_infix = "keystone_policy.d" _rule = {'identity/rule.yaml': yaml.dump({ 'identity:list_domains': '!', 'identity:get_domain': '!', diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index e469458..d65c7b3 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -62,6 +62,13 @@ class PolicydTest(object): policyd: service: keystone """ + good = { + "file1.yaml": "{'rule1': '!'}" + } + bad = { + "file2.yaml": "{'rule': '!}" + } + path_infix = "" @classmethod def setUpClass(cls, application_name=None): @@ -115,9 +122,7 @@ class PolicydTest(object): def test_001_policyd_good_yaml(self): """Test that the policyd with a good zipped yaml file.""" - good = { - 'file1.yaml': "{'rule1': '!'}" - } + good = self.good good_zip_path = self._make_zip_file_from('good.zip', good) logging.info("Attaching good zip file as a resource.") zaza_model.attach_resource(self.application_name, @@ -127,8 +132,13 @@ class PolicydTest(object): logging.debug("Now setting config to true") self._set_config(True) # check that the file gets to the right location - path = os.path.join( - "/etc", self._service_name, "policy.d", 'file1.yaml') + if self.path_infix: + path = os.path.join( + "/etc", self._service_name, "policy.d", self.path_infix, + 'file1.yaml') + else: + path = os.path.join( + "/etc", self._service_name, "policy.d", 'file1.yaml') logging.info("Now checking for file contents: {}".format(path)) zaza_model.block_until_file_has_contents(self.application_name, path, @@ -162,9 +172,7 @@ class PolicydTest(object): def test_002_policyd_bad_yaml(self): """Test bad yaml file in the zip file is handled.""" - bad = { - "file2.yaml": "{'rule': '!}" - } + bad = self.bad bad_zip_path = self._make_zip_file_from('bad.zip', bad) logging.info("Attaching bad zip file as a resource") zaza_model.attach_resource(self.application_name, @@ -182,8 +190,13 @@ class PolicydTest(object): logging.debug("App status is valid for broken yaml file") zaza_model.block_until_all_units_idle() # now verify that no file got landed on the machine - path = os.path.join( - "/etc", self._service_name, "policy.d", 'file2.yaml') + if self.path_infix: + path = os.path.join( + "/etc", self._service_name, "policy.d", self.path_infix, + 'file2.yaml') + else: + path = os.path.join( + "/etc", self._service_name, "policy.d", 'file2.yaml') logging.info("Now checking that file {} is not present.".format(path)) zaza_model.block_until_file_missing(self.application_name, path) self._set_config(False) From 1b5613ac00f62f7d006065403dd11209038d4773 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 21 Nov 2019 09:22:57 +0000 Subject: [PATCH 196/898] Fix pep8 issue --- zaza/openstack/charm_tests/policyd/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index d65c7b3..f933a5d 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -62,6 +62,7 @@ class PolicydTest(object): policyd: service: keystone """ + good = { "file1.yaml": "{'rule1': '!'}" } From 4ac88e92bba4d51d0d205f9921441c5fb06fd3bb Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 21 Nov 2019 13:04:14 +0000 Subject: [PATCH 197/898] Fix nit re: copy/paste comment --- zaza/openstack/charm_tests/openstack_dashboard/tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 991733f..d15a603 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -148,8 +148,6 @@ def _login(dashboard_ip, domain, username, password): if expect not in response.text: msg = 'FAILURE code={} text="{}"'.format(response, response.text) - # NOTE(thedac) amulet.raise_status exits on exception. - # Raise a custom exception. logging.info("Yeah, wen't wrong: {}".format(msg)) raise FailedAuth(msg) logging.info("Logged into okay") From b00e570e57b32fda5736c656e03cb4f57e9d7e4c Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 7 Nov 2019 18:34:46 +0000 Subject: [PATCH 198/898] Add octavia policyd tests Providing a policy.d test for Octavia is a bit more complicated as the obvious policy is just to prevent listing of load balancers; unfortunately it doesn't work in the upstream, as the code doesn't check the policy in octavia. However, "showing" an load balancer does involve a policy check and so the test is more complicated than 'normal' policy tests in that a resource is created, then the policy override is checked and then the resource is deleted. --- requirements.txt | 4 +- zaza/openstack/charm_tests/policyd/tests.py | 142 +++++++++++++++++++- 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5d27b15..3b28f23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,8 @@ - aiounittest async_generator juju juju_wait -PyYAML<=4.2,>=3.0 +PyYAML<=4.2,>=3.0 flake8>=2.2.4,<=3.5.0 flake8-docstrings flake8-per-file-ignores @@ -35,6 +34,7 @@ python-swiftclient tenacity distro-info paramiko + # Documentation requirements sphinx sphinxcontrib-asyncio diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index f933a5d..e820803 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -35,13 +35,15 @@ import logging import os import shutil import tempfile +import tenacity import unittest import zipfile -import keystoneauth1 -import glanceclient.common.exceptions +from octaviaclient.api.v2 import octavia as octaviaclient import cinderclient.exceptions import heatclient.exc +import glanceclient.common.exceptions +import keystoneauth1 import zaza.model as zaza_model @@ -285,6 +287,28 @@ class BasePolicydSpecialization(PolicydTest, "not valid if {}.rule is not configured" .format(cls.__name__)) + def setup_for_attempt_operation(self, ip): + """Set-up for the policy override if needed. + + This method allows the test being performed in + get_client_and_attempt_operation() to have some setup done before the + test is performed. This is because the method + get_client_and_attempt_operation() is run twice; once to succeed and + once to fail. + + :param ip: the ip of for keystone. + :type ip: str + """ + pass + + def cleanup_for_attempt_operation(self, ip): + """Clean-up after a successful (or not) policy override operation. + + :param ip: the ip of for keystone. + :type ip: str + """ + pass + def get_client_and_attempt_operation(self, keystone_session): """Override this method to perform the operation. @@ -385,6 +409,10 @@ class BasePolicydSpecialization(PolicydTest, # note policyd override only works with Xenial-queens and so keystone # is already v3 + # Allow the overriden class to setup the environment before the policyd + # test is performed. + self.setup_for_attempt_operation(self.keystone_ips[0]) + # verify that the operation works before performing the policyd # override. zaza_model.block_until_wl_status_info_starts_with( @@ -394,6 +422,7 @@ class BasePolicydSpecialization(PolicydTest, try: self.get_client_and_attempt_operation(self.keystone_ips[0]) except Exception as e: + self.cleanup_for_attempt_operation(self.keystone_ips[0]) raise zaza_exceptions.PolicydError( 'Service action failed and should have passed. "{}"' .format(str(e))) @@ -418,6 +447,7 @@ class BasePolicydSpecialization(PolicydTest, logging.info("exception was: {}".format(e.__class__.__name__)) import traceback logging.info(traceback.format_exc()) + self.cleanup_for_attempt_operation(self.keystone_ips[0]) raise zaza_exceptions.PolicydError( 'Service action failed in an unexpected way: {}' .format(str(e))) @@ -446,6 +476,8 @@ class BasePolicydSpecialization(PolicydTest, 'Service action failed and should have passed after removing ' 'policy override: "{}"' .format(str(e))) + finally: + self.cleanup_for_attempt_operation(self.keystone_ips[0]) logging.info('OK') @@ -607,3 +639,109 @@ class HeatTests(BasePolicydSpecialization): list(heat_client.stacks.list()) except heatclient.exc.HTTPForbidden: raise PolicydOperationFailedException() + + +class OctaviaTests(BasePolicydSpecialization): + """Test the policyd override using the octavia client.""" + + _rule = "{'os_load-balancer_api:loadbalancer:get_one': '!'}" + + @classmethod + def setUpClass(cls, application_name=None): + """Run class setup for running OctaviaTests charm operation tests.""" + super(OctaviaTests, cls).setUpClass(application_name="octavia") + cls.application_name = "octavia" + + def setup_for_attempt_operation(self, ip): + """Create a loadbalancer. + + This is necessary so that the attempt is to show the load-balancer and + this is an operator that the policy can stop. Unfortunately, octavia, + whilst it has a policy for just listing load-balancers, unfortunately, + it doesn't work; whereas showing the load-balancer can be stopped. + + NB this only works if the setup phase of the octavia tests have been + completed. + + :param ip: the ip of for keystone. + :type ip: str + """ + logging.info("Setting up loadbalancer.") + auth = openstack_utils.get_overcloud_auth(address=ip) + sess = openstack_utils.get_keystone_session(auth) + + octavia_client = openstack_utils.get_octavia_session_client(sess) + neutron_client = openstack_utils.get_neutron_session_client(sess) + + resp = neutron_client.list_networks(name="private_lb_fip_network") + + vip_subnet_id = resp['networks'][0]['subnets'][0] + + res = octavia_client.load_balancer_create( + json={ + 'loadbalancer': { + 'description': 'Created by Zaza', + 'admin_state_up': True, + 'vip_subnet_id': vip_subnet_id, + 'name': 'zaza-lb-0', + }}) + self.lb_id = res['loadbalancer']['id'] + # now wait for it to get to the active state + + @tenacity.retry(wait=tenacity.wait_fixed(1), + reraise=True, stop=tenacity.stop_after_delay(900)) + def wait_for_lb_resource(client, resource_id): + resp = client.load_balancer_show(resource_id) + logging.info(resp['provisioning_status']) + assert resp['provisioning_status'] == 'ACTIVE', ( + 'load balancer resource has not reached ' + 'expected provisioning status: {}' + .format(resp)) + return resp + + logging.info('Awaiting loadbalancer to reach provisioning_status ' + '"ACTIVE"') + resp = wait_for_lb_resource(octavia_client, self.lb_id) + logging.info(resp) + logging.info("Setup loadbalancer complete.") + + def cleanup_for_attempt_operation(self, ip): + """Remove the loadbalancer. + + :param ip: the ip of for keystone. + :type ip: str + """ + logging.info("Deleting loadbalancer {}.".format(self.lb_id)) + auth = openstack_utils.get_overcloud_auth(address=ip) + sess = openstack_utils.get_keystone_session(auth) + + octavia_client = openstack_utils.get_octavia_session_client(sess) + octavia_client.load_balancer_delete(self.lb_id) + logging.info("Deleting loadbalancer in progress ...") + + @tenacity.retry(wait=tenacity.wait_fixed(1), + reraise=True, stop=tenacity.stop_after_delay(900)) + def wait_til_deleted(client, lb_id): + lb_list = client.load_balancer_list() + ids = [lb['id'] for lb in lb_list['loadbalancers']] + assert lb_id not in ids, 'load balancer still deleting' + + wait_til_deleted(octavia_client, self.lb_id) + logging.info("Deleted loadbalancer.") + + def get_client_and_attempt_operation(self, ip): + """Attempt to show the loadbalancer as a policyd override. + + This operation should pass normally, and fail when + the rule has been overriden (see the `rule` class variable. + + :param ip: the IP address to get the session against. + :type ip: str + :raises: PolicydOperationFailedException if operation fails. + """ + octavia_client = openstack_utils.get_octavia_session_client( + self.get_keystone_session_admin_user(ip)) + try: + octavia_client.load_balancer_show(self.lb_id) + except octaviaclient.OctaviaClientException: + raise PolicydOperationFailedException() From d5f8f762e8946b01b2d248d5dfc1cc7c294aa063 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 21 Nov 2019 14:55:28 +0000 Subject: [PATCH 199/898] Fix up octavia test to work with refactored _rule after openstack-dashboard test merge --- zaza/openstack/charm_tests/policyd/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index e820803..0c7ad3e 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -644,7 +644,7 @@ class HeatTests(BasePolicydSpecialization): class OctaviaTests(BasePolicydSpecialization): """Test the policyd override using the octavia client.""" - _rule = "{'os_load-balancer_api:loadbalancer:get_one': '!'}" + _rule = {'rule.yaml': "{'os_load-balancer_api:loadbalancer:get_one': '!'}"} @classmethod def setUpClass(cls, application_name=None): From 70282da7c18881becc2a01c07961a30f823df196 Mon Sep 17 00:00:00 2001 From: Sahid Orentino Ferdjaoui Date: Thu, 21 Nov 2019 14:57:20 +0000 Subject: [PATCH 200/898] vault: add pause/resume test case Signed-off-by: Sahid Orentino Ferdjaoui --- zaza/openstack/charm_tests/vault/tests.py | 57 ++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index ec398b7..4719c4b 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -16,7 +16,9 @@ """Collection of tests for vault.""" +import contextlib import hvac +import logging import time import unittest import uuid @@ -31,12 +33,15 @@ import zaza.openstack.utilities.openstack import zaza.model -class BaseVaultTest(unittest.TestCase): +class BaseVaultTest(test_utils.OpenStackBaseTest): """Base class for vault tests.""" @classmethod def setUpClass(cls): """Run setup for Vault tests.""" + cls.model_name = zaza.model.get_juju_model() + cls.lead_unit = zaza.model.get_lead_unit_name( + "vault", model_name=cls.model_name) cls.clients = vault_utils.get_clients() cls.vip_client = vault_utils.get_vip_client() if cls.vip_client: @@ -46,6 +51,39 @@ class BaseVaultTest(unittest.TestCase): vault_utils.auth_all(cls.clients, cls.vault_creds['root_token']) vault_utils.ensure_secret_backend(cls.clients[0]) + @contextlib.contextmanager + def pause_resume(self, services, pgrep_full=False): + """Override pause_resume for Vault behavior.""" + zaza.model.block_until_service_status( + self.lead_unit, + services, + 'running', + model_name=self.model_name) + zaza.model.block_until_unit_wl_status( + self.lead_unit, + 'active', + model_name=self.model_name) + zaza.model.block_until_all_units_idle(model_name=self.model_name) + zaza.model.run_action( + self.lead_unit, + 'pause', + model_name=self.model_name) + zaza.model.block_until_service_status( + self.lead_unit, + services, + 'blocked', # Service paused + model_name=self.model_name) + yield + zaza.model.run_action( + self.lead_unit, + 'resume', + model_name=self.model_name) + zaza.model.block_until_service_status( + self.lead_unit, + services, + 'blocked', # Service sealed + model_name=self.model_name) + class UnsealVault(BaseVaultTest): """Unseal Vault only. @@ -197,6 +235,23 @@ class VaultTest(BaseVaultTest): 'local-charm-policy', client.hvac_client.list_policies()) + def test_zzz_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped, then resume and check + they are started. + """ + # Restarting vault process will set it as sealed so it's + # important to have the test executed at the end. + vault_actions = zaza.model.get_actions( + 'vault') + if 'pause' not in vault_actions or 'resume' not in vault_actions: + raise unittest.SkipTest("The version of charm-vault tested does " + "not have pause/resume actions") + with self.pause_resume(['vault']): + logging.info("Testing pause resume") + self.assertTrue(self.clients[0].hvac_client.seal_status['sealed']) + if __name__ == '__main__': unittest.main() From 748915646a2fb5b22ca39aea37530618e61398fe Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 25 Nov 2019 10:49:50 +0000 Subject: [PATCH 201/898] First set of updates to set model name explicitly --- .../utilities/test_zaza_utilities_juju.py | 33 ++++--- zaza/openstack/utilities/generic.py | 8 +- zaza/openstack/utilities/juju.py | 89 +++++++++++++------ zaza/openstack/utilities/openstack.py | 83 +++++++++++------ 4 files changed, 148 insertions(+), 65 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_juju.py b/unit_tests/utilities/test_zaza_utilities_juju.py index d62534a..11bbcbe 100644 --- a/unit_tests/utilities/test_zaza_utilities_juju.py +++ b/unit_tests/utilities/test_zaza_utilities_juju.py @@ -93,7 +93,7 @@ class TestJujuUtils(ut_utils.BaseTestCase): def test_get_full_juju_status(self): self.assertEqual(juju_utils.get_full_juju_status(), self.juju_status) - self.model.get_status.assert_called_once_with() + self.model.get_status.assert_called_once_with(model_name=None) def test_get_machines_for_application(self): self.patch_object(juju_utils, "get_application_status") @@ -106,7 +106,7 @@ class TestJujuUtils(ut_utils.BaseTestCase): self.get_application_status.assert_called_once() # Subordinate application has no units - def _get_application_status(application): + def _get_application_status(application, model_name=None): _apps = { self.application: self.application_data, self.subordinate_application: @@ -155,7 +155,7 @@ class TestJujuUtils(ut_utils.BaseTestCase): self.application)), self.machine_data.get("instance-id")) self.get_machines_for_application.assert_called_once_with( - self.application) + self.application, model_name=None) def test_get_provider_type(self): self.patch_object(juju_utils, "get_cloud_configs") @@ -171,12 +171,17 @@ class TestJujuUtils(ut_utils.BaseTestCase): self.assertEqual(juju_utils.remote_run(self.unit, _cmd), self.run_output["Stdout"]) self.model.run_on_unit.assert_called_once_with( - self.unit, _cmd, timeout=None) + self.unit, _cmd, timeout=None, model_name=None) # Non-fatal failure self.model.run_on_unit.return_value = self.error_run_output - self.assertEqual(juju_utils.remote_run(self.unit, _cmd, fatal=False), - self.error_run_output["Stderr"]) + self.assertEqual( + juju_utils.remote_run( + self.unit, + _cmd, + fatal=False, + model_name=None), + self.error_run_output["Stderr"]) # Fatal failure with self.assertRaises(Exception): @@ -205,7 +210,8 @@ class TestJujuUtils(ut_utils.BaseTestCase): 'arelation') self.model.run_on_unit.assert_called_with( 'aunit/0', - 'relation-get --format=yaml -r "42" - "otherunit/0"') + 'relation-get --format=yaml -r "42" - "otherunit/0"', + model_name=None) self.yaml.safe_load.assert_called_with(str(data)) def test_get_relation_from_unit_fails(self): @@ -220,7 +226,8 @@ class TestJujuUtils(ut_utils.BaseTestCase): 'arelation') self.model.run_on_unit.assert_called_with( 'aunit/0', - 'relation-get --format=yaml -r "42" - "otherunit/0"') + 'relation-get --format=yaml -r "42" - "otherunit/0"', + model_name=None) self.assertFalse(self.yaml.safe_load.called) def test_leader_get(self): @@ -231,7 +238,7 @@ class TestJujuUtils(ut_utils.BaseTestCase): 'Code': 0, 'Stdout': str(data)} juju_utils.leader_get('application') self.model.run_on_leader.assert_called_with( - 'application', 'leader-get --format=yaml ') + 'application', 'leader-get --format=yaml ', model_name=None) self.yaml.safe_load.assert_called_with(str(data)) def test_leader_get_key(self): @@ -242,7 +249,7 @@ class TestJujuUtils(ut_utils.BaseTestCase): 'Code': 0, 'Stdout': data['foo']} juju_utils.leader_get('application', 'foo') self.model.run_on_leader.assert_called_with( - 'application', 'leader-get --format=yaml foo') + 'application', 'leader-get --format=yaml foo', model_name=None) self.yaml.safe_load.assert_called_with(data['foo']) def test_leader_get_fails(self): @@ -253,7 +260,8 @@ class TestJujuUtils(ut_utils.BaseTestCase): with self.assertRaises(Exception): juju_utils.leader_get('application') self.model.run_on_leader.assert_called_with( - 'application', 'leader-get --format=yaml ') + 'application', 'leader-get --format=yaml ', + model_name=None) self.assertFalse(self.yaml.safe_load.called) def test_get_machine_series(self): @@ -267,6 +275,7 @@ class TestJujuUtils(ut_utils.BaseTestCase): actual = juju_utils.get_machine_series('6') self._get_machine_status.assert_called_with( machine='6', - key='series' + key='series', + model_name=None ) self.assertEqual(expected, actual) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 7c66793..dbeb7d3 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -85,21 +85,23 @@ def get_unit_hostnames(units): return host_names -def get_pkg_version(application, pkg): +def get_pkg_version(application, pkg, model_name=None): """Return package version. :param application: Application name :type application: string :param pkg: Package name :type pkg: string + :param model_name: Name of model to query. + :type model_name: str :returns: List of package version :rtype: list """ versions = [] - units = model.get_units(application) + units = model.get_units(application, model_name=model_name) for unit in units: cmd = 'dpkg -l | grep {}'.format(pkg) - out = juju_utils.remote_run(unit.entity_id, cmd) + out = juju_utils.remote_run(unit.entity_id, cmd, model_name=model_name) versions.append(out.split('\n')[0].split()[2]) if len(set(versions)) != 1: raise Exception('Unexpected output from pkg version check') diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index a5edba2..fd7abc1 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -25,13 +25,15 @@ from zaza import ( from zaza.openstack.utilities import generic as generic_utils -def get_application_status(application=None, unit=None): +def get_application_status(application=None, unit=None, model_name=None): """Return the juju status for an application. :param application: Application name :type application: string :param unit: Specific unit :type unit: string + :param model_name: Name of model to query. + :type model_name: str :returns: Juju status output for an application :rtype: dict """ @@ -66,95 +68,112 @@ def get_cloud_configs(cloud=None): return generic_utils.get_yaml_config(cloud_config) -def get_full_juju_status(): +def get_full_juju_status(model_name=None): """Return the full juju status output. + :param model_name: Name of model to query. + :type model_name: str :returns: Full juju status output :rtype: dict """ - status = model.get_status() + status = model.get_status(model_name=model_name) return status -def get_machines_for_application(application): +def get_machines_for_application(application, model_name=None): """Return machines for a given application. :param application: Application name :type application: string + :param model_name: Name of model to query. + :type model_name: str :returns: machines for an application :rtype: Iterator[str] """ - status = get_application_status(application) + status = get_application_status(application, model_name=model_name) if not status: raise StopIteration # libjuju juju status no longer has units for subordinate charms # Use the application it is subordinate-to to find machines if status.get("units") is None and status.get("subordinate-to"): - status = get_application_status(status.get("subordinate-to")[0]) + status = get_application_status(status.get("subordinate-to")[0], + model_name=model_name) for unit in status.get("units").keys(): yield status.get("units").get(unit).get("machine") -def get_unit_name_from_host_name(host_name, application): +def get_unit_name_from_host_name(host_name, application, model_name=None): """Return the juju unit name corresponding to a hostname. :param host_name: Host name to map to unit name. :type host_name: string :param application: Application name :type application: string + :param model_name: Name of model to query. + :type model_name: str """ # Assume that a juju managed hostname always ends in the machine number. machine_number = host_name.split('-')[-1] unit_names = [ u.entity_id - for u in model.get_units(application_name=application) + for u in model.get_units(application_name=application, + model_name=model_name) if int(u.data['machine-id']) == int(machine_number)] return unit_names[0] -def get_machine_status(machine, key=None): +def get_machine_status(machine, key=None, model_name=None): """Return the juju status for a machine. :param machine: Machine number :type machine: string :param key: Key option requested :type key: string + :param model_name: Name of model to query. + :type model_name: str :returns: Juju status output for a machine :rtype: dict """ - status = get_full_juju_status() + status = get_full_juju_status(model_name=model_name) status = status.machines.get(machine) if key: status = status.get(key) return status -def get_machine_series(machine): +def get_machine_series(machine, model_name=None): """Return the juju series for a machine. :param machine: Machine number :type machine: string + :param model_name: Name of model to query. + :type model_name: str :returns: Juju series :rtype: string """ return get_machine_status( machine=machine, - key='series' + key='series', + model_name=model_name ) -def get_machine_uuids_for_application(application): +def get_machine_uuids_for_application(application, model_name=None): """Return machine uuids for a given application. :param application: Application name :type application: string + :param model_name: Name of model to query. + :type model_name: str :returns: machine uuuids for an application :rtype: Iterator[str] """ - for machine in get_machines_for_application(application): - yield get_machine_status(machine, key="instance-id") + for machine in get_machines_for_application(application, + model_name=model_name): + yield get_machine_status(machine, key="instance-id", + model_name=model_name) def get_provider_type(): @@ -175,7 +194,7 @@ def get_provider_type(): return "openstack" -def remote_run(unit, remote_cmd, timeout=None, fatal=None): +def remote_run(unit, remote_cmd, timeout=None, fatal=None, model_name=None): """Run command on unit and return the output. NOTE: This function is pre-deprecated. As soon as libjuju unit.run is able @@ -187,13 +206,19 @@ def remote_run(unit, remote_cmd, timeout=None, fatal=None): :type arg: int :param fatal: Command failure condidered fatal or not :type fatal: boolean + :param model_name: Name of model to query. + :type model_name: str :returns: Juju run output :rtype: string :raises: model.CommandRunFailed """ if fatal is None: fatal = True - result = model.run_on_unit(unit, remote_cmd, timeout=timeout) + result = model.run_on_unit( + unit, + remote_cmd, + timeout=timeout, + model_name=model_name) if result: if int(result.get("Code")) == 0: return result.get("Stdout") @@ -203,7 +228,7 @@ def remote_run(unit, remote_cmd, timeout=None, fatal=None): return result.get("Stderr") -def _get_unit_names(names): +def _get_unit_names(names, model_name=None): """Resolve given application names to first unit name of said application. Helper function that resolves application names to first unit name of @@ -211,6 +236,8 @@ def _get_unit_names(names): :param names: List of units/applications to translate :type names: list(str) + :param model_name: Name of model to query. + :type model_name: str :returns: List of units :rtype: list(str) """ @@ -219,11 +246,14 @@ def _get_unit_names(names): if '/' in name: result.append(name) else: - result.append(model.get_first_unit_name(name)) + result.append(model.get_first_unit_name( + name, + model_name=model_name)) return result -def get_relation_from_unit(entity, remote_entity, remote_interface_name): +def get_relation_from_unit(entity, remote_entity, remote_interface_name, + model_name=None): """Get relation data passed between two units. Get relation data for relation with `remote_interface_name` between @@ -240,6 +270,8 @@ def get_relation_from_unit(entity, remote_entity, remote_interface_name): :param remote_interface_name: Name of interface to query on remote end of relation :type remote_interface_name: str + :param model_name: Name of model to query. + :type model_name: str :returns: dict with relation data :rtype: dict :raises: model.CommandRunFailed @@ -247,27 +279,34 @@ def get_relation_from_unit(entity, remote_entity, remote_interface_name): application = entity.split('/')[0] remote_application = remote_entity.split('/')[0] rid = model.get_relation_id(application, remote_application, - remote_interface_name=remote_interface_name) - (unit, remote_unit) = _get_unit_names([entity, remote_entity]) + remote_interface_name=remote_interface_name, + model_name=model_name) + (unit, remote_unit) = _get_unit_names( + [entity, remote_entity], + model_name=model_name) cmd = 'relation-get --format=yaml -r "{}" - "{}"' .format(rid, remote_unit) - result = model.run_on_unit(unit, cmd) + result = model.run_on_unit(unit, cmd, model_name=model_name) if result and int(result.get('Code')) == 0: return yaml.safe_load(result.get('Stdout')) else: raise model.CommandRunFailed(cmd, result) -def leader_get(application, key=''): +def leader_get(application, key='', model_name=None): """Get leader settings from leader unit of named application. :param application: Application to get leader settings from. :type application: str + :param key: Key option requested + :type key: string + :param model_name: Name of model to query. + :type model_name: str :returns: dict with leader settings :rtype: dict :raises: model.CommandRunFailed """ cmd = 'leader-get --format=yaml {}'.format(key) - result = model.run_on_leader(application, cmd) + result = model.run_on_leader(application, cmd, model_name=model_name) if result and int(result.get('Code')) == 0: return yaml.safe_load(result.get('Stdout')) else: diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 9a755e8..cb1b6fd 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -321,13 +321,16 @@ def get_aodh_session_client(session): return aodh_client.Client(session=session) -def get_keystone_scope(): +def get_keystone_scope(model_name=None): """Return Keystone scope based on OpenStack release of the overcloud. + :param model_name: Name of model to query. + :type model_name: str :returns: String keystone scope :rtype: string """ - os_version = get_current_os_versions("keystone")["keystone"] + os_version = get_current_os_versions("keystone", + model_name=model_name)["keystone"] # Keystone policy.json shipped the charm with liberty requires a domain # scoped token. Bug #1649106 if os_version == "liberty": @@ -362,17 +365,20 @@ def get_keystone_session(openrc_creds, scope='PROJECT', verify=None): return session.Session(auth=auth, verify=verify) -def get_overcloud_keystone_session(verify=None): +def get_overcloud_keystone_session(verify=None, model_name=None): """Return Over cloud keystone session. :param verify: Control TLS certificate verification behaviour :type verify: any + :param model_name: Name of model to query. + :type model_name: str :returns keystone_session: keystoneauth1.session.Session object :rtype: keystoneauth1.session.Session """ - return get_keystone_session(get_overcloud_auth(), - scope=get_keystone_scope(), - verify=verify) + return get_keystone_session( + get_overcloud_auth(model_name=model_name), + scope=get_keystone_scope(model_name=model_name), + verify=verify) def get_undercloud_keystone_session(verify=None): @@ -572,7 +578,6 @@ def add_interface_to_netplan(server_name, mac_address): unit_name = juju_utils.get_unit_name_from_host_name( server_name, application_name) - run_cmd_nic = "ip -f link -br -o addr|grep {}".format(mac_address) interface = model.run_on_unit(unit_name, run_cmd_nic) interface = interface['Stdout'].split(' ')[0] @@ -1394,11 +1399,13 @@ def get_os_code_info(package, pkg_version): return OPENSTACK_CODENAMES[vers] -def get_current_os_versions(deployed_applications): +def get_current_os_versions(deployed_applications, model_name=None): """Determine OpenStack codename of deployed applications. :param deployed_applications: List of deployed applications :type deployed_applications: list + :param model_name: Name of model to query. + :type model_name: str :returns: List of aplication to codenames dictionaries :rtype: list """ @@ -1408,7 +1415,8 @@ def get_current_os_versions(deployed_applications): continue version = generic_utils.get_pkg_version(application['name'], - application['type']['pkg']) + application['type']['pkg'], + model_name=model_name) versions[application['name']] = ( get_os_code_info(application['type']['pkg'], version)) return versions @@ -1473,17 +1481,21 @@ def get_os_release(release_pair=None): return index -def get_application_config_option(application, option): +def get_application_config_option(application, option, model_name=None): """Return application configuration. :param application: Name of application :type application: string :param option: Specific configuration option :type option: string + :param model_name: Name of model to query. + :type model_name: str :returns: Value of configuration option :rtype: Configuration option value type """ - application_config = model.get_application_config(application) + application_config = model.get_application_config( + application, + model_name=model_name) try: return application_config.get(option).get('value') except AttributeError: @@ -1557,27 +1569,39 @@ def get_undercloud_auth(): # Openstack Client helpers -def get_keystone_ip(): +def get_keystone_ip(model_name=None): """Return the IP address to use when communicating with keystone api. + :param model_name: Name of model to query. + :type model_name: str :returns: IP address :rtype: str """ - if get_application_config_option('keystone', 'vip'): - return get_application_config_option('keystone', 'vip') - unit = model.get_units('keystone')[0] + vip_option = get_application_config_option( + 'keystone', + 'vip', + model_name=model_name) + if vip_option: + return vip_option + unit = model.get_units('keystone', model_name=model_name)[0] return unit.public_address -def get_keystone_api_version(): +def get_keystone_api_version(model_name=None): """Return the keystone api version. + :param model_name: Name of model to query. + :type model_name: str :returns: Keystone's api version :rtype: int """ - os_version = get_current_os_versions('keystone')['keystone'] - api_version = get_application_config_option('keystone', - 'preferred-api-version') + os_version = get_current_os_versions( + 'keystone', + model_name=model_name)['keystone'] + api_version = get_application_config_option( + 'keystone', + 'preferred-api-version', + model_name=model_name) if os_version >= 'queens': api_version = 3 elif api_version is None: @@ -1586,15 +1610,21 @@ def get_keystone_api_version(): return int(api_version) -def get_overcloud_auth(address=None): +def get_overcloud_auth(address=None, model_name=None): """Get overcloud OpenStack authentication from the environment. + :param model_name: Name of model to query. + :type model_name: str :returns: Dictionary of authentication settings :rtype: dict """ tls_rid = model.get_relation_id('keystone', 'vault', + model_name=model_name, remote_interface_name='certificates') - ssl_config = get_application_config_option('keystone', 'ssl_cert') + ssl_config = get_application_config_option( + 'keystone', + 'ssl_cert', + model_name=model_name) if tls_rid or ssl_config: transport = 'https' port = 35357 @@ -1603,11 +1633,14 @@ def get_overcloud_auth(address=None): port = 5000 if not address: - address = get_keystone_ip() + address = get_keystone_ip(model_name=model_name) - password = juju_utils.leader_get('keystone', 'admin_passwd') + password = juju_utils.leader_get( + 'keystone', + 'admin_passwd', + model_name=model_name) - if get_keystone_api_version() == 2: + if get_keystone_api_version(model_name=model_name) == 2: # V2 Explicitly, or None when charm does not possess the config key logging.info('Using keystone API V2 for overcloud auth') auth_settings = { @@ -1633,7 +1666,7 @@ def get_overcloud_auth(address=None): 'API_VERSION': 3, } if tls_rid: - unit = model.get_first_unit_name('keystone') + unit = model.get_first_unit_name('keystone', model_name=model_name) model.scp_from_unit( unit, KEYSTONE_REMOTE_CACERT, From 9c34bc42da2c4f72e685778f8fe7994bdf686dac Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 25 Nov 2019 17:40:03 +0000 Subject: [PATCH 202/898] Expose model_name option in get_keystone_session_from_relation --- unit_tests/utilities/test_zaza_utilities_openstack.py | 6 ++++-- zaza/openstack/utilities/openstack.py | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 4576564..7e9a57d 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1108,7 +1108,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.get_relation_from_unit.assert_called_once_with( 'swift-proxy', 'keystone', - 'identity-service') + 'identity-service', + model_name=None) self.get_keystone_session.assert_called_once_with( { 'OS_AUTH_URL': 'http://10.5.0.61:5000/v3', @@ -1155,7 +1156,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.get_relation_from_unit.assert_called_once_with( 'swift-proxy', 'keystone', - 'identity-service') + 'identity-service', + model_name=None) self.get_keystone_session.assert_called_once_with( { 'OS_AUTH_URL': 'http://10.5.0.36:5000/v2.0', diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index cb1b6fd..5e5addc 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2450,7 +2450,8 @@ def get_keystone_session_from_relation(client_app, identity_app='keystone', relation_name='identity-service', scope='PROJECT', - verify=None): + verify=None, + model_name=None): """Extract credentials information from a relation & return a session. :param client_app: Name of application receiving credentials. @@ -2472,10 +2473,11 @@ def get_keystone_session_from_relation(client_app, relation = juju_utils.get_relation_from_unit( client_app, identity_app, - relation_name) + relation_name, + model_name=model_name) api_version = int(relation.get('api_version', 2)) - creds = get_overcloud_auth() + creds = get_overcloud_auth(model_name=model_name) creds['OS_USERNAME'] = relation['service_username'] creds['OS_PASSWORD'] = relation['service_password'] creds['OS_PROJECT_NAME'] = relation['service_tenant'] From bd903fa32592e2adf1d7620636545f7bfebec46c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 25 Nov 2019 17:41:29 +0000 Subject: [PATCH 203/898] Add docstring entry --- zaza/openstack/utilities/openstack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 5e5addc..1776296 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2467,6 +2467,8 @@ def get_keystone_session_from_relation(client_app, False - do not verify, None - defer to requests library to find certs, str - path to a CA cert bundle) + :param model_name: Name of model to query. + :type model_name: str :returns: Keystone session object :rtype: keystoneauth1.session.Session object """ From b871907e84500fafaa7754affebfe890dd720bb6 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 27 Nov 2019 08:58:03 +0100 Subject: [PATCH 204/898] neutron: Do not run tear down on test failure Fixes #120 --- zaza/openstack/charm_tests/neutron/tests.py | 24 +++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 7433793..e7d609d 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -135,17 +135,24 @@ class NeutronNetworkingTest(unittest.TestCase): openstack_utils.get_overcloud_keystone_session()) cls.nova_client = ( openstack_utils.get_nova_session_client(cls.keystone_session)) + # NOTE(fnordahl): in the event of a test failure we do not want to run + # tear down code as it will make debugging a problem virtually + # impossible. To alleviate each test method will set the + # `run_tearDown` instance variable at the end which will let us run + # tear down only when there were no failure. + cls.run_tearDown = False @classmethod def tearDown(cls): """Remove test resources.""" - logging.info('Running teardown') - for server in cls.nova_client.servers.list(): - if server.name.startswith(cls.RESOURCE_PREFIX): - openstack_utils.delete_resource( - cls.nova_client.servers, - server.id, - msg="server") + if cls.run_tearDown: + logging.info('Running teardown') + for server in cls.nova_client.servers.list(): + if server.name.startswith(cls.RESOURCE_PREFIX): + openstack_utils.delete_resource( + cls.nova_client.servers, + server.id, + msg="server") def test_instances_have_networking(self): """Validate North/South and East/West networking.""" @@ -176,6 +183,9 @@ class NeutronNetworkingTest(unittest.TestCase): self.validate_instance_can_reach_router(instance_1, verify) self.validate_instance_can_reach_router(instance_2, verify) + # If we get here, it means the tests passed + self.run_tearDown = True + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(8)) def validate_instance_can_reach_other(self, From 39cb8ea0ccbf2c2fc8894f9bb721509e68d54041 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 27 Nov 2019 09:05:01 +0000 Subject: [PATCH 205/898] Add support for tests defining resource_cleanup Add support for tests having a resource_cleanup method. Whether this is run or not is dictated by the class variable 'run_resource_cleanup'. By default this will be True but any test can switch it to False so that resources created by the tests are preserved. Helpful for debugging failures. FWIW future work might include automatically disabling resource cleanup if a test fail is detected. --- zaza/openstack/charm_tests/test_utils.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index eff6b41..74d880f 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -93,6 +93,24 @@ def audit_assertions(action, class OpenStackBaseTest(unittest.TestCase): """Generic helpers for testing OpenStack API charms.""" + @classmethod + def resource_cleanup(cls): + """Cleanup any resources created during the test run. + + Override this method with a method which removes any resources + which were created during the test run. If the test sets + "self.run_resource_cleanup = False" then cleanup will be + skipped. + """ + pass + + @classmethod + def tearDown(cls): + """Run teardown for test class.""" + if cls.run_resource_cleanup: + logging.info('Running resource cleanup') + cls.resource_cleanup() + @classmethod def setUpClass(cls, application_name=None): """Run setup for test class to create common resourcea.""" @@ -106,6 +124,7 @@ class OpenStackBaseTest(unittest.TestCase): cls.lead_unit = model.get_lead_unit_name( cls.application_name, model_name=cls.model_name) + cls.run_resource_cleanup = True logging.debug('Leader unit is {}'.format(cls.lead_unit)) @contextlib.contextmanager From e839a19321e4f86aa47e9a51c34971f2ed7737fb Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 27 Nov 2019 10:29:06 +0000 Subject: [PATCH 206/898] Flip default for cleanup to False --- zaza/openstack/charm_tests/test_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 74d880f..c800bee 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -99,8 +99,8 @@ class OpenStackBaseTest(unittest.TestCase): Override this method with a method which removes any resources which were created during the test run. If the test sets - "self.run_resource_cleanup = False" then cleanup will be - skipped. + "self.run_resource_cleanup = True" then cleanup will be + performed. """ pass @@ -124,7 +124,7 @@ class OpenStackBaseTest(unittest.TestCase): cls.lead_unit = model.get_lead_unit_name( cls.application_name, model_name=cls.model_name) - cls.run_resource_cleanup = True + cls.run_resource_cleanup = False logging.debug('Leader unit is {}'.format(cls.lead_unit)) @contextlib.contextmanager From b6c649c00b337babf51a7d746cfe189c12a8e78a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 28 Nov 2019 14:59:46 +0000 Subject: [PATCH 207/898] Guard against run_resource_cleanup being unset A test may not call the base class setup so may be missing run_resource_cleanup in which case teardown will fail. Guard against that and default to False. --- zaza/openstack/charm_tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index c800bee..57e6e6c 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -107,7 +107,7 @@ class OpenStackBaseTest(unittest.TestCase): @classmethod def tearDown(cls): """Run teardown for test class.""" - if cls.run_resource_cleanup: + if getattr(cls, 'run_resource_cleanup', False): logging.info('Running resource cleanup') cls.resource_cleanup() From 5e9440dd9715435030fb2f713f789dd1a1b9254c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 28 Nov 2019 15:17:32 +0000 Subject: [PATCH 208/898] Define run_resource_cleanup in OpenStackBaseTest so all sub-classes inherit it --- zaza/openstack/charm_tests/test_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 57e6e6c..6d7376f 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -93,6 +93,8 @@ def audit_assertions(action, class OpenStackBaseTest(unittest.TestCase): """Generic helpers for testing OpenStack API charms.""" + run_resource_cleanup = False + @classmethod def resource_cleanup(cls): """Cleanup any resources created during the test run. @@ -107,7 +109,7 @@ class OpenStackBaseTest(unittest.TestCase): @classmethod def tearDown(cls): """Run teardown for test class.""" - if getattr(cls, 'run_resource_cleanup', False): + if cls.run_resource_cleanup: logging.info('Running resource cleanup') cls.resource_cleanup() @@ -124,7 +126,6 @@ class OpenStackBaseTest(unittest.TestCase): cls.lead_unit = model.get_lead_unit_name( cls.application_name, model_name=cls.model_name) - cls.run_resource_cleanup = False logging.debug('Leader unit is {}'.format(cls.lead_unit)) @contextlib.contextmanager From d004a4152d048ae72c51393da7aabcb39c71c339 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Thu, 28 Nov 2019 16:45:44 +0100 Subject: [PATCH 209/898] rename cinder backup swift to cinder backup swift proxy (#134) --- .../__init__.py | 2 +- .../setup.py | 15 ++++++++------- .../tests.py | 0 3 files changed, 9 insertions(+), 8 deletions(-) rename zaza/openstack/charm_tests/{cinder_backup_swift => cinder_backup_swift_proxy}/__init__.py (97%) rename zaza/openstack/charm_tests/{cinder_backup_swift => cinder_backup_swift_proxy}/setup.py (78%) rename zaza/openstack/charm_tests/{cinder_backup_swift => cinder_backup_swift_proxy}/tests.py (100%) diff --git a/zaza/openstack/charm_tests/cinder_backup_swift/__init__.py b/zaza/openstack/charm_tests/cinder_backup_swift_proxy/__init__.py similarity index 97% rename from zaza/openstack/charm_tests/cinder_backup_swift/__init__.py rename to zaza/openstack/charm_tests/cinder_backup_swift_proxy/__init__.py index 3d7e48c..e093594 100644 --- a/zaza/openstack/charm_tests/cinder_backup_swift/__init__.py +++ b/zaza/openstack/charm_tests/cinder_backup_swift_proxy/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Collection of code for setting up and testing cinder-backup-swift.""" +"""Collection of code for setting up and testing cinder-backup-swift-proxy.""" diff --git a/zaza/openstack/charm_tests/cinder_backup_swift/setup.py b/zaza/openstack/charm_tests/cinder_backup_swift_proxy/setup.py similarity index 78% rename from zaza/openstack/charm_tests/cinder_backup_swift/setup.py rename to zaza/openstack/charm_tests/cinder_backup_swift_proxy/setup.py index d84ad2e..d4c92e4 100644 --- a/zaza/openstack/charm_tests/cinder_backup_swift/setup.py +++ b/zaza/openstack/charm_tests/cinder_backup_swift_proxy/setup.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Code for configuring cinder-backup-swift.""" +"""Code for configuring cinder-backup-swift-proxy.""" import zaza.model as zaza_model import zaza.openstack.charm_tests.test_utils def configure_cinder_backup(): - """Configure cinder-backup-swift.""" + """Configure cinder-backup-swift-proxy.""" keystone_ip = zaza_model.get_app_ips( 'swift-keystone')[0] swift_ip = zaza_model.get_app_ips( @@ -32,17 +32,18 @@ def configure_cinder_backup(): else: auth_url = 'http://{}:5000/v3'.format(keystone_ip) endpoint_url = 'http://{}:8080/v1/AUTH'.format(swift_ip) - cinder_backup_swift_conf = { + cinder_backup_swift_proxy_conf = { 'endpoint-url': endpoint_url, 'auth-url': auth_url } - juju_service = 'cinder-backup-swift' - zaza_model.set_application_config(juju_service, cinder_backup_swift_conf) + juju_service = 'cinder-backup-swift-proxy' + zaza_model.set_application_config(juju_service, + cinder_backup_swift_proxy_conf) zaza_model.wait_for_agent_status() zaza_model.wait_for_application_states() _singleton = zaza.openstack.charm_tests.test_utils.OpenStackBaseTest() _singleton.setUpClass() - with _singleton.config_change(cinder_backup_swift_conf, - cinder_backup_swift_conf): + with _singleton.config_change(cinder_backup_swift_proxy_conf, + cinder_backup_swift_proxy_conf): # wait for configuration to be applied then return pass diff --git a/zaza/openstack/charm_tests/cinder_backup_swift/tests.py b/zaza/openstack/charm_tests/cinder_backup_swift_proxy/tests.py similarity index 100% rename from zaza/openstack/charm_tests/cinder_backup_swift/tests.py rename to zaza/openstack/charm_tests/cinder_backup_swift_proxy/tests.py From e63fb86df2f3d96104f476331d288e4d30309313 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 29 Nov 2019 11:04:53 +0000 Subject: [PATCH 210/898] Make OpenStackBaseTest model_alias aware --- zaza/openstack/charm_tests/test_utils.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index c800bee..4f2f712 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -112,10 +112,15 @@ class OpenStackBaseTest(unittest.TestCase): cls.resource_cleanup() @classmethod - def setUpClass(cls, application_name=None): + def setUpClass(cls, application_name=None, model_alias=None): """Run setup for test class to create common resourcea.""" - cls.keystone_session = openstack_utils.get_overcloud_keystone_session() - cls.model_name = model.get_juju_model() + cls.model_aliases = model.get_juju_model_aliases() + if model_alias: + cls.model_name = cls.model_aliases[model_alias] + else: + cls.model_name = model.get_juju_model() + cls.keystone_session = openstack_utils.get_overcloud_keystone_session( + model_name=cls.model_name) cls.test_config = lifecycle_utils.get_charm_config(fatal=False) if application_name: cls.application_name = application_name From d2904e228d4f4e7af5b8741a26f202009c8186af Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 29 Nov 2019 09:31:05 +0000 Subject: [PATCH 211/898] Add Swift Global Replication Tests --- unit_tests/utilities/swift_test_data.py | 69 ++++ .../utilities/test_zaza_utilities_swift.py | 187 +++++++++++ zaza/openstack/charm_tests/swift/tests.py | 115 +++++++ zaza/openstack/utilities/swift.py | 295 ++++++++++++++++++ 4 files changed, 666 insertions(+) create mode 100644 unit_tests/utilities/swift_test_data.py create mode 100644 unit_tests/utilities/test_zaza_utilities_swift.py create mode 100644 zaza/openstack/utilities/swift.py diff --git a/unit_tests/utilities/swift_test_data.py b/unit_tests/utilities/swift_test_data.py new file mode 100644 index 0000000..ac3145a --- /dev/null +++ b/unit_tests/utilities/swift_test_data.py @@ -0,0 +1,69 @@ +# flake8: noqa + +SWIFT_GET_NODES_STDOUT = """ +Account 23934cb1850c4d28b1ca113a24c0e46b +Container zaza-swift-gr-tests-f3129278-container +Object zaza_test_object.txt + + +Partition 146 +Hash 928c2f8006efeeb4b1164f4cce035887 + +Server:Port Device 10.5.0.38:6000 loop0 +Server:Port Device 10.5.0.4:6000 loop0 +Server:Port Device 10.5.0.9:6000 loop0 [Handoff] +Server:Port Device 10.5.0.34:6000 loop0 [Handoff] +Server:Port Device 10.5.0.15:6000 loop0 [Handoff] +Server:Port Device 10.5.0.18:6000 loop0 [Handoff] + + +curl -g -I -XHEAD "http://10.5.0.38:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" +curl -g -I -XHEAD "http://10.5.0.4:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" +curl -g -I -XHEAD "http://10.5.0.9:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff] +curl -g -I -XHEAD "http://10.5.0.34:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff] +curl -g -I -XHEAD "http://10.5.0.15:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff] +curl -g -I -XHEAD "http://10.5.0.18:6000/loop0/146/23934cb1850c4d28b1ca113a24c0e46b/zaza-swift-gr-tests-f3129278-container/zaza_test_object.txt" # [Handoff] + + +Use your own device location of servers: +such as "export DEVICE=/srv/node" +ssh 10.5.0.38 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" +ssh 10.5.0.4 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" +ssh 10.5.0.9 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff] +ssh 10.5.0.34 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff] +ssh 10.5.0.15 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff] +ssh 10.5.0.18 "ls -lah ${DEVICE:-/srv/node*}/loop0/objects/146/887/928c2f8006efeeb4b1164f4cce035887" # [Handoff] + +note: `/srv/node*` is used as default value of `devices`, the real value is set in the config file on each storage node. +""" + +STORAGE_TOPOLOGY = { + '10.5.0.18': { + 'app_name': 'swift-storage-region1-zone1', + 'unit': "swift-storage-region1-zone1/0", + 'region': 1, + 'zone': 1}, + '10.5.0.34': { + 'app_name': 'swift-storage-region1-zone2', + 'unit': "swift-storage-region1-zone2/0", + 'region': 1, + 'zone': 2}, + '10.5.0.4': { + 'app_name': 'swift-storage-region1-zone3', + 'unit': "swift-storage-region1-zone3/0", + 'region': 1, + 'zone': 3}, + '10.5.0.9': { + 'app_name': 'swift-storage-region2-zone1', + 'unit': "swift-storage-region2-zone1/0", + 'region': 2, + 'zone': 1}, + '10.5.0.15': { + 'app_name': 'swift-storage-region2-zone2', + 'unit': "swift-storage-region2-zone2/0", + 'region': 2, 'zone': 2}, + '10.5.0.38': { + 'app_name': 'swift-storage-region2-zone3', + 'unit': "swift-storage-region2-zone3/0", + 'region': 2, + 'zone': 3}} diff --git a/unit_tests/utilities/test_zaza_utilities_swift.py b/unit_tests/utilities/test_zaza_utilities_swift.py new file mode 100644 index 0000000..8c87556 --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_swift.py @@ -0,0 +1,187 @@ +import copy +import mock +import unit_tests.utils as ut_utils +import uuid + +import zaza.model +import zaza.openstack.utilities.swift as swift_utils +import zaza.openstack.utilities.juju as juju_utils + +import unit_tests.utilities.swift_test_data as swift_test_data + + +class TestSwiftUtils(ut_utils.BaseTestCase): + + def setUp(self): + super(TestSwiftUtils, self).setUp() + + def test_ObjectReplica_init(self): + obj_rep = swift_utils.ObjectReplica( + "Server:Port Device 10.5.0.38:6000 loop0") + self.assertEqual( + obj_rep.server, + "10.5.0.38") + self.assertEqual( + obj_rep.port, + "6000") + self.assertEqual( + obj_rep.device, + "loop0") + self.assertFalse(obj_rep.handoff_device) + obj_rep = swift_utils.ObjectReplica( + "Server:Port Device 10.5.0.9:6000 loop0 [Handoff]") + self.assertTrue(obj_rep.handoff_device) + + def test_ObjectReplicas(self): + self.patch_object(zaza.model, 'run_on_leader') + self.run_on_leader.return_value = { + 'Stdout': swift_test_data.SWIFT_GET_NODES_STDOUT} + obj_replicas = swift_utils.ObjectReplicas( + 'swift-proxy-region1', + 'account123', + 'my-container', + 'my-object', + swift_test_data.STORAGE_TOPOLOGY, + 'my-model') + self.assertEqual( + sorted(obj_replicas.hand_off_ips), + ['10.5.0.15', '10.5.0.18', '10.5.0.34', '10.5.0.9']) + self.assertEqual( + sorted(obj_replicas.storage_ips), + ['10.5.0.38', '10.5.0.4']) + self.assertEqual( + obj_replicas.placements, + [ + { + 'app_name': 'swift-storage-region2-zone3', + 'region': 2, + 'unit': 'swift-storage-region2-zone3/0', + 'zone': 3}, + { + 'app_name': 'swift-storage-region1-zone3', + 'region': 1, + 'unit': 'swift-storage-region1-zone3/0', + 'zone': 3}]) + self.assertEqual( + obj_replicas.distinct_regions, + [1, 2]) + self.assertEqual( + sorted(obj_replicas.all_zones), + [(1, 3), (2, 3)]) + self.assertEqual( + sorted(obj_replicas.distinct_zones), + [(1, 3), (2, 3)]) + + def test_get_swift_storage_topology(self): + unit_r1z1_mock = mock.MagicMock(public_address='10.5.0.18') + unit_r1z2_mock = mock.MagicMock(public_address='10.5.0.34') + unit_r1z3_mock = mock.MagicMock(public_address='10.5.0.4') + unit_r2z1_mock = mock.MagicMock(public_address='10.5.0.9') + unit_r2z2_mock = mock.MagicMock(public_address='10.5.0.15') + unit_r2z3_mock = mock.MagicMock(public_address='10.5.0.38') + app_units = { + 'swift-storage-region1-zone1': [unit_r1z1_mock], + 'swift-storage-region1-zone2': [unit_r1z2_mock], + 'swift-storage-region1-zone3': [unit_r1z3_mock], + 'swift-storage-region2-zone1': [unit_r2z1_mock], + 'swift-storage-region2-zone2': [unit_r2z2_mock], + 'swift-storage-region2-zone3': [unit_r2z3_mock]} + + expected_topology = copy.deepcopy(swift_test_data.STORAGE_TOPOLOGY) + self.patch_object(juju_utils, 'get_full_juju_status') + self.patch_object(zaza.model, 'get_application_config') + self.patch_object(zaza.model, 'get_units') + juju_status = mock.MagicMock() + juju_status.applications = {} + self.get_full_juju_status.return_value = juju_status + + for app_name, units in app_units.items(): + expected_topology[units[0].public_address]['unit'] = units[0] + + app_config = {} + for app_name in app_units.keys(): + juju_status.applications[app_name] = {'charm': 'cs:swift-storage'} + region = int(app_name.split('-')[2].replace('region', '')) + zone = int(app_name.split('-')[3].replace('zone', '')) + app_config[app_name] = { + 'region': {'value': region}, + 'zone': {'value': zone}} + + self.get_application_config.side_effect = \ + lambda x, model_name: app_config[x] + self.get_units.side_effect = lambda x, model_name: app_units[x] + self.assertEqual( + swift_utils.get_swift_storage_topology(), + expected_topology) + + def test_setup_test_container(self): + swift_client = mock.MagicMock() + self.patch_object(uuid, 'uuid1', return_value='auuid') + swift_client.get_account.return_value = ( + {'x-account-project-domain-id': 'domain-id'}, + 'bob-auuid-container') + self.assertEqual( + swift_utils.setup_test_container(swift_client, 'bob'), + ('bob-auuid-container', 'domain-id')) + swift_client.put_container.assert_called_once_with( + 'bob-auuid-container') + + def test_apply_proxy_config(self): + self.patch_object(zaza.model, 'block_until_all_units_idle') + self.patch_object( + zaza.model, + 'get_application_config', + return_value={ + 'go-faster': { + 'value': False}}) + self.patch_object(zaza.model, 'set_application_config') + swift_utils.apply_proxy_config( + 'proxy-app', + {'go-faster': True}) + self.set_application_config.assert_called_once_with( + 'proxy-app', {'go-faster': True}, model_name=None) + + def test_apply_proxy_config_noop(self): + self.patch_object(zaza.model, 'block_until_all_units_idle') + self.patch_object( + zaza.model, + 'get_application_config', + return_value={ + 'go-faster': { + 'value': True}}) + self.patch_object(zaza.model, 'set_application_config') + swift_utils.apply_proxy_config( + 'proxy-app', + {'go-faster': True}) + self.assertFalse(self.set_application_config.called) + + def test_create_object(self): + self.patch_object(swift_utils, 'setup_test_container') + self.setup_test_container.return_value = ('new-container', 'domain-id') + self.patch_object( + swift_utils, + 'ObjectReplicas', + return_value='obj_replicas') + swift_client = mock.MagicMock() + self.assertEqual( + swift_utils.create_object( + swift_client, + 'proxy-app', + swift_test_data.STORAGE_TOPOLOGY, + 'my-prefix'), + ('new-container', 'zaza_test_object.txt', 'obj_replicas')) + self.setup_test_container.assert_called_once_with( + swift_client, + 'my-prefix') + swift_client.put_object.assert_called_once_with( + 'new-container', + 'zaza_test_object.txt', + content_type='text/plain', + contents='File contents') + self.ObjectReplicas.assert_called_once_with( + 'proxy-app', + 'domain-id', + 'new-container', + 'zaza_test_object.txt', + swift_test_data.STORAGE_TOPOLOGY, + model_name=None) diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index ad2c0e5..a62d56f 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -17,11 +17,14 @@ """Encapsulate swift testing.""" import logging +import tenacity +import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.configure.guest import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.utilities.swift as swift_utils class SwiftImageCreateTest(test_utils.OpenStackBaseTest): @@ -110,3 +113,115 @@ class SwiftStorageTests(test_utils.OpenStackBaseTest): 'swift-container-sync'] with self.pause_resume(services): logging.info("Testing pause resume") + + +class SwiftGlobalReplicationTests(test_utils.OpenStackBaseTest): + """Test swift global replication.""" + + RESOURCE_PREFIX = 'zaza-swift-gr-tests' + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + cls.region1_model_alias = 'swift_gr_region1' + cls.region1_proxy_app = 'swift-proxy-region1' + cls.region2_model_alias = 'swift_gr_region2' + cls.region2_proxy_app = 'swift-proxy-region2' + super(SwiftGlobalReplicationTests, cls).setUpClass( + application_name=cls.region1_proxy_app, + model_alias=cls.region1_model_alias) + cls.region1_model_name = cls.model_aliases[cls.region1_model_alias] + cls.region2_model_name = cls.model_aliases[cls.region2_model_alias] + cls.storage_topology = swift_utils.get_swift_storage_topology( + model_name=cls.region1_model_name) + cls.storage_topology.update( + swift_utils.get_swift_storage_topology( + model_name=cls.region2_model_name)) + cls.swift_session = openstack_utils.get_keystone_session_from_relation( + cls.region1_proxy_app, + model_name=cls.region1_model_name) + cls.swift_region1 = openstack_utils.get_swift_session_client( + cls.swift_session, + region_name='RegionOne') + cls.swift_region2 = openstack_utils.get_swift_session_client( + cls.swift_session, + region_name='RegionTwo') + + @classmethod + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=16, max=600), + reraise=True, + stop=tenacity.stop_after_attempt(10)) + def tearDown(cls): + """Remove test resources. + + The retry decotator is needed as the deletes are async so objects + are sometime not fully deleted before their container. + """ + logging.info('Running teardown') + resp_headers, containers = cls.swift_region1.get_account() + logging.info('Found containers {}'.format(containers)) + for container in containers: + if not container['name'].startswith(cls.RESOURCE_PREFIX): + continue + for obj in cls.swift_region1.get_container(container['name'])[1]: + logging.info('Deleting object {} from {}'.format( + obj['name'], + container['name'])) + cls.swift_region1.delete_object( + container['name'], + obj['name']) + logging.info('Deleting container {}'.format(container['name'])) + cls.swift_region1.delete_container(container['name']) + + def test_two_regions_any_zones_two_replicas(self): + """Create an object with two replicas across two regions.""" + swift_utils.apply_proxy_config( + self.region1_proxy_app, + { + 'write-affinity': 'r1, r2', + 'write-affinity-node-count': '1', + 'replicas': '2'}, + self.region1_model_name) + container_name, obj_name, obj_replicas = swift_utils.create_object( + self.swift_region1, + self.region1_proxy_app, + self.storage_topology, + self.RESOURCE_PREFIX, + model_name=self.region1_model_name) + # Check object is accessible from other regions proxy. + self.swift_region2.head_object(container_name, obj_name) + # Check there is at least one replica in each region. + self.assertEqual( + sorted(obj_replicas.distinct_regions), + [1, 2]) + # Check there are two relicas + self.assertEqual( + len(obj_replicas.all_zones), + 2) + + def test_two_regions_any_zones_three_replicas(self): + """Create an object with three replicas across two regions.""" + swift_utils.apply_proxy_config( + self.region1_proxy_app, + { + 'write-affinity': 'r1, r2', + 'write-affinity-node-count': '1', + 'replicas': '3'}, + self.region1_model_name) + container_name, obj_name, obj_replicas = swift_utils.create_object( + self.swift_region1, + self.region1_proxy_app, + self.storage_topology, + self.RESOURCE_PREFIX, + model_name=self.region1_model_name) + # Check object is accessible from other regions proxy. + self.swift_region2.head_object(container_name, obj_name) + # Check there is at least one replica in each region. + self.assertEqual( + sorted(obj_replicas.distinct_regions), + [1, 2]) + # Check there are three relicas + self.assertEqual( + len(obj_replicas.all_zones), + 3) diff --git a/zaza/openstack/utilities/swift.py b/zaza/openstack/utilities/swift.py new file mode 100644 index 0000000..43a977d --- /dev/null +++ b/zaza/openstack/utilities/swift.py @@ -0,0 +1,295 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Swift utilities.""" + +import logging +import uuid +import zaza.model +import zaza.openstack.utilities.juju as juju_utils + + +class ObjectReplica: + """A replica of an object. + + The replica attributes show the location of an object replica. + + server: IP address or hostname of machine hosting replica + port: Port of swift object server running on machine hosting replica + device: Path to device hosting replica + handoff_device: Whether this is a handoff devices. Handoff devices pass + the replica on to a remote storage node. + """ + + def __init__(self, raw_line): + """Extract storage info from text.""" + rl = raw_line.split() + self.server, self.port = rl[2].split(':') + self.device = rl[3] + self.handoff_device = rl[-1] == '[Handoff]' + + +class ObjectReplicas: + """Replicas of an object.""" + + def __init__(self, proxy_app, account, container_name, object_name, + storage_topology, model_name=None): + """Find all replicas of given object. + + :param proxy_app: Name of proxy application + :type proxy_app: str + :param account: Account that owns the container. + :type account: str + :param container_name: Name of container that contains the object. + :type container_name: str + :param object_name: Name of object. + :type object_name: str + :param storage_topology: Dictionary keyed on IP of storage node info. + :type storage_topology: {} + :param model_name: Model to point environment at + :type model_name: str + """ + self.replicas = [] + self.replica_placements = {} + self.storage_topology = storage_topology + raw_output = self.run_get_nodes( + proxy_app, + account, + container_name, + object_name, + model_name=model_name) + for line in self.extract_storage_lines(raw_output): + self.add_replica(line) + + def add_replica(self, storage_line): + """Add a replica to the replica set.""" + self.replicas.append(ObjectReplica(storage_line)) + + def extract_storage_lines(self, raw_output): + """Extract replica list from output of swift-get-nodes. + + :param storage_line: Output of swift-get-nodes + :type storage_line: str + :returns: List of lines relating to replicas. + :rtype: [str, ...] + """ + storage_lines = [] + for line in raw_output.split('\n'): + if line.startswith('Server:Port '): + storage_lines.append(line) + return storage_lines + + def run_get_nodes(self, proxy_app, account, container_name, object_name, + model_name=None): + """Run swift-get-nodes for an object on a proxy unit. + + :param proxy_app: Name of proxy application + :type proxy_app: str + :param account: Account that owns the container. + :type account: str + :param container_name: Name of container that contains the object. + :type container_name: str + :param object_name: Name of object. + :type object_name: str + :param model_name: Model to point environment at + :type model_name: str + """ + ring_file = '/etc/swift/object.ring.gz' + obj_cmd = "swift-get-nodes -a {} {} {} {}".format( + ring_file, + account, + container_name, + object_name) + cmd_result = zaza.model.run_on_leader( + proxy_app, + obj_cmd, + model_name=model_name) + return cmd_result['Stdout'] + + @property + def hand_off_ips(self): + """Replicas which are marked as handoff devices. + + These are not real replicas. They hand off the replica to other node. + + :returns: List of IPS of handoff nodes for object. + :rtype: [str, ...] + """ + return [r.server for r in self.replicas if r.handoff_device] + + @property + def storage_ips(self): + """Ip addresses of nodes that are housing a replica. + + :returns: List of IPS of storage nodes holding a replica of the object. + :rtype: [str, ...] + """ + return [r.server for r in self.replicas if not r.handoff_device] + + @property + def placements(self): + """Region an zone information for each replica. + + :returns: List of dicts with region and zone information. + :rtype: [{ + 'app_name': str, + 'unit': juju.Unit, + 'region': int, + 'zone': int}, ...] + """ + return [self.storage_topology[ip] for ip in self.storage_ips] + + @property + def distinct_regions(self): + """List of distinct regions that have a replica. + + :returns: List of regions that have a replica + :rtype: [int, ...] + """ + return list(set([p['region'] for p in self.placements])) + + @property + def all_zones(self): + """List of all zones that have a replica. + + :returns: List of tuples (region, zone) that have a replica. + :rtype: [(r1, z1), ...] + """ + return [(p['region'], p['zone']) for p in self.placements] + + @property + def distinct_zones(self): + """List of distinct region + zones that have a replica. + + :returns: List of tuples (region, zone) that have a replica. + :rtype: [(r1, z1), ...] + """ + return list(set(self.all_zones)) + + +def get_swift_storage_topology(model_name=None): + """Get details of storage nodes and which region and zones they belong in. + + :param model_name: Model to point environment at + :type model_name: str + :returns: Dictionary of storage nodes and their region/zone information. + :rtype: { + 'ip (str)': { + 'app_name': str, + 'unit': juju.Unit + 'region': int, + 'zone': int}, + ...} + """ + topology = {} + status = juju_utils.get_full_juju_status(model_name=model_name) + for app_name, app_dep_config in status.applications.items(): + if 'swift-storage' in app_dep_config['charm']: + app_config = zaza.model.get_application_config( + app_name, + model_name=model_name) + region = app_config['region']['value'] + zone = app_config['zone']['value'] + for unit in zaza.model.get_units(app_name, model_name=model_name): + topology[unit.public_address] = { + 'app_name': app_name, + 'unit': unit, + 'region': region, + 'zone': zone} + return topology + + +def setup_test_container(swift_client, resource_prefix): + """Create a swift container for use be tests. + + :param swift_client: Swift client to use for object creation + :type swift_client: swiftclient.Client + :returns: (container_name, account_name) Container name and account + name for new container + :rtype: (str, str) + """ + run_id = str(uuid.uuid1()).split('-')[0] + container_name = '{}-{}-container'.format(resource_prefix, run_id) + swift_client.put_container(container_name) + resp_headers, containers = swift_client.get_account() + account = resp_headers['x-account-project-domain-id'] + return container_name, account + + +def apply_proxy_config(proxy_app, config, model_name=None): + """Update the give proxy_app with new charm config. + + :param proxy_app: Name of proxy application + :type proxy_app: str + :param config: Dictionary of configuration setting(s) to apply + :type config: dict + :param model_name: Name of model to query. + :type model_name: str + """ + current_config = zaza.model.get_application_config( + proxy_app, + model_name=model_name) + # Although there is no harm in applying config that is a noop it + # does affect the expected behaviour afterwards. So, only apply + # genuine changes so we can safely expect the charm to fire a hook. + for key, value in config.items(): + if str(config[key]) != str(current_config[key]['value']): + break + else: + logging.info( + 'Config update for {} not required.'.format(proxy_app)) + return + logging.info('Updating {} charm settings'.format(proxy_app)) + zaza.model.set_application_config( + proxy_app, + config, + model_name=model_name) + zaza.model.block_until_all_units_idle() + + +def create_object(swift_client, proxy_app, storage_topology, resource_prefix, + model_name=None): + """Create a test object in a new container. + + :param swift_client: Swift client to use for object creation + :type swift_client: swiftclient.Client + :param proxy_app: Name of proxy application + :type proxy_app: str + :param storage_topology: Dictionary keyed on IP of storage node info. + :type storage_topology: {} + :param resource_prefix: Prefix to use when naming new resources + :type resource_prefix: str + :param model_name: Model to point environment at + :type model_name: str + :returns: (container_name, object_name, object replicas) + :rtype: (str, str, ObjectReplicas) + """ + container_name, account = setup_test_container( + swift_client, + resource_prefix) + object_name = 'zaza_test_object.txt' + swift_client.put_object( + container_name, + object_name, + contents='File contents', + content_type='text/plain' + ) + obj_replicas = ObjectReplicas( + proxy_app, + account, + container_name, + object_name, + storage_topology, + model_name=model_name) + return container_name, object_name, obj_replicas From 296119633308f22b4e06718e8293a8c2f2f5b19b Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 29 Nov 2019 16:32:07 +0100 Subject: [PATCH 212/898] neutron: Retry operation after change of config There is a race between `neutron-api` charm readiness and actual API readiness. Fixes #138 --- zaza/openstack/charm_tests/policyd/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 0c7ad3e..532495f 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -522,6 +522,12 @@ class NeutronApiTests(BasePolicydSpecialization): super(NeutronApiTests, cls).setUpClass(application_name="neutron-api") cls.application_name = "neutron-api" + # NOTE(fnordahl): There is a race between `neutron-api` signalling unit is + # ready and the service actually being ready to serve requests. The test + # will fail intermittently unless we gracefully accept this. + # Issue: openstack-charmers/zaza-openstack-tests#138 + @tenacity.retry(wait=tenacity.wait_fixed(1), + reraise=True, stop=tenacity.stop_after_delay(8)) def get_client_and_attempt_operation(self, ip): """Attempt to list the networks as a policyd override. From 8b40d28ad1050e7e2d3485bb19aaebb3b1c664ea Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 3 Dec 2019 17:15:44 +0100 Subject: [PATCH 213/898] address tinwood's comments --- zaza/openstack/charm_tests/manila/tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index e2ae24b..4caaffd 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -44,5 +44,4 @@ packages: def test_manila_api(self): """Test that the Manila API is working.""" # now just try a list the shares - # NOTE(AJK) the 'search_opts={}' is to work around Bug#1707303 - self.manila.shares.list(search_opts={}) + self.manila.shares.list() From a6bde7168310f8d80eb9a34ba1b23efa568f2059 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 4 Dec 2019 10:19:53 +0100 Subject: [PATCH 214/898] remove unnecessary class variables --- zaza/openstack/charm_tests/manila/tests.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index 4caaffd..3ce7589 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -26,12 +26,6 @@ import zaza.openstack.utilities.openstack as openstack_utils class ManilaTests(test_utils.OpenStackBaseTest): """Encapsulate Manila tests.""" - RESOURCE_PREFIX = 'zaza-manilatests' - INSTANCE_USERDATA = """#cloud-config -packages: -- nfs-common -""" - @classmethod def setUpClass(cls): """Run class setup for running tests.""" From f1092b08111161adc3e9576fa775a8cac445f30b Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 4 Dec 2019 10:24:04 +0000 Subject: [PATCH 215/898] Doc string fixes --- zaza/openstack/charm_tests/swift/tests.py | 5 +++-- zaza/openstack/utilities/swift.py | 21 +++++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index a62d56f..f5e2370 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -155,8 +155,9 @@ class SwiftGlobalReplicationTests(test_utils.OpenStackBaseTest): def tearDown(cls): """Remove test resources. - The retry decotator is needed as the deletes are async so objects - are sometime not fully deleted before their container. + The retry decorator is needed as it is luck of the draw as to whether + a delete of a newly created container will result in a 404. Retrying + will eventually result in the delete being accepted. """ logging.info('Running teardown') resp_headers, containers = cls.swift_region1.get_account() diff --git a/zaza/openstack/utilities/swift.py b/zaza/openstack/utilities/swift.py index 43a977d..67daca0 100644 --- a/zaza/openstack/utilities/swift.py +++ b/zaza/openstack/utilities/swift.py @@ -104,6 +104,8 @@ class ObjectReplicas: :type object_name: str :param model_name: Model to point environment at :type model_name: str + :returns: Stdout of command + :rtype: str """ ring_file = '/etc/swift/object.ring.gz' obj_cmd = "swift-get-nodes -a {} {} {} {}".format( @@ -124,7 +126,7 @@ class ObjectReplicas: These are not real replicas. They hand off the replica to other node. :returns: List of IPS of handoff nodes for object. - :rtype: [str, ...] + :rtype: List[str] """ return [r.server for r in self.replicas if r.handoff_device] @@ -141,12 +143,15 @@ class ObjectReplicas: def placements(self): """Region an zone information for each replica. + Zone info is in the form: + [{ + 'app_name': str, + 'unit': juju.Unit, + 'region': int, + 'zone': int}, ...] + :returns: List of dicts with region and zone information. - :rtype: [{ - 'app_name': str, - 'unit': juju.Unit, - 'region': int, - 'zone': int}, ...] + :rtype: List[Dict[str, Union[str,int]]] """ return [self.storage_topology[ip] for ip in self.storage_ips] @@ -164,7 +169,7 @@ class ObjectReplicas: """List of all zones that have a replica. :returns: List of tuples (region, zone) that have a replica. - :rtype: [(r1, z1), ...] + :rtype: List[Tuple[str, str]] """ return [(p['region'], p['zone']) for p in self.placements] @@ -217,7 +222,7 @@ def setup_test_container(swift_client, resource_prefix): :type swift_client: swiftclient.Client :returns: (container_name, account_name) Container name and account name for new container - :rtype: (str, str) + :rtype: Tuple[str, str] """ run_id = str(uuid.uuid1()).split('-')[0] container_name = '{}-{}-container'.format(resource_prefix, run_id) From bb675982b66a0bcde7e8da45e3d1adc7c479952a Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 4 Dec 2019 11:32:17 +0100 Subject: [PATCH 216/898] Fix mis-named variable --- zaza/openstack/charm_tests/manila/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index 3ce7589..54f7ade 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -38,4 +38,4 @@ class ManilaTests(test_utils.OpenStackBaseTest): def test_manila_api(self): """Test that the Manila API is working.""" # now just try a list the shares - self.manila.shares.list() + self.manila_client.shares.list() From 7d23285ce00d9491e5cc3846b893cb65c4cb1291 Mon Sep 17 00:00:00 2001 From: Jose Guedez Date: Thu, 5 Dec 2019 16:22:29 +1100 Subject: [PATCH 217/898] Ported neutron-gateway tests to zaza (900/910) --- zaza/openstack/charm_tests/neutron/tests.py | 91 +++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index e7d609d..e3d5cd7 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -29,6 +29,97 @@ import zaza.openstack.configure.guest as guest import zaza.openstack.utilities.openstack as openstack_utils +class NeutronGatewayTest(test_utils.OpenStackBaseTest): + """Test basic Neutron Gateway Charm functionality.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Neutron Gateway tests.""" + super(NeutronGatewayTest, cls).setUpClass() + cls.current_os_release = openstack_utils.get_os_release() + cls.services = cls._get_services() + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change. + + Change disk format and assert then change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + current_value = zaza.model.get_application_config( + 'neutron-gateway')['debug']['value'] + new_value = str(not bool(current_value)).title() + current_value = str(current_value).title() + + set_default = {'debug': current_value} + set_alternate = {'debug': new_value} + default_entry = {'DEFAULT': {'debug': [current_value]}} + alternate_entry = {'DEFAULT': {'debug': [new_value]}} + + # Config file affected by juju set config change + conf_file = '/etc/neutron/neutron.conf' + + bionic_stein = openstack_utils.get_os_release('bionic_stein') + + # Make config change, check for service restarts + logging.info( + 'Setting verbose on neutron-api {}'.format(set_alternate)) + if self.current_os_release >= bionic_stein: + pgrep_full = True + else: + pgrep_full = False + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + self.services, + pgrep_full=pgrep_full) + + def test_910_pause_and_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + bionic_stein = openstack_utils.get_os_release('bionic_stein') + if self.current_os_release >= bionic_stein: + pgrep_full = True + else: + pgrep_full = False + with self.pause_resume( + self.services, + pgrep_full=pgrep_full): + logging.info("Testing pause resume") + + @classmethod + def _get_services(cls): + """ + Return the services expected in Neutron Gateway. + + :returns: A list of services + :rtype: list[str] + """ + services = ['neutron-dhcp-agent', + 'neutron-metadata-agent', + 'neutron-metering-agent', + 'neutron-openvswitch-agent'] + + trusty_icehouse = openstack_utils.get_os_release('trusty_icehouse') + xenial_newton = openstack_utils.get_os_release('xenial_newton') + bionic_train = openstack_utils.get_os_release('bionic_train') + + if cls.current_os_release <= trusty_icehouse: + services.append('neutron-vpn-agent') + if cls.current_os_release < xenial_newton: + services.append('neutron-lbaas-agent') + if xenial_newton <= cls.current_os_release < bionic_train: + services.append('neutron-lbaasv2-agent') + + return services + + class NeutronApiTest(test_utils.OpenStackBaseTest): """Test basic Neutron API Charm functionality.""" From 4fd543a0fe3f37f234845b40182355509980b536 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 6 Dec 2019 12:01:38 -0800 Subject: [PATCH 218/898] Hanlde new StopIteration for iterators --- zaza/openstack/utilities/juju.py | 2 +- zaza/openstack/utilities/openstack.py | 52 ++++++++++++++------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index fd7abc1..9eb38d5 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -92,7 +92,7 @@ def get_machines_for_application(application, model_name=None): """ status = get_application_status(application, model_name=model_name) if not status: - raise StopIteration + return # libjuju juju status no longer has units for subordinate charms # Use the application it is subordinate-to to find machines diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 1776296..81c61f7 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -674,31 +674,35 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, if not net_id: net_id = get_admin_net(neutronclient)['id'] - for uuid in uuids: - server = novaclient.servers.get(uuid) - ext_port_name = "{}_ext-port".format(server.name) - for port in neutronclient.list_ports(device_id=server.id)['ports']: - if port['name'] == ext_port_name: - logging.warning('Neutron Gateway already has additional port') - break - else: - logging.info('Attaching additional port to instance, ' - 'connected to net id: {}'.format(net_id)) - body_value = { - "port": { - "admin_state_up": True, - "name": ext_port_name, - "network_id": net_id, - "port_security_enabled": False, + try: + for uuid in uuids: + server = novaclient.servers.get(uuid) + ext_port_name = "{}_ext-port".format(server.name) + for port in neutronclient.list_ports(device_id=server.id)['ports']: + if port['name'] == ext_port_name: + logging.warning( + 'Neutron Gateway already has additional port') + break + else: + logging.info('Attaching additional port to instance, ' + 'connected to net id: {}'.format(net_id)) + body_value = { + "port": { + "admin_state_up": True, + "name": ext_port_name, + "network_id": net_id, + "port_security_enabled": False, + } } - } - port = neutronclient.create_port(body=body_value) - server.interface_attach(port_id=port['port']['id'], - net_id=None, fixed_ip=None) - if add_dataport_to_netplan: - mac_address = get_mac_from_port(port, neutronclient) - add_interface_to_netplan(server.name, - mac_address=mac_address) + port = neutronclient.create_port(body=body_value) + server.interface_attach(port_id=port['port']['id'], + net_id=None, fixed_ip=None) + if add_dataport_to_netplan: + mac_address = get_mac_from_port(port, neutronclient) + add_interface_to_netplan(server.name, + mac_address=mac_address) + except StopIteration: + pass ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: if 'ext-port' in port['name']: From 1314c6af2f9f1cb91846643aa45ce590986223fc Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 6 Dec 2019 13:42:09 -0800 Subject: [PATCH 219/898] Don't use StopIteration --- zaza/openstack/utilities/openstack.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 81c61f7..3291e58 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -649,9 +649,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, application_names = ['neutron-openvswitch'] try: ngw = 'neutron-gateway' - next(juju_utils.get_machine_uuids_for_application(ngw)) + model.get_application(ngw) application_names.append(ngw) - except StopIteration: + except KeyError: # neutron-gateway not in deployment pass elif ovn_present(): @@ -659,9 +659,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, application_names = ['ovn-chassis'] try: ovn_dc_name = 'ovn-dedicated-chassis' - next(juju_utils.get_machine_uuids_for_application(ovn_dc_name)) + model.get_application(ngw) application_names.append(ovn_dc_name) - except StopIteration: + except KeyError: # ovn-dedicated-chassis not in deployment pass port_config_key = 'interface-bridge-mappings' @@ -1450,8 +1450,8 @@ def get_current_os_release_pair(application='keystone'): :raises: exceptions.OSVersionNotFound """ try: - machine = next(juju_utils.get_machines_for_application(application)) - except StopIteration: + machine = juju_utils.get_machines_for_application(application) + except KeyError: raise exceptions.ApplicationNotFound(application) series = juju_utils.get_machine_series(machine) From ad9eecc49e1bbf26dcab24664b3be68fee49acbf Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 6 Dec 2019 14:35:37 -0800 Subject: [PATCH 220/898] Pick just one machine --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 3291e58..5c8719b 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1450,7 +1450,7 @@ def get_current_os_release_pair(application='keystone'): :raises: exceptions.OSVersionNotFound """ try: - machine = juju_utils.get_machines_for_application(application) + machine = list(juju_utils.get_machines_for_application(application))[0] except KeyError: raise exceptions.ApplicationNotFound(application) From 1a4e02e98c4f6506fd66852775a13847f1cba52d Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 6 Dec 2019 15:02:57 -0800 Subject: [PATCH 221/898] Fix bug --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 5c8719b..ffcfc23 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -659,7 +659,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, application_names = ['ovn-chassis'] try: ovn_dc_name = 'ovn-dedicated-chassis' - model.get_application(ngw) + model.get_application(ovn_dc_name) application_names.append(ovn_dc_name) except KeyError: # ovn-dedicated-chassis not in deployment From 2704fe654cb895c8c95a1166dbd33eb7a3b086c8 Mon Sep 17 00:00:00 2001 From: Jose Guedez Date: Mon, 9 Dec 2019 17:33:25 +1100 Subject: [PATCH 222/898] Ported neutron-gateway tests to zaza (500) --- zaza/openstack/charm_tests/neutron/tests.py | 26 +++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index e3d5cd7..6821f30 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -120,6 +120,32 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): return services +class NeutronGatewaySecurityTest(test_utils.OpenStackBaseTest): + """Neutron Gateway Security Tests.""" + + def test_security_checklist(self): + """Verify expected state with security-checklist.""" + expected_failures = [] + expected_passes = [ + 'validate-file-ownership', + 'validate-file-permissions', + ] + + for unit in zaza.model.get_units('neutron-gateway', + model_name=self.model_name): + logging.info('Running `security-checklist` action' + ' on unit {}'.format(unit.entity_id)) + test_utils.audit_assertions( + zaza.model.run_action( + unit.entity_id, + 'security-checklist', + model_name=self.model_name, + action_params={}), + expected_passes, + expected_failures, + expected_to_pass=True) + + class NeutronApiTest(test_utils.OpenStackBaseTest): """Test basic Neutron API Charm functionality.""" From c109abbe259578073c8f970155ed24004ccf7c7f Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 9 Dec 2019 09:22:01 -0800 Subject: [PATCH 223/898] Fix unit test --- unit_tests/utilities/test_zaza_utilities_openstack.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 7e9a57d..8519318 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -15,7 +15,6 @@ import copy import datetime import io -import itertools import mock import tenacity @@ -814,14 +813,16 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): name='_get_machine_series' ) + _machine = mock.MagicMock() + # No machine returned - self._get_machines.side_effect = StopIteration + self._get_machines.side_effect = KeyError with self.assertRaises(exceptions.ApplicationNotFound): openstack_utils.get_current_os_release_pair() self._get_machines.side_effect = None # No series returned - self._get_machines.return_value = itertools.repeat('6') + self._get_machines.return_value = [_machine] self._get_machine_series.return_value = None with self.assertRaises(exceptions.SeriesNotFound): openstack_utils.get_current_os_release_pair() From a3cf1bf7e70fe4a736ccbf687d83f3b09a087733 Mon Sep 17 00:00:00 2001 From: Jose Guedez Date: Mon, 9 Dec 2019 18:36:33 +1100 Subject: [PATCH 224/898] Ported neutron-gateway tests to zaza (400/401) --- zaza/openstack/charm_tests/neutron/tests.py | 82 +++++++++++++++++---- 1 file changed, 69 insertions(+), 13 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 6821f30..5d75099 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -17,6 +17,7 @@ """Encapsulating `neutron-openvswitch` testing.""" +import time import logging import tenacity import unittest @@ -39,6 +40,72 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): cls.current_os_release = openstack_utils.get_os_release() cls.services = cls._get_services() + # set up clients + cls.neutron_client = ( + openstack_utils.get_neutron_session_client(cls.keystone_session)) + + bionic_stein = openstack_utils.get_os_release('bionic_stein') + + cls.pgrep_full = (True if cls.current_os_release >= bionic_stein + else False) + + def test_400_create_network(self): + """Create a network, verify that it exists, and then delete it.""" + logging.debug('Creating neutron network...') + self.neutron_client.format = 'json' + net_name = 'test_net' + + # Verify that the network doesn't exist + networks = self.neutron_client.list_networks(name=net_name) + net_count = len(networks['networks']) + assert net_count == 0, ( + "Expected zero networks, found {}".format(net_count)) + + # Create a network and verify that it exists + network = {'name': net_name} + self.neutron_client.create_network({'network': network}) + + networks = self.neutron_client.list_networks(name=net_name) + logging.debug('Networks: {}'.format(networks)) + net_len = len(networks['networks']) + assert net_len == 1, ( + "Expected 1 network, found {}".format(net_len)) + + logging.debug('Confirming new neutron network...') + network = networks['networks'][0] + assert network['name'] == net_name, "network ext_net not found" + + # Cleanup + logging.debug('Deleting neutron network...') + self.neutron_client.delete_network(network['id']) + + def test_401_enable_qos(self): + """Check qos settings set via neutron-api charm.""" + if (self.current_os_release >= + openstack_utils.get_os_release('trusty_mitaka')): + logging.info('running qos check') + + with self.config_change( + {'enable-qos': 'False'}, + {'enable-qos': 'True'}, + application_name="neutron-api"): + + # wait for neutron-gateway to complete as well + time.sleep(60) + + # obtain dhcp agent to identify the neutron-gateway host + dhcp_agent = self.neutron_client.list_agents( + binary='neutron-dhcp-agent')['agents'][0] + neutron_gw_host = dhcp_agent['host'] + logging.debug('neutron gw host: {}'.format(neutron_gw_host)) + + # check extensions on the ovs agent to validate qos + ovs_agent = self.neutron_client.list_agents( + binary='neutron-openvswitch-agent', + host=neutron_gw_host)['agents'][0] + + self.assertIn('qos', ovs_agent['configurations']['extensions']) + def test_900_restart_on_config_change(self): """Checking restart happens on config change. @@ -59,15 +126,9 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): # Config file affected by juju set config change conf_file = '/etc/neutron/neutron.conf' - bionic_stein = openstack_utils.get_os_release('bionic_stein') - # Make config change, check for service restarts logging.info( 'Setting verbose on neutron-api {}'.format(set_alternate)) - if self.current_os_release >= bionic_stein: - pgrep_full = True - else: - pgrep_full = False self.restart_on_changed( conf_file, set_default, @@ -75,7 +136,7 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): default_entry, alternate_entry, self.services, - pgrep_full=pgrep_full) + pgrep_full=self.pgrep_full) def test_910_pause_and_resume(self): """Run pause and resume tests. @@ -83,14 +144,9 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): Pause service and check services are stopped then resume and check they are started """ - bionic_stein = openstack_utils.get_os_release('bionic_stein') - if self.current_os_release >= bionic_stein: - pgrep_full = True - else: - pgrep_full = False with self.pause_resume( self.services, - pgrep_full=pgrep_full): + pgrep_full=self.pgrep_full): logging.info("Testing pause resume") @classmethod From 99eed41697a3526c8390b291971fbc43e3033ee7 Mon Sep 17 00:00:00 2001 From: Jose Guedez Date: Mon, 9 Dec 2019 18:52:23 +1100 Subject: [PATCH 225/898] Ported neutron-gateway tests to zaza (920) --- zaza/openstack/charm_tests/neutron/tests.py | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 5d75099..e39043e 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -149,6 +149,34 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): pgrep_full=self.pgrep_full): logging.info("Testing pause resume") + def test_920_change_aa_profile(self): + """Test changing the Apparmor profile mode.""" + services = ['neutron-openvswitch-agent', + 'neutron-dhcp-agent', + 'neutron-l3-agent', + 'neutron-metadata-agent', + 'neutron-metering-agent'] + + set_default = {'aa-profile-mode': 'disable'} + set_alternate = {'aa-profile-mode': 'complain'} + + mtime = zaza.model.get_unit_time( + self.lead_unit, + model_name=self.model_name) + logging.debug('Remote unit timestamp {}'.format(mtime)) + + with self.config_change(set_default, set_alternate): + for unit in zaza.model.get_units('neutron-gateway', + model_name=self.model_name): + logging.info('Checking number of profiles in complain ' + 'mode in {}'.format(unit.entity_id)) + run = zaza.model.run_on_unit( + unit.entity_id, + 'aa-status --complaining', + model_name=self.model_name) + output = run['Stdout'] + self.assertTrue(int(output) >= len(services)) + @classmethod def _get_services(cls): """ From 40e9c86df4209082eba3ab82a4f24fdfd5e4cbab Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 10 Dec 2019 08:36:22 -0800 Subject: [PATCH 226/898] Test the condition not the implementation --- unit_tests/utilities/test_zaza_utilities_openstack.py | 2 +- zaza/openstack/utilities/openstack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 8519318..915d661 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -816,7 +816,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): _machine = mock.MagicMock() # No machine returned - self._get_machines.side_effect = KeyError + self._get_machines.return_value = [] with self.assertRaises(exceptions.ApplicationNotFound): openstack_utils.get_current_os_release_pair() self._get_machines.side_effect = None diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index ffcfc23..efcdd26 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1451,7 +1451,7 @@ def get_current_os_release_pair(application='keystone'): """ try: machine = list(juju_utils.get_machines_for_application(application))[0] - except KeyError: + except IndexError: raise exceptions.ApplicationNotFound(application) series = juju_utils.get_machine_series(machine) From 23e0dd5325a9ed23f905358198e1ce69dfcf7fcf Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 10 Dec 2019 10:29:57 -0800 Subject: [PATCH 227/898] Catching StopIteration is not necessary --- zaza/openstack/utilities/openstack.py | 53 +++++++++++++-------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index efcdd26..5582a89 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -674,35 +674,32 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, if not net_id: net_id = get_admin_net(neutronclient)['id'] - try: - for uuid in uuids: - server = novaclient.servers.get(uuid) - ext_port_name = "{}_ext-port".format(server.name) - for port in neutronclient.list_ports(device_id=server.id)['ports']: - if port['name'] == ext_port_name: - logging.warning( - 'Neutron Gateway already has additional port') - break - else: - logging.info('Attaching additional port to instance, ' - 'connected to net id: {}'.format(net_id)) - body_value = { - "port": { - "admin_state_up": True, - "name": ext_port_name, - "network_id": net_id, - "port_security_enabled": False, - } + for uuid in uuids: + server = novaclient.servers.get(uuid) + ext_port_name = "{}_ext-port".format(server.name) + for port in neutronclient.list_ports(device_id=server.id)['ports']: + if port['name'] == ext_port_name: + logging.warning( + 'Neutron Gateway already has additional port') + break + else: + logging.info('Attaching additional port to instance, ' + 'connected to net id: {}'.format(net_id)) + body_value = { + "port": { + "admin_state_up": True, + "name": ext_port_name, + "network_id": net_id, + "port_security_enabled": False, } - port = neutronclient.create_port(body=body_value) - server.interface_attach(port_id=port['port']['id'], - net_id=None, fixed_ip=None) - if add_dataport_to_netplan: - mac_address = get_mac_from_port(port, neutronclient) - add_interface_to_netplan(server.name, - mac_address=mac_address) - except StopIteration: - pass + } + port = neutronclient.create_port(body=body_value) + server.interface_attach(port_id=port['port']['id'], + net_id=None, fixed_ip=None) + if add_dataport_to_netplan: + mac_address = get_mac_from_port(port, neutronclient) + add_interface_to_netplan(server.name, + mac_address=mac_address) ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: if 'ext-port' in port['name']: From 407487ca66455d7a4471583585842e5344d04491 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 10 Dec 2019 16:47:46 -0800 Subject: [PATCH 228/898] Use Netplan for OVN --- zaza/openstack/utilities/openstack.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 1776296..5abf185 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -573,6 +573,9 @@ def add_interface_to_netplan(server_name, mac_address): """ if dvr_enabled(): application_name = 'neutron-openvswitch' + elif ovn_present(): + # OVN chassis is a subordinate to nova-compute + application_name = 'nova-compute' else: application_name = 'neutron-gateway' @@ -666,7 +669,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, pass port_config_key = 'interface-bridge-mappings' config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) - add_dataport_to_netplan = False + add_dataport_to_netplan = True else: uuids = itertools.islice(get_gateway_uuids(), limit_gws) application_names = ['neutron-gateway'] From 79bb61243f25d3264ab81edfd9279036d930dd5d Mon Sep 17 00:00:00 2001 From: Jose Guedez Date: Wed, 11 Dec 2019 12:51:43 +1100 Subject: [PATCH 229/898] Use a single security test class for neutron-api and neutron-gateway --- zaza/openstack/charm_tests/neutron/tests.py | 69 +++++++++------------ 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index e39043e..3d4588b 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -204,32 +204,6 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): return services -class NeutronGatewaySecurityTest(test_utils.OpenStackBaseTest): - """Neutron Gateway Security Tests.""" - - def test_security_checklist(self): - """Verify expected state with security-checklist.""" - expected_failures = [] - expected_passes = [ - 'validate-file-ownership', - 'validate-file-permissions', - ] - - for unit in zaza.model.get_units('neutron-gateway', - model_name=self.model_name): - logging.info('Running `security-checklist` action' - ' on unit {}'.format(unit.entity_id)) - test_utils.audit_assertions( - zaza.model.run_action( - unit.entity_id, - 'security-checklist', - model_name=self.model_name, - action_params={}), - expected_passes, - expected_failures, - expected_to_pass=True) - - class NeutronApiTest(test_utils.OpenStackBaseTest): """Test basic Neutron API Charm functionality.""" @@ -288,28 +262,41 @@ class NeutronApiTest(test_utils.OpenStackBaseTest): class SecurityTest(test_utils.OpenStackBaseTest): - """Neutron APIsecurity tests tests.""" + """Neutron Security Tests.""" def test_security_checklist(self): """Verify expected state with security-checklist.""" - tls_checks = [ - 'validate-uses-tls-for-keystone', - ] - expected_failures = [ - 'validate-enables-tls', # LP: #1851610 - ] + expected_failures = [] expected_passes = [ 'validate-file-ownership', 'validate-file-permissions', - 'validate-uses-keystone', ] - if zaza.model.get_relation_id( - 'neutron-api', 'vault', remote_interface_name='certificates'): - expected_passes.extend(tls_checks) - else: - expected_failures.extend(tls_checks) + expected_to_pass = True - for unit in zaza.model.get_units('neutron-api', + # override settings depending on application name so we can reuse + # the class for multiple charms + if self.application_name == 'neutron-api': + tls_checks = [ + 'validate-uses-tls-for-keystone', + ] + + expected_failures = [ + 'validate-enables-tls', # LP: #1851610 + ] + + expected_passes.append('validate-uses-keystone') + + if zaza.model.get_relation_id( + 'neutron-api', + 'vault', + remote_interface_name='certificates'): + expected_passes.extend(tls_checks) + else: + expected_failures.extend(tls_checks) + + expected_to_pass = False + + for unit in zaza.model.get_units(self.application_name, model_name=self.model_name): logging.info('Running `security-checklist` action' ' on unit {}'.format(unit.entity_id)) @@ -321,7 +308,7 @@ class SecurityTest(test_utils.OpenStackBaseTest): action_params={}), expected_passes, expected_failures, - expected_to_pass=False) + expected_to_pass=expected_to_pass) class NeutronNetworkingTest(unittest.TestCase): From f79f2fb54c062651324f7140e9f90941c565c302 Mon Sep 17 00:00:00 2001 From: Jose Guedez Date: Wed, 11 Dec 2019 14:05:19 +1100 Subject: [PATCH 230/898] NeutronGatewayTest: refactor test 401 to use tenacity --- zaza/openstack/charm_tests/neutron/tests.py | 33 +++++++++++---------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 3d4588b..f6cde67 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -17,7 +17,6 @@ """Encapsulating `neutron-openvswitch` testing.""" -import time import logging import tenacity import unittest @@ -90,21 +89,7 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): {'enable-qos': 'True'}, application_name="neutron-api"): - # wait for neutron-gateway to complete as well - time.sleep(60) - - # obtain dhcp agent to identify the neutron-gateway host - dhcp_agent = self.neutron_client.list_agents( - binary='neutron-dhcp-agent')['agents'][0] - neutron_gw_host = dhcp_agent['host'] - logging.debug('neutron gw host: {}'.format(neutron_gw_host)) - - # check extensions on the ovs agent to validate qos - ovs_agent = self.neutron_client.list_agents( - binary='neutron-openvswitch-agent', - host=neutron_gw_host)['agents'][0] - - self.assertIn('qos', ovs_agent['configurations']['extensions']) + self._validate_openvswitch_agent_qos() def test_900_restart_on_config_change(self): """Checking restart happens on config change. @@ -177,6 +162,22 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): output = run['Stdout'] self.assertTrue(int(output) >= len(services)) + @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60)) + def _validate_openvswitch_agent_qos(self): + """Validate that the qos extension is enabled in the ovs agent.""" + # obtain the dhcp agent to identify the neutron-gateway host + dhcp_agent = self.neutron_client.list_agents( + binary='neutron-dhcp-agent')['agents'][0] + neutron_gw_host = dhcp_agent['host'] + logging.debug('neutron gw host: {}'.format(neutron_gw_host)) + + # check extensions on the ovs agent to validate qos + ovs_agent = self.neutron_client.list_agents( + binary='neutron-openvswitch-agent', + host=neutron_gw_host)['agents'][0] + + self.assertIn('qos', ovs_agent['configurations']['extensions']) + @classmethod def _get_services(cls): """ From 55fc718f981f22cb7a555de993960841d8d5196a Mon Sep 17 00:00:00 2001 From: Jose Guedez Date: Wed, 11 Dec 2019 15:05:12 +1100 Subject: [PATCH 231/898] Move tests (400/401) from neutron-gateway to neutron-api --- zaza/openstack/charm_tests/neutron/tests.py | 132 ++++++++++---------- 1 file changed, 69 insertions(+), 63 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index f6cde67..4df7021 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -39,58 +39,11 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): cls.current_os_release = openstack_utils.get_os_release() cls.services = cls._get_services() - # set up clients - cls.neutron_client = ( - openstack_utils.get_neutron_session_client(cls.keystone_session)) - bionic_stein = openstack_utils.get_os_release('bionic_stein') cls.pgrep_full = (True if cls.current_os_release >= bionic_stein else False) - def test_400_create_network(self): - """Create a network, verify that it exists, and then delete it.""" - logging.debug('Creating neutron network...') - self.neutron_client.format = 'json' - net_name = 'test_net' - - # Verify that the network doesn't exist - networks = self.neutron_client.list_networks(name=net_name) - net_count = len(networks['networks']) - assert net_count == 0, ( - "Expected zero networks, found {}".format(net_count)) - - # Create a network and verify that it exists - network = {'name': net_name} - self.neutron_client.create_network({'network': network}) - - networks = self.neutron_client.list_networks(name=net_name) - logging.debug('Networks: {}'.format(networks)) - net_len = len(networks['networks']) - assert net_len == 1, ( - "Expected 1 network, found {}".format(net_len)) - - logging.debug('Confirming new neutron network...') - network = networks['networks'][0] - assert network['name'] == net_name, "network ext_net not found" - - # Cleanup - logging.debug('Deleting neutron network...') - self.neutron_client.delete_network(network['id']) - - def test_401_enable_qos(self): - """Check qos settings set via neutron-api charm.""" - if (self.current_os_release >= - openstack_utils.get_os_release('trusty_mitaka')): - logging.info('running qos check') - - with self.config_change( - {'enable-qos': 'False'}, - {'enable-qos': 'True'}, - application_name="neutron-api"): - - self._validate_openvswitch_agent_qos() - def test_900_restart_on_config_change(self): """Checking restart happens on config change. @@ -162,22 +115,6 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): output = run['Stdout'] self.assertTrue(int(output) >= len(services)) - @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60)) - def _validate_openvswitch_agent_qos(self): - """Validate that the qos extension is enabled in the ovs agent.""" - # obtain the dhcp agent to identify the neutron-gateway host - dhcp_agent = self.neutron_client.list_agents( - binary='neutron-dhcp-agent')['agents'][0] - neutron_gw_host = dhcp_agent['host'] - logging.debug('neutron gw host: {}'.format(neutron_gw_host)) - - # check extensions on the ovs agent to validate qos - ovs_agent = self.neutron_client.list_agents( - binary='neutron-openvswitch-agent', - host=neutron_gw_host)['agents'][0] - - self.assertIn('qos', ovs_agent['configurations']['extensions']) - @classmethod def _get_services(cls): """ @@ -208,6 +145,59 @@ class NeutronGatewayTest(test_utils.OpenStackBaseTest): class NeutronApiTest(test_utils.OpenStackBaseTest): """Test basic Neutron API Charm functionality.""" + @classmethod + def setUpClass(cls): + """Run class setup for running Neutron Gateway tests.""" + super(NeutronApiTest, cls).setUpClass() + cls.current_os_release = openstack_utils.get_os_release() + + # set up clients + cls.neutron_client = ( + openstack_utils.get_neutron_session_client(cls.keystone_session)) + + def test_400_create_network(self): + """Create a network, verify that it exists, and then delete it.""" + logging.debug('Creating neutron network...') + self.neutron_client.format = 'json' + net_name = 'test_net' + + # Verify that the network doesn't exist + networks = self.neutron_client.list_networks(name=net_name) + net_count = len(networks['networks']) + assert net_count == 0, ( + "Expected zero networks, found {}".format(net_count)) + + # Create a network and verify that it exists + network = {'name': net_name} + self.neutron_client.create_network({'network': network}) + + networks = self.neutron_client.list_networks(name=net_name) + logging.debug('Networks: {}'.format(networks)) + net_len = len(networks['networks']) + assert net_len == 1, ( + "Expected 1 network, found {}".format(net_len)) + + logging.debug('Confirming new neutron network...') + network = networks['networks'][0] + assert network['name'] == net_name, "network ext_net not found" + + # Cleanup + logging.debug('Deleting neutron network...') + self.neutron_client.delete_network(network['id']) + + def test_401_enable_qos(self): + """Check qos settings set via neutron-api charm.""" + if (self.current_os_release >= + openstack_utils.get_os_release('trusty_mitaka')): + logging.info('running qos check') + + with self.config_change( + {'enable-qos': 'False'}, + {'enable-qos': 'True'}, + application_name="neutron-api"): + + self._validate_openvswitch_agent_qos() + def test_900_restart_on_config_change(self): """Checking restart happens on config change. @@ -261,6 +251,22 @@ class NeutronApiTest(test_utils.OpenStackBaseTest): pgrep_full=pgrep_full): logging.info("Testing pause resume") + @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60)) + def _validate_openvswitch_agent_qos(self): + """Validate that the qos extension is enabled in the ovs agent.""" + # obtain the dhcp agent to identify the neutron-gateway host + dhcp_agent = self.neutron_client.list_agents( + binary='neutron-dhcp-agent')['agents'][0] + neutron_gw_host = dhcp_agent['host'] + logging.debug('neutron gw host: {}'.format(neutron_gw_host)) + + # check extensions on the ovs agent to validate qos + ovs_agent = self.neutron_client.list_agents( + binary='neutron-openvswitch-agent', + host=neutron_gw_host)['agents'][0] + + self.assertIn('qos', ovs_agent['configurations']['extensions']) + class SecurityTest(test_utils.OpenStackBaseTest): """Neutron Security Tests.""" From f3ebe916f282c080d50501e3edd895a40d3d15b8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 11 Dec 2019 12:12:25 +0000 Subject: [PATCH 232/898] Add get_subordinate_units Add zaza.openstack.utilities.juju.get_subordinate_units to get a list of all subordinate units associated with units in unit_list. Subordinate can be filtered by using 'charm_name' which will only return subordinate units which have 'charm_name' in the name of the charm. --- .../utilities/test_zaza_utilities_juju.py | 31 +++++++++++++ zaza/openstack/utilities/juju.py | 46 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_juju.py b/unit_tests/utilities/test_zaza_utilities_juju.py index 11bbcbe..4255f9e 100644 --- a/unit_tests/utilities/test_zaza_utilities_juju.py +++ b/unit_tests/utilities/test_zaza_utilities_juju.py @@ -279,3 +279,34 @@ class TestJujuUtils(ut_utils.BaseTestCase): model_name=None ) self.assertEqual(expected, actual) + + def test_get_subordinate_units(self): + juju_status = mock.MagicMock() + juju_status.applications = { + 'nova-compute': { + 'units': { + 'nova-compute/0': { + 'subordinates': { + 'neutron-openvswitch/2': { + 'charm': 'cs:neutron-openvswitch-22'}}}}}, + 'cinder': { + 'units': { + 'cinder/1': { + 'subordinates': { + 'cinder-hacluster/0': { + 'charm': 'cs:hacluster-42'}, + 'cinder-ceph/3': { + 'charm': 'cs:cinder-ceph-2'}}}}}, + } + self.assertEqual( + sorted(juju_utils.get_subordinate_units( + ['nova-compute/0', 'cinder/1'], + status=juju_status)), + sorted(['neutron-openvswitch/2', 'cinder-hacluster/0', + 'cinder-ceph/3'])) + self.assertEqual( + juju_utils.get_subordinate_units( + ['nova-compute/0', 'cinder/1'], + charm_name='ceph', + status=juju_status), + ['cinder-ceph/3']) diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index fd7abc1..99f1e10 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -311,3 +311,49 @@ def leader_get(application, key='', model_name=None): return yaml.safe_load(result.get('Stdout')) else: raise model.CommandRunFailed(cmd, result) + + +def get_subordinate_units(unit_list, charm_name=None, status=None, + model_name=None): + """Get a list of all subordinate units associated with units in unit_list. + + Get a list of all subordinate units associated with units in unit_list. + Subordinate can be filtered by using 'charm_name' which will only return + subordinate units which have 'charm_name' in the name of the charm e.g. + + get_subordinate_units( + ['cinder/1']) would return ['cinder-hacluster/1', + 'cinder-ceph/2']) + where as + + get_subordinate_units( + ['cinder/1'], charm_name='hac') would return ['cinder-hacluster/1'] + + NOTE: The charm_name match is against the name of the charm not the + application name. + + :param charm_name: List of unit names + :type unit_list: [] + :param charm_name: charm_name to match against, can be a sub-string. + :type charm_name: str + :param status: Juju status to query against, + :type status: juju.client._definitions.FullStatus + :param model_name: Name of model to query. + :type model_name: str + :returns: List of matching unit names. + :rtype: [] + """ + if not status: + status = model.get_status(model_name=model_name) + sub_units = [] + for unit_name in unit_list: + app_name = unit_name.split('/')[0] + subs = status.applications[app_name]['units'][unit_name].get( + 'subordinates') or {} + if charm_name: + for unit_name, unit_data in subs.items(): + if charm_name in unit_data['charm']: + sub_units.append(unit_name) + else: + sub_units.extend([n for n in subs.keys()]) + return sub_units From 4af37d955691e0c1da317d9f8f6af5527b2f7113 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 12 Dec 2019 13:57:55 +0000 Subject: [PATCH 233/898] Add functions for performing OpenStack upgrades --- .../test_zaza_utilities_openstack_upgrade.py | 273 +++++++++++++ zaza/openstack/utilities/openstack_upgrade.py | 371 ++++++++++++++++++ 2 files changed, 644 insertions(+) create mode 100644 unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py create mode 100755 zaza/openstack/utilities/openstack_upgrade.py diff --git a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py new file mode 100644 index 0000000..f711bb2 --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py @@ -0,0 +1,273 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import mock + +import unit_tests.utils as ut_utils +import zaza.openstack.utilities.openstack_upgrade as openstack_upgrade + + +class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): + + async def _arun_action_on_units(self, units, cmd, model_name=None, + raise_on_failure=True): + pass + + def setUp(self): + super(TestOpenStackUpgradeUtils, self).setUp() + self.patch_object( + openstack_upgrade.zaza.model, + "async_run_action_on_units") + self.async_run_action_on_units.side_effect = self._arun_action_on_units + self.patch_object( + openstack_upgrade.zaza.model, + "get_units") + self.juju_status = mock.MagicMock() + self.patch_object( + openstack_upgrade.zaza.model, + "get_status", + return_value=self.juju_status) + self.patch_object( + openstack_upgrade.zaza.model, + "set_application_config") + self.patch_object( + openstack_upgrade.zaza.model, + "get_application_config") + + def _get_application_config(app, model_name=None): + app_config = { + 'ceph-mon': {'verbose': True, 'source': 'old-src'}, + 'neutron-openvswitch': {'verbose': True}, + 'ntp': {'verbose': True}, + 'percona-cluster': {'verbose': True, 'source': 'old-src'}, + 'cinder': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + 'neutron-api': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + 'nova-compute': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + } + return app_config[app] + self.get_application_config.side_effect = _get_application_config + self.juju_status.applications = { + 'mydb': { # Filter as it is on UPGRADE_EXCLUDE_LIST + 'charm': 'cs:percona-cluster'}, + 'neutron-openvswitch': { # Filter as it is a subordinates + 'charm': 'cs:neutron-openvswitch', + 'subordinate-to': 'nova-compute'}, + 'ntp': { # Filter as it has no source option + 'charm': 'cs:ntp'}, + 'nova-compute': { + 'charm': 'cs:nova-compute', + 'units': { + 'nova-compute/0': { + 'subordinates': { + 'neutron-openvswitch/2': { + 'charm': 'cs:neutron-openvswitch-22'}}}}}, + 'cinder': { + 'charm': 'cs:cinder-23', + 'units': { + 'cinder/1': { + 'subordinates': { + 'cinder-hacluster/0': { + 'charm': 'cs:hacluster-42'}, + 'cinder-ceph/3': { + 'charm': 'cs:cinder-ceph-2'}}}}}} + + def test_pause_units(self): + openstack_upgrade.pause_units(['cinder/1', 'glance/2']) + self.async_run_action_on_units.assert_called_once_with( + ['cinder/1', 'glance/2'], + 'pause', + model_name=None, + raise_on_failure=True) + + def test_resume_units(self): + openstack_upgrade.resume_units(['cinder/1', 'glance/2']) + self.async_run_action_on_units.assert_called_once_with( + ['cinder/1', 'glance/2'], + 'resume', + model_name=None, + raise_on_failure=True) + + def test_action_unit_upgrade(self): + openstack_upgrade.action_unit_upgrade(['cinder/1', 'glance/2']) + self.async_run_action_on_units.assert_called_once_with( + ['cinder/1', 'glance/2'], + 'openstack-upgrade', + model_name=None, + raise_on_failure=True) + + def test_action_upgrade_group(self): + self.patch_object(openstack_upgrade, "pause_units") + self.patch_object(openstack_upgrade, "action_unit_upgrade") + self.patch_object(openstack_upgrade, "resume_units") + mock_nova_compute_0 = mock.MagicMock() + mock_nova_compute_0.entity_id = 'nova-compute/0' + mock_cinder_1 = mock.MagicMock() + mock_cinder_1.entity_id = 'cinder/1' + units = { + 'nova-compute': [mock_nova_compute_0], + 'cinder': [mock_cinder_1]} + self.get_units.side_effect = lambda app, model_name: units[app] + openstack_upgrade.action_upgrade_group(['nova-compute', 'cinder']) + pause_calls = [ + mock.call(['cinder-hacluster/0'], model_name=None), + mock.call(['nova-compute/0', 'cinder/1'], model_name=None)] + self.pause_units.assert_has_calls(pause_calls, any_order=False) + action_unit_upgrade_calls = [ + mock.call(['nova-compute/0', 'cinder/1'], model_name=None)] + self.action_unit_upgrade.assert_has_calls( + action_unit_upgrade_calls, + any_order=False) + resume_calls = [ + mock.call(['nova-compute/0', 'cinder/1'], model_name=None), + mock.call(['cinder-hacluster/0'], model_name=None)] + self.resume_units.assert_has_calls(resume_calls, any_order=False) + + def test_set_upgrade_application_config(self): + openstack_upgrade.set_upgrade_application_config( + ['neutron-api', 'cinder'], + 'new-src') + set_app_calls = [ + mock.call( + 'neutron-api', + { + 'openstack-origin': 'new-src', + 'action-managed-upgrade': 'True'}, + model_name=None), + mock.call( + 'cinder', + { + 'openstack-origin': 'new-src', + 'action-managed-upgrade': 'True'}, + model_name=None)] + self.set_application_config.assert_has_calls(set_app_calls) + + self.set_application_config.reset_mock() + openstack_upgrade.set_upgrade_application_config( + ['percona-cluster'], + 'new-src', + action_managed=False) + self.set_application_config.assert_called_once_with( + 'percona-cluster', + {'source': 'new-src'}, + model_name=None) + + def test__extract_charm_name_from_url(self): + self.assertEqual( + openstack_upgrade._extract_charm_name_from_url( + 'local:bionic/heat-12'), + 'heat') + self.assertEqual( + openstack_upgrade._extract_charm_name_from_url( + 'cs:bionic/heat-12'), + 'heat') + self.assertEqual( + openstack_upgrade._extract_charm_name_from_url('cs:heat'), + 'heat') + + def test_get_upgrade_candidates(self): + expect = copy.deepcopy(self.juju_status.applications) + del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST + del expect['ntp'] # Filter as it has no source option + del expect['neutron-openvswitch'] # Filter as it is a subordinates + self.assertEqual( + openstack_upgrade.get_upgrade_candidates(), + expect) + + def test_get_upgrade_groups(self): + self.assertEqual( + openstack_upgrade.get_upgrade_groups(), + { + 'Compute': ['nova-compute'], + 'Control Plane': ['cinder'], + 'Core Identity': [], + 'Storage': [], + 'sweep_up': []}) + + def test_is_action_upgradable(self): + self.assertTrue( + openstack_upgrade.is_action_upgradable('cinder')) + self.assertFalse( + openstack_upgrade.is_action_upgradable('percona-cluster')) + + def test_run_action_upgrade(self): + self.patch_object(openstack_upgrade, "set_upgrade_application_config") + self.patch_object(openstack_upgrade, "action_upgrade_group") + openstack_upgrade.run_action_upgrade( + ['cinder', 'neutron-api'], + 'new-src') + self.set_upgrade_application_config.assert_called_once_with( + ['cinder', 'neutron-api'], + 'new-src', + model_name=None) + self.action_upgrade_group.assert_called_once_with( + ['cinder', 'neutron-api'], + model_name=None) + + def test_run_all_in_one_upgrade(self): + self.patch_object(openstack_upgrade, "set_upgrade_application_config") + self.patch_object( + openstack_upgrade.zaza.model, + 'block_until_all_units_idle') + openstack_upgrade.run_all_in_one_upgrade( + ['percona-cluster'], + 'new-src') + self.set_upgrade_application_config.assert_called_once_with( + ['percona-cluster'], + 'new-src', + action_managed=False, + model_name=None) + self.block_until_all_units_idle.assert_called_once_with() + + def test_run_upgrade(self): + self.patch_object(openstack_upgrade, "run_all_in_one_upgrade") + self.patch_object(openstack_upgrade, "run_action_upgrade") + openstack_upgrade.run_upgrade( + ['cinder', 'neutron-api', 'ceph-mon'], + 'new-src') + self.run_all_in_one_upgrade.assert_called_once_with( + ['ceph-mon'], + 'new-src', + model_name=None) + self.run_action_upgrade.assert_called_once_with( + ['cinder', 'neutron-api'], + 'new-src', + model_name=None) + + def test_run_upgrade_tests(self): + self.patch_object(openstack_upgrade, "run_upgrade") + self.patch_object(openstack_upgrade, "get_upgrade_groups") + self.get_upgrade_groups.return_value = { + 'Compute': ['nova-compute'], + 'Control Plane': ['cinder', 'neutron-api'], + 'Core Identity': ['keystone'], + 'Storage': ['ceph-mon'], + 'sweep_up': ['designate']} + openstack_upgrade.run_upgrade_tests('new-src', model_name=None) + run_upgrade_calls = [ + mock.call(['keystone'], 'new-src', model_name=None), + mock.call(['ceph-mon'], 'new-src', model_name=None), + mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None), + mock.call(['nova-compute'], 'new-src', model_name=None), + mock.call(['designate'], 'new-src', model_name=None)] + self.run_upgrade.assert_has_calls(run_upgrade_calls, any_order=False) diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py new file mode 100755 index 0000000..3c4aa8f --- /dev/null +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -0,0 +1,371 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for performing OpenStack upgrades. + +This module contains a number of functions for upgrading OpenStack. +""" +import re +import logging +import zaza.openstack.utilities.juju as juju_utils + +import zaza.model +from zaza import sync_wrapper + +SERVICE_GROUPS = { + 'Core Identity': ['keystone'], + 'Storage': [ + 'ceph-mon', 'ceph-osd', 'ceph-fs', 'ceph-radosgw', 'swift-proxy', + 'swift-storage'], + 'Control Plane': [ + 'aodh', 'barbican', 'ceilometer', 'cinder', 'designate', + 'designate-bind', 'glance', 'gnocchi', 'heat', 'manila', + 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', + 'nova-cloud-controller', 'openstack-dashboard'], + 'Compute': ['nova-compute']} + +UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] + + +async def async_pause_units(units, model_name=None): + """Pause all units in unit list. + + Pause all units in unit list. Wait for pause action + to complete. + + :param units: List of unit names. + :type units: [] + :param model_name: Name of model to query. + :type model_name: str + :rtype: juju.action.Action + :raises: zaza.model.ActionFailed + """ + logging.info("Pausing {}".format(', '.join(units))) + await zaza.model.async_run_action_on_units( + units, + 'pause', + model_name=model_name, + raise_on_failure=True) + +pause_units = sync_wrapper(async_pause_units) + + +async def async_resume_units(units, model_name=None): + """Resume all units in unit list. + + Resume all units in unit list. Wait for resume action + to complete. + + :param units: List of unit names. + :type units: [] + :param model_name: Name of model to query. + :type model_name: str + :rtype: juju.action.Action + :raises: zaza.model.ActionFailed + """ + logging.info("Resuming {}".format(', '.join(units))) + await zaza.model.async_run_action_on_units( + units, + 'resume', + model_name=model_name, + raise_on_failure=True) + +resume_units = sync_wrapper(async_resume_units) + + +async def async_action_unit_upgrade(units, model_name=None): + """Run openstack-upgrade on all units in unit list. + + Upgrade payload on all units in unit list. Wait for action + to complete. + + :param units: List of unit names. + :type units: [] + :param model_name: Name of model to query. + :type model_name: str + :rtype: juju.action.Action + :raises: zaza.model.ActionFailed + """ + logging.info("Upgrading {}".format(', '.join(units))) + await zaza.model.async_run_action_on_units( + units, + 'openstack-upgrade', + model_name=model_name, + raise_on_failure=True) + +action_unit_upgrade = sync_wrapper(async_action_unit_upgrade) + + +def action_upgrade_group(applications, model_name=None): + """Upgrade units using action managed upgrades. + + Upgrade all units of the given applications using action managed upgrades. + This involves the following process: + 1) Take a unit from each application which has not been upgraded yet. + 2) Pause all hacluster units assocaiated with units to be upgraded. + 3) Pause target units. + 4) Upgrade target units. + 5) Resume target units. + 6) Resume hacluster units paused in step 2. + 7) Repeat until all units are upgraded. + + :param applications: List of application names. + :type applications: [] + :param model_name: Name of model to query. + :type model_name: str + """ + status = zaza.model.get_status(model_name=model_name) + done = [] + while True: + target = [] + for app in applications: + for unit in zaza.model.get_units(app, model_name=model_name): + if unit.entity_id not in done: + target.append(unit.entity_id) + break + else: + logging.info("All units of {} upgraded".format(app)) + if not target: + break + hacluster_units = juju_utils.get_subordinate_units( + target, + 'hacluster', + status=status, + model_name=model_name) + + pause_units(hacluster_units, model_name=model_name) + pause_units(target, model_name=model_name) + + action_unit_upgrade(target, model_name=model_name) + + resume_units(target, model_name=model_name) + resume_units(hacluster_units, model_name=model_name) + + done.extend(target) + + +def set_upgrade_application_config(applications, new_source, + action_managed=True, model_name=None): + """Set the charm config for upgrade. + + Set the charm config for upgrade. + + :param applications: List of application names. + :type applications: [] + :param new_source: New package origin. + :type new_source: str + :param action_managed: Whether to set action-managed-upgrade config option. + :type action_managed: bool + :param model_name: Name of model to query. + :type model_name: str + """ + for app in applications: + src_option = 'openstack-origin' + charm_options = zaza.model.get_application_config( + app, model_name=model_name) + try: + charm_options[src_option] + except KeyError: + src_option = 'source' + config = { + src_option: new_source} + if action_managed: + config['action-managed-upgrade'] = 'True' + logging.info("Setting config for {} to {}".format(app, config)) + zaza.model.set_application_config( + app, + config, + model_name=model_name) + + +def _extract_charm_name_from_url(charm_url): + """Extract the charm name from the charm url. + + E.g. Extract 'heat' from local:bionic/heat-12 + + :param charm_url: Name of model to query. + :type charm_url: str + :returns: Charm name + :rtype: str + """ + charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) + return charm_name.split(':')[-1] + + +def get_upgrade_candidates(model_name=None): + """Extract list of apps from model that can be upgraded. + + :param model_name: Name of model to query. + :type model_name: str + :returns: List of application that can have their payload upgraded. + :rtype: [] + """ + status = zaza.model.get_status(model_name=model_name) + candidates = {} + for app, app_config in status.applications.items(): + # Filter out subordinates + if app_config.get("subordinate-to"): + logging.warning( + "Excluding {} from upgrade, it is a subordinate".format(app)) + continue + + # Filter out charms on the naughty list + charm_name = _extract_charm_name_from_url(app_config['charm']) + if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: + logging.warning( + "Excluding {} from upgrade, on the exclude list".format(app)) + continue + + # Filter out charms that have no source option + charm_options = zaza.model.get_application_config( + app, model_name=model_name).keys() + src_options = ['openstack-origin', 'source'] + if not [x for x in src_options if x in charm_options]: + logging.warning( + "Excluding {} from upgrade, no src option".format(app)) + continue + + candidates[app] = app_config + return candidates + + +def get_upgrade_groups(model_name=None): + """Place apps in the model into their upgrade groups. + + Place apps in the model into their upgrade groups. If an app is deployed + but is not in SERVICE_GROUPS then it is placed in a sweep_up group. + + :param model_name: Name of model to query. + :type model_name: str + :returns: Dict of group lists keyed on group name. + :rtype: {} + """ + apps_in_model = get_upgrade_candidates(model_name=model_name) + + groups = {} + for phase_name, charms in SERVICE_GROUPS.items(): + group = [] + for app, app_config in apps_in_model.items(): + charm_name = _extract_charm_name_from_url(app_config['charm']) + if charm_name in charms: + group.append(app) + groups[phase_name] = group + + sweep_up = [] + for app in apps_in_model: + if not (app in [a for group in groups.values() for a in group]): + sweep_up.append(app) + + groups['sweep_up'] = sweep_up + return groups + + +def is_action_upgradable(app, model_name=None): + """Can application be upgraded using action managed upgrade method. + + :param new_source: New package origin. + :type new_source: str + :param model_name: Name of model to query. + :type model_name: str + :returns: Whether app be upgraded using action managed upgrade method. + :rtype: bool + """ + config = zaza.model.get_application_config(app, model_name=model_name) + try: + config['action-managed-upgrade'] + supported = True + except KeyError: + supported = False + return supported + + +def run_action_upgrade(group, new_source, model_name=None): + """Upgrade payload of all applications in group using action upgrades. + + :param group: List of applications to upgrade. + :type group + :param new_source: New package origin. + :type new_source: str + :param model_name: Name of model to query. + :type model_name: str + """ + set_upgrade_application_config(group, new_source, model_name=model_name) + action_upgrade_group(group, model_name=model_name) + + +def run_all_in_one_upgrade(group, new_source, model_name=None): + """Upgrade payload of all applications in group using all-in-one method. + + :param group: List of applications to upgrade. + :type group: [] + :source: New package origin. + :type new_source: str + :param model_name: Name of model to query. + :type model_name: str + """ + set_upgrade_application_config( + group, + new_source, + model_name=model_name, + action_managed=False) + zaza.model.block_until_all_units_idle() + + +def run_upgrade(group, new_source, model_name=None): + """Upgrade payload of all applications in group. + + Upgrade apps using action managed upgrades where possible and fallback to + all_in_one method. + + :param group: List of applications to upgrade. + :type group: [] + :param new_source: New package origin. + :type new_source: str + :param model_name: Name of model to query. + :type model_name: str + """ + action_upgrade = [] + all_in_one_upgrade = [] + for app in group: + if is_action_upgradable(app, model_name=model_name): + action_upgrade.append(app) + else: + all_in_one_upgrade.append(app) + run_all_in_one_upgrade( + all_in_one_upgrade, + new_source, + model_name=model_name) + run_action_upgrade( + action_upgrade, + new_source, + model_name=model_name) + + +def run_upgrade_tests(new_source, model_name=None): + """Upgrade payload of all applications in model. + + This the most basic upgrade test. It should be adapted to add/remove + elements from the environment and add tests at intermediate stages. + + :param new_source: New package origin. + :type new_source: str + :param model_name: Name of model to query. + :type model_name: str + """ + groups = get_upgrade_groups(model_name=model_name) + run_upgrade(groups['Core Identity'], new_source, model_name=model_name) + run_upgrade(groups['Storage'], new_source, model_name=model_name) + run_upgrade(groups['Control Plane'], new_source, model_name=model_name) + run_upgrade(groups['Compute'], new_source, model_name=model_name) + run_upgrade(groups['sweep_up'], new_source, model_name=model_name) From 2c9e9739d6609c899203327357a1bfe1d8efd29c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 17 Dec 2019 11:57:23 +0000 Subject: [PATCH 234/898] Fix name of swift storage-region option --- zaza/openstack/utilities/swift.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/swift.py b/zaza/openstack/utilities/swift.py index 67daca0..2b1f46a 100644 --- a/zaza/openstack/utilities/swift.py +++ b/zaza/openstack/utilities/swift.py @@ -204,7 +204,7 @@ def get_swift_storage_topology(model_name=None): app_config = zaza.model.get_application_config( app_name, model_name=model_name) - region = app_config['region']['value'] + region = app_config['storage-region']['value'] zone = app_config['zone']['value'] for unit in zaza.model.get_units(app_name, model_name=model_name): topology[unit.public_address] = { From 721f3bc90554bc22969c96cb9a8f0671d0c6863a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 17 Dec 2019 12:21:39 +0000 Subject: [PATCH 235/898] Fix unit test --- unit_tests/utilities/test_zaza_utilities_swift.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/utilities/test_zaza_utilities_swift.py b/unit_tests/utilities/test_zaza_utilities_swift.py index 8c87556..c5fc3da 100644 --- a/unit_tests/utilities/test_zaza_utilities_swift.py +++ b/unit_tests/utilities/test_zaza_utilities_swift.py @@ -104,7 +104,7 @@ class TestSwiftUtils(ut_utils.BaseTestCase): region = int(app_name.split('-')[2].replace('region', '')) zone = int(app_name.split('-')[3].replace('zone', '')) app_config[app_name] = { - 'region': {'value': region}, + 'storage-region': {'value': region}, 'zone': {'value': zone}} self.get_application_config.side_effect = \ From 2f88a94f97204e434e48bcb38f1e0e6852d5f687 Mon Sep 17 00:00:00 2001 From: Chris Johnston Date: Tue, 17 Dec 2019 16:56:24 +0000 Subject: [PATCH 236/898] Add Zaza tests for HAProxy expose functionality. Partial-Bug: #1710208 --- .../charm_tests/openstack_dashboard/tests.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index d15a603..df39b65 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -14,6 +14,7 @@ """Encapsulate horizon (openstack-dashboard) charm testing.""" +import base64 import http.client import logging import requests @@ -26,6 +27,7 @@ import zaza.model as zaza_model import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.juju as openstack_juju +import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.charm_tests.policyd.tests as policyd @@ -206,6 +208,64 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): target_status='running' ) + def test_200_haproxy_stats_config(self): + """Verify that the HAProxy stats are properly setup.""" + logging.info('Checking dashboard HAProxy settings...') + application_name = 'openstack-dashboard' + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name(application_name)) + keystone_unit = zaza_model.get_lead_unit_name('keystone') + dashboard_relation = openstack_juju.get_relation_from_unit( + keystone_unit, unit.entity_id, 'identity-service') + dashboard_ip = dashboard_relation['private-address'] + logging.debug("... dashboard_ip is:{}".format(dashboard_ip)) + conf = '/etc/haproxy/haproxy.cfg' + port = '8888' + set_alternate = { + 'haproxy-expose-stats': 'True', + } + + request = urllib.request.Request( + 'http://{}:{}'.format(dashboard_ip, port)) + + # NOTE(ajkavanagh): it seems that apache2 doesn't start quickly enough + # for the test, and so it gets reset errors; repeat until either that + # stops or there is a failure + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, + min=5, max=10), + retry=tenacity.retry_if_exception_type( + http.client.RemoteDisconnected), + reraise=True) + def _do_request(): + return urllib.request.urlopen(request) + + output = str(generic_utils.get_file_contents(unit, conf)) + + for line in output.split('\n'): + if "stats auth" in line: + password = line.split(':')[1] + base64string = base64.b64encode(bytes('%s:%s' % ('admin', password), + 'ascii')) + request.add_header( + "Authorization", "Basic %s" % base64string.decode('utf-8')) + + # Expect default config to not be available externally. + expected = 'bind 127.0.0.1:{}'.format(port) + self.assertIn(expected, output) + with self.assertRaises(urllib.error.URLError): + _do_request() + + zaza_model.set_application_config(application_name, set_alternate) + zaza_model.block_until_all_units_idle(model_name=self.model_name) + + # Once exposed, expect HAProxy stats to be available externally + output = str(generic_utils.get_file_contents(unit, conf)) + expected = 'bind 0.0.0.0:{}'.format(port) + html = _do_request().read().decode(encoding='utf-8') + self.assertIn(expected, output) + self.assertIn('Statistics Report for HAProxy', html, + "HAProxy stats check failed") + def test_302_router_settings(self): """Verify that the horizon router settings are correct. From 9fc0602c645fc8d4f6703249fb319a518888ecbc Mon Sep 17 00:00:00 2001 From: Chris Johnston Date: Tue, 17 Dec 2019 21:19:12 +0000 Subject: [PATCH 237/898] Updates per review --- .../charm_tests/openstack_dashboard/tests.py | 136 ++++++------------ 1 file changed, 46 insertions(+), 90 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index df39b65..4986f7b 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -26,7 +26,6 @@ import zaza.model as zaza_model import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.utilities.juju as openstack_juju import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.charm_tests.policyd.tests as policyd @@ -43,21 +42,6 @@ class FailedAuth(AuthExceptions): pass -def _get_dashboard_ip(): - """Get the IP of the dashboard. - - :returns: The IP of the dashboard - :rtype: str - """ - unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') - keystone_unit = zaza_model.get_lead_unit_name('keystone') - dashboard_relation = openstack_juju.get_relation_from_unit( - keystone_unit, unit_name, 'identity-service') - dashboard_ip = dashboard_relation['private-address'] - logging.debug("dashboard_ip is: {}".format(dashboard_ip)) - return dashboard_ip - - # NOTE: intermittent authentication fails. Wrap in a retry loop. @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), @@ -156,6 +140,17 @@ def _login(dashboard_ip, domain, username, password): return client, response +# NOTE(ajkavanagh): it seems that apache2 doesn't start quickly enough +# for the test, and so it gets reset errors; repeat until either that +# stops or there is a failure +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), + retry=tenacity.retry_if_exception_type( + http.client.RemoteDisconnected), + reraise=True) +def _do_request(request): + return urllib.request.urlopen(request) + + class OpenStackDashboardTests(test_utils.OpenStackBaseTest): """Encapsulate openstack dashboard charm tests.""" @@ -211,14 +206,9 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): def test_200_haproxy_stats_config(self): """Verify that the HAProxy stats are properly setup.""" logging.info('Checking dashboard HAProxy settings...') - application_name = 'openstack-dashboard' unit = zaza_model.get_unit_from_name( - zaza_model.get_lead_unit_name(application_name)) - keystone_unit = zaza_model.get_lead_unit_name('keystone') - dashboard_relation = openstack_juju.get_relation_from_unit( - keystone_unit, unit.entity_id, 'identity-service') - dashboard_ip = dashboard_relation['private-address'] - logging.debug("... dashboard_ip is:{}".format(dashboard_ip)) + zaza_model.get_lead_unit_name(self.application_name)) + logging.debug("... dashboard_ip is:{}".format(unit.public_address)) conf = '/etc/haproxy/haproxy.cfg' port = '8888' set_alternate = { @@ -226,26 +216,15 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): } request = urllib.request.Request( - 'http://{}:{}'.format(dashboard_ip, port)) - - # NOTE(ajkavanagh): it seems that apache2 doesn't start quickly enough - # for the test, and so it gets reset errors; repeat until either that - # stops or there is a failure - @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, - min=5, max=10), - retry=tenacity.retry_if_exception_type( - http.client.RemoteDisconnected), - reraise=True) - def _do_request(): - return urllib.request.urlopen(request) + 'http://{}:{}'.format(unit.public_address, port)) output = str(generic_utils.get_file_contents(unit, conf)) for line in output.split('\n'): if "stats auth" in line: password = line.split(':')[1] - base64string = base64.b64encode(bytes('%s:%s' % ('admin', password), - 'ascii')) + base64string = base64.b64encode( + bytes('{}:{}'.format('admin', password), 'ascii')) request.add_header( "Authorization", "Basic %s" % base64string.decode('utf-8')) @@ -253,15 +232,15 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): expected = 'bind 127.0.0.1:{}'.format(port) self.assertIn(expected, output) with self.assertRaises(urllib.error.URLError): - _do_request() + _do_request(request) - zaza_model.set_application_config(application_name, set_alternate) + zaza_model.set_application_config(self.application_name, set_alternate) zaza_model.block_until_all_units_idle(model_name=self.model_name) # Once exposed, expect HAProxy stats to be available externally output = str(generic_utils.get_file_contents(unit, conf)) expected = 'bind 0.0.0.0:{}'.format(port) - html = _do_request().read().decode(encoding='utf-8') + html = _do_request(request).read().decode(encoding='utf-8') self.assertIn(expected, output) self.assertIn('Statistics Report for HAProxy', html, "HAProxy stats check failed") @@ -311,33 +290,20 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): """ logging.info('Checking dashboard http response...') - unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') - keystone_unit = zaza_model.get_lead_unit_name('keystone') - dashboard_relation = openstack_juju.get_relation_from_unit( - keystone_unit, unit_name, 'identity-service') - dashboard_ip = dashboard_relation['private-address'] - logging.debug("... dashboard_ip is:{}".format(dashboard_ip)) + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name(self.application_name)) + logging.debug("... dashboard_ip is:{}".format(unit.public_address)) - # NOTE(fnordahl) there is a eluding issue that currently makes the - # first request to the OpenStack Dashboard error out - # with 500 Internal Server Error in CI. Temporarilly - # add retry logic to unwedge the gate. This issue - # should be revisited and root caused properly when time - # allows. - @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, - min=5, max=10), - reraise=True) - def do_request(): + request = urllib.request.Request( + 'http://{}/horizon'.format(unit.public_address)) + try: logging.info("... trying to fetch the page") - try: - response = urllib.request.urlopen('http://{}/horizon' - .format(dashboard_ip)) - logging.info("... fetched page") - except Exception as e: - logging.info("... exception raised was {}".format(str(e))) - raise - return response.read().decode('utf-8') - html = do_request() + html = _do_request(request) + logging.info("... fetched page") + except Exception as e: + logging.info("... exception raised was {}".format(str(e))) + raise + return html.read().decode('utf-8') self.assertIn('OpenStack Dashboard', html, "Dashboard frontpage check failed") @@ -345,7 +311,8 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): """Validate that authentication succeeds for client log in.""" logging.info('Checking authentication through dashboard...') - dashboard_ip = _get_dashboard_ip() + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name(self.application_name)) overcloud_auth = openstack_utils.get_overcloud_auth() password = overcloud_auth['OS_PASSWORD'], logging.info("admin password is {}".format(password)) @@ -354,7 +321,7 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): domain = 'admin_domain', username = 'admin', password = overcloud_auth['OS_PASSWORD'], - _login(dashboard_ip, domain, username, password) + _login(unit.public_address, domain, username, password) logging.info('OK') def test_404_connection(self): @@ -364,34 +331,21 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): """ logging.info('Checking apache mod_status gets disabled.') - unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') - keystone_unit = zaza_model.get_lead_unit_name('keystone') - dashboard_relation = openstack_juju.get_relation_from_unit( - keystone_unit, unit_name, 'identity-service') - dashboard_ip = dashboard_relation['private-address'] - logging.debug("... dashboard_ip is:{}".format(dashboard_ip)) + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name(self.application_name)) + logging.debug("... dashboard_ip is: {}".format(unit.public_address)) logging.debug('Maybe enabling hardening for apache...') _app_config = zaza_model.get_application_config(self.application_name) logging.info(_app_config['harden']) - # NOTE(ajkavanagh): it seems that apache2 doesn't start quickly enough - # for the test, and so it gets reset errors; repeat until either that - # stops or there is a failure - @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, - min=5, max=10), - retry=tenacity.retry_if_exception_type( - http.client.RemoteDisconnected), - reraise=True) - def _do_request(): - return urllib.request.urlopen('http://{}/server-status' - .format(dashboard_ip)) - + request = urllib.request.Request( + 'http://{}/server-status'.format(unit.public_address)) with self.config_change( {'harden': _app_config['harden'].get('value', '')}, {'harden': 'apache'}): try: - _do_request() + _do_request(request) except urllib.request.HTTPError as e: # test failed if it didn't return 404 msg = "Apache mod_status check failed." @@ -491,8 +445,9 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): :type ip: str :raises: PolicydOperationFailedException if operation fails. """ - dashboard_ip = _get_dashboard_ip() - logging.info("Dashboard is at {}".format(dashboard_ip)) + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name(self.application_name)) + logging.info("Dashboard is at {}".format(unit.public_address)) overcloud_auth = openstack_utils.get_overcloud_auth() password = overcloud_auth['OS_PASSWORD'], logging.info("admin password is {}".format(password)) @@ -501,9 +456,10 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): domain = 'admin_domain', username = 'admin', password = overcloud_auth['OS_PASSWORD'], - client, response = _login(dashboard_ip, domain, username, password) + client, response = _login( + unit.public_address, domain, username, password) # now attempt to get the domains page - _url = self.url.format(dashboard_ip) + _url = self.url.format(unit.public_address) result = client.get(_url) if result.status_code == 403: raise policyd.PolicydOperationFailedException("Not authenticated") From 7a2c0fb65104d840d37136c96c271f570395567c Mon Sep 17 00:00:00 2001 From: Chris Johnston Date: Tue, 17 Dec 2019 22:19:01 +0000 Subject: [PATCH 238/898] Additional tweak --- zaza/openstack/charm_tests/openstack_dashboard/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 4986f7b..5964572 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -226,7 +226,7 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): base64string = base64.b64encode( bytes('{}:{}'.format('admin', password), 'ascii')) request.add_header( - "Authorization", "Basic %s" % base64string.decode('utf-8')) + "Authorization", "Basic {}".format(base64string.decode('utf-8'))) # Expect default config to not be available externally. expected = 'bind 127.0.0.1:{}'.format(port) From 50f59911ddcdc6b5ecb5f56deae0e554e91acdcf Mon Sep 17 00:00:00 2001 From: Chris Johnston Date: Wed, 18 Dec 2019 00:37:07 +0000 Subject: [PATCH 239/898] Add docstring --- zaza/openstack/charm_tests/openstack_dashboard/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 5964572..187cdc3 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -148,6 +148,14 @@ def _login(dashboard_ip, domain, username, password): http.client.RemoteDisconnected), reraise=True) def _do_request(request): + """Open a webpage via urlopen. + + :param request: A urllib request object. + :type request: object + :returns: HTTPResponse object + :rtype: object + :raises: URLError on protocol errors + """ return urllib.request.urlopen(request) From 574531aaa98ed45261e221e9052741346065ed12 Mon Sep 17 00:00:00 2001 From: Xiyue Wang Date: Thu, 19 Dec 2019 23:06:31 +0000 Subject: [PATCH 240/898] Port hacluster tests from Amulet to Zaza Signed-off-by: Xiyue Wang --- .../charm_tests/hacluster/__init__.py | 15 +++ zaza/openstack/charm_tests/hacluster/tests.py | 125 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 zaza/openstack/charm_tests/hacluster/__init__.py create mode 100644 zaza/openstack/charm_tests/hacluster/tests.py diff --git a/zaza/openstack/charm_tests/hacluster/__init__.py b/zaza/openstack/charm_tests/hacluster/__init__.py new file mode 100644 index 0000000..3825f7d --- /dev/null +++ b/zaza/openstack/charm_tests/hacluster/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing hacluster.""" diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py new file mode 100644 index 0000000..7ac67dc --- /dev/null +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""HACluster testing.""" + +import logging +import os + +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.juju as juju_utils +import zaza.openstack.configure.hacluster + + +class HaclusterTest(test_utils.OpenStackBaseTest): + """hacluster tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running hacluster tests.""" + super(HaclusterTest, cls).setUpClass() + cls.vip = os.environ.get("TEST_VIP00") + + def test_900_action_cleanup(self): + """The services can be cleaned up.""" + status = zaza.model.get_status().applications[self.application_name] + + # libjuju juju status no longer has units for subordinate charms + # Use the application it is subordinate-to to check workload status + if status.get("units") is None and status.get("subordinate-to"): + primary_status = juju_utils.get_application_status( + status.get("subordinate-to")[0]) + leader = None + for unit in primary_status["units"]: + if primary_status["units"][unit].get('leader'): + leader = unit + + if primary_status["units"][leader].get("subordinates"): + for subordinate in primary_status["units"][leader]["subordinates"]: + logging.info("Cleaning {}".format(subordinate)) + _action = "cleanup" + action_id = zaza.model.run_action(subordinate, "cleanup") + assert "success" in action_id.data["results"]["result"], ( + "Set hacluster action {} failed: {}" + .format(_action, action_id.data)) + + logging.info("Cleaning action w/resource {}" + .format(subordinate)) + params = {'resource': 'res_ks_haproxy'} + _action = "cleanup res_ks_haproxy" + zaza.model.run_action(subordinate, "cleanup", + action_params=params) + assert "success" in action_id.data["results"]["result"], ( + "Set hacluster action {} failed: {}" + .format(_action, action_id.data)) + + def test_910_pause_and_resume(self): + """The services can be paused and resumed.""" + logging.debug('Checking pause and resume actions...') + + status = zaza.model.get_status().applications[self.application_name] + + # libjuju juju status no longer has units for subordinate charms + # Use the application it is subordinate-to to check workload status + if status.get("units") is None and status.get("subordinate-to"): + primary_status = juju_utils.get_application_status( + status.get("subordinate-to")[0]) + leader = None + for unit in primary_status["units"]: + if primary_status["units"][unit].get('leader'): + leader = unit + + if primary_status["units"][leader].get("subordinates"): + for subordinate in primary_status["units"][leader]["subordinates"]: + logging.info("Pausing {}".format(subordinate)) + zaza.model.run_action(subordinate, "pause") + zaza.model.block_until_unit_wl_status(leader, "blocked") + + logging.info("Resuming {}".format(subordinate)) + zaza.model.run_action(subordinate, "resume") + zaza.model.block_until_unit_wl_status(leader, "active") + + _states = {"hacluster": { + "workload-status": "active", + "workload-status-message": "Unit is ready and clustered"}} + zaza.model.wait_for_application_states(states=_states) + logging.debug('OK') + + def _toggle_maintenance_and_wait(self, expected): + """Configure cluster maintenance-mode. + + :param expected: expected value to set maintenance-mode + """ + config = {"maintenance-mode": expected} + logging.info("Setting config to {}".format(config)) + zaza.model.set_application_config(self.application_name, config) + if expected == 'true': + _states = {"hacluster": { + "workload-status": "maintenance", + "workload-status-message": "Pacemaker in maintenance mode"}} + else: + _states = {"hacluster": { + "workload-status": "active", + "workload-status-message": "Unit is ready and clustered"}} + zaza.model.wait_for_application_states(states=_states) + logging.debug('OK') + + def test_920_put_in_maintenance(self): + """Put pacemaker in maintenance mode.""" + logging.debug('Setting cluster in maintenance mode') + + self._toggle_maintenance_and_wait('true') + self._toggle_maintenance_and_wait('false') From 799519913603cc4880b293aad67246aa00958e0e Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 9 Jan 2020 20:17:19 +0000 Subject: [PATCH 241/898] Run complete-cluster-series-upgrade After perfomring a series upgrade on rabbitmq run complete-cluster-series-upgrade --- zaza/openstack/charm_tests/series_upgrade/tests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 3d378cb..107ac80 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -109,6 +109,15 @@ class SeriesUpgradeTest(unittest.TestCase): files=self.files, post_upgrade_functions=post_upgrade_functions) + if "rabbitmq-server" in applications[application]["charm"]: + logging.info( + "Running complete-cluster-series-upgrade action on leader") + model.run_action_on_leader( + 'rabbitmq-server', + 'complete-cluster-series-upgrade', + action_params={}) + model.block_until_all_units_idle() + class OpenStackSeriesUpgrade(SeriesUpgradeTest): """OpenStack Series Upgrade. From ccec8f0c6833eca3041befdaa7bc59bcb5db5f4c Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 10 Jan 2020 08:24:30 +0100 Subject: [PATCH 242/898] Add a short exponential retry to the mount setup bits --- .../charm_tests/manila_ganesha/tests.py | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index eb4f2be..44ed2cd 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -16,6 +16,7 @@ """Encapsulate Manila Ganesha testing.""" +from tenacity import Retrying, stop_after_attempt, wait_exponential from manilaclient import client as manilaclient @@ -88,21 +89,31 @@ packages: mount_path = share.export_locations[0] - openstack_utils.ssh_command( - username, fip_1, 'instance-1', - 'sudo mkdir -p /mnt/ceph && ' - 'sudo mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph && ' - 'echo "test" | sudo tee /mnt/ceph/test'.format( - mount_path), - password=password, privkey=privkey, verify=verify_setup) + for attempt in Retrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=10)): + with attempt: + openstack_utils.ssh_command( + username, fip_1, 'instance-1', + 'sudo mkdir -p /mnt/ceph && ' + 'sudo mount -t nfs -o nfsvers=4.1,proto=tcp ' + '{} /mnt/ceph && ' + 'echo "test" | sudo tee /mnt/ceph/test'.format( + mount_path), + password=password, privkey=privkey, verify=verify_setup) - # Read that file on instance_2 - openstack_utils.ssh_command( - username, fip_2, 'instance-2', - 'sudo mkdir -p /mnt/ceph && ' - 'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp {} /mnt/ceph' - .format(mount_path), - password=password, privkey=privkey, verify=verify_setup) + for attempt in Retrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=10)): + with attempt: + # Setup that file on instance_2 + openstack_utils.ssh_command( + username, fip_2, 'instance-2', + 'sudo mkdir -p /mnt/ceph && ' + 'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp ' + '{} /mnt/ceph' + .format(mount_path), + password=password, privkey=privkey, verify=verify_setup) def verify(stdin, stdout, stderr): status = stdout.channel.recv_exit_status() From c48801d2027c4d7d3cf9afd7adf0915d65fb05cd Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 13 Jan 2020 15:26:00 +0100 Subject: [PATCH 243/898] Allow us to retry the Manila client connection It is possible that Manila looks ready before it is, so we should retry a bit to ensure that the APi service is really down before giving up --- zaza/openstack/charm_tests/manila/tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index 54f7ade..a3365de 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -16,6 +16,7 @@ """Encapsulate Manila testing.""" +from tenacity import Retrying, stop_after_attempt, wait_exponential from manilaclient import client as manilaclient @@ -38,4 +39,7 @@ class ManilaTests(test_utils.OpenStackBaseTest): def test_manila_api(self): """Test that the Manila API is working.""" # now just try a list the shares - self.manila_client.shares.list() + for attempt in Retrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=10)): + self.manila_client.shares.list() From 0d47a4028adb37a98f81ac1b3ffd879dbd5bf8d7 Mon Sep 17 00:00:00 2001 From: Arif Ali Date: Mon, 23 Dec 2019 19:32:25 +0000 Subject: [PATCH 244/898] Add Neutron OpenvSwitch Functional Tests --- zaza/openstack/charm_tests/neutron/tests.py | 200 ++++++++++++++++++++ 1 file changed, 200 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 4df7021..de80d87 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -318,6 +318,206 @@ class SecurityTest(test_utils.OpenStackBaseTest): expected_to_pass=expected_to_pass) +class NeutronOpenvSwitchTest(test_utils.OpenStackBaseTest): + """Test basic Neutron Openvswitch Charm functionality.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Neutron Openvswitch tests.""" + super(NeutronOpenvSwitchTest, cls).setUpClass() + + cls.current_os_release = openstack_utils.get_os_release() + + cls.compute_unit = zaza.model.get_units('nova-compute')[0] + cls.neutron_api_unit = zaza.model.get_units('neutron-api')[0] + cls.n_ovs_unit = zaza.model.get_units('neutron-openvswitch')[0] + + cls.bionic_stein = openstack_utils.get_os_release('bionic_stein') + cls.trusty_mitaka = openstack_utils.get_os_release('trusty_mitaka') + + if cls.current_os_release >= cls.bionic_stein: + cls.pgrep_full = True + else: + cls.pgrep_full = False + + # set up client + cls.neutron_client = ( + openstack_utils.get_neutron_session_client(cls.keystone_session)) + + def test_101_neutron_sriov_config(self): + """Verify data in the sriov agent config file.""" + trusty_kilo = openstack_utils.get_os_release('trusty_kilo') + if self.current_os_release < trusty_kilo: + logging.debug('Skipping test, sriov agent not supported on < ' + 'trusty/kilo') + return + + zaza.model.set_application_config( + self.application_name, + {'enable-sriov': 'True'}) + + zaza.model.wait_for_application_states() + + self._check_settings_in_config( + self.application_name, + 'sriov-device-mappings', + 'physical_device_mappings', + ['', 'physnet42:eth42'], + 'sriov_nic', + '/etc/neutron/plugins/ml2/sriov_agent.ini') + + # the CI environment does not expose an actual SR-IOV NIC to the + # functional tests. consequently the neutron-sriov agent will not + # run, and the charm will update its status as such. this will prevent + # the success of pause/resume test. + # + # disable sriov after validation of config file is complete. + logging.info('Disabling SR-IOV after verifying config file data...') + + zaza.model.set_application_config( + self.application_name, + {'enable-sriov': 'False'}) + + logging.info('Waiting for config-changes to complete...') + zaza.model.wait_for_application_states() + + logging.debug('OK') + + def _check_settings_in_config(self, service, charm_key, + config_file_key, vpair, + section, conf_file): + + set_default = {charm_key: vpair[0]} + set_alternate = {charm_key: vpair[1]} + app_name = service + + expected = { + section: { + config_file_key: [vpair[1]], + }, + } + + with self.config_change(set_default, + set_alternate, + application_name=app_name): + zaza.model.block_until_oslo_config_entries_match( + self.application_name, + conf_file, + expected, + ) + logging.debug('OK') + + def test_201_l2pop_propagation(self): + """Verify that l2pop setting propagates to neutron-ovs.""" + self._check_settings_in_config( + 'neutron-api', + 'l2-population', + 'l2_population', + ['False', 'True'], + 'agent', + '/etc/neutron/plugins/ml2/openvswitch_agent.ini') + + def test_202_nettype_propagation(self): + """Verify that nettype setting propagates to neutron-ovs.""" + self._check_settings_in_config( + 'neutron-api', + 'overlay-network-type', + 'tunnel_types', + ['vxlan', 'gre'], + 'agent', + '/etc/neutron/plugins/ml2/openvswitch_agent.ini') + + def test_301_secgroup_propagation_local_override(self): + """Verify disable-security-groups overrides what neutron-api says.""" + if self.current_os_release >= self.trusty_mitaka: + conf_file = "/etc/neutron/plugins/ml2/openvswitch_agent.ini" + else: + conf_file = "/etc/neutron/plugins/ml2/ml2_conf.ini" + + zaza.model.set_application_config( + 'neutron-api', + {'neutron-security-groups': 'True'}) + zaza.model.set_application_config( + 'neutron-openvswitch', + {'disable-security-groups': 'True'}) + + zaza.model.wait_for_application_states() + + expected = { + 'securitygroup': { + 'enable_security_group': ['False'], + }, + } + + zaza.model.block_until_oslo_config_entries_match( + self.application_name, + conf_file, + expected, + ) + + logging.info('Restoring to default configuration...') + zaza.model.set_application_config( + 'neutron-openvswitch', + {'disable-security-groups': 'False'}) + zaza.model.set_application_config( + 'neutron-api', + {'neutron-security-groups': 'False'}) + + zaza.model.wait_for_application_states() + + def test_401_restart_on_config_change(self): + """Verify that the specified services are restarted. + + When the config is changed we need to make sure that the services are + restarted. + """ + self.restart_on_changed( + '/etc/neutron/neutron.conf', + {'debug': 'false'}, + {'debug': 'true'}, + {'DEFAULT': {'debug': ['False']}}, + {'DEFAULT': {'debug': ['True']}}, + ['neutron-openvswitch-agent'], + pgrep_full=self.pgrep_full) + + def test_501_enable_qos(self): + """Check qos settings set via neutron-api charm.""" + if self.current_os_release < self.trusty_mitaka: + logging.debug('Skipping test') + return + + set_default = {'enable-qos': 'false'} + set_alternate = {'enable-qos': 'true'} + app_name = 'neutron-api' + + conf_file = '/etc/neutron/plugins/ml2/openvswitch_agent.ini' + expected = { + 'agent': { + 'extensions': ['qos'], + }, + } + + with self.config_change(set_default, + set_alternate, + application_name=app_name): + zaza.model.block_until_oslo_config_entries_match( + self.application_name, + conf_file, + expected, + ) + logging.debug('OK') + + def test_901_pause_and_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + with self.pause_resume(['neutron-openvswitch-agent'], + pgrep_full=self.pgrep_full): + logging.info('Testing pause resume') + + class NeutronNetworkingTest(unittest.TestCase): """Ensure that openstack instances have valid networking.""" From 3b6a2de1cab8281f4e4f01420350acd07f53174b Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 17 Jan 2020 12:23:25 +0100 Subject: [PATCH 245/898] octavia: Conditionally use DVR specific flags The current test code for octavia requires DVR extensions to be present on the Neutron API server. We need to conditionally enable this based on enabled extensions to allow the test to operate on multiple deployment configurations. --- zaza/openstack/charm_tests/octavia/setup.py | 3 +++ zaza/openstack/charm_tests/octavia/tests.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/setup.py b/zaza/openstack/charm_tests/octavia/setup.py index 639c68d..4f93097 100644 --- a/zaza/openstack/charm_tests/octavia/setup.py +++ b/zaza/openstack/charm_tests/octavia/setup.py @@ -137,6 +137,9 @@ def centralized_fip_network(): 4: https://review.opendev.org/#/c/437986/ 5: https://review.opendev.org/#/c/466434/ """ + if not openstack.dvr_enabled(): + logging.info('DVR not enabled, skip.') + return keystone_session = openstack.get_overcloud_keystone_session() neutron_client = openstack.get_neutron_session_client( keystone_session) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 3c5ee71..c022b02 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -61,10 +61,13 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): payload_ips.append(server.networks['private'][0]) self.assertTrue(len(payload_ips) > 0) - resp = neutron_client.list_networks(name='private_lb_fip_network') - vip_subnet_id = resp['networks'][0]['subnets'][0] resp = neutron_client.list_networks(name='private') subnet_id = resp['networks'][0]['subnets'][0] + if openstack_utils.dvr_enabled(): + resp = neutron_client.list_networks(name='private_lb_fip_network') + vip_subnet_id = resp['networks'][0]['subnets'][0] + else: + vip_subnet_id = subnet_id octavia_client = openstack_utils.get_octavia_session_client( keystone_session) result = octavia_client.load_balancer_create( From b5e250513b15de7575efd444dcfa79f1f648ce07 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 21 Jan 2020 10:07:01 -0800 Subject: [PATCH 246/898] Fix for NDR testing race condition Previous attempts to fix LP Bug#1784083 added a workaround (commit 820ed808) which is being removed here. The root cause seems to be upstream in the dragent. It may never have been envisioned to run the agent by itself the way the charm does. So that even if neutron-api completes its amqp relation first, neutron-dynamic-routing can still see oslo_messaging.exceptions.MessagingTimeout errors. Some operation must occur against neutron before dragent is truly ready. i.e. some post deploy openstack command. So it is outside the purview of the charm. This change adds a service restart late. Partial-Bug: #1841459 --- .../utilities/test_zaza_utilities_generic.py | 25 +++++++++++++ .../charm_tests/dragent/configure.py | 11 ++++++ zaza/openstack/configure/bgp_speaker.py | 35 ------------------- zaza/openstack/utilities/generic.py | 26 ++++++++++++++ 4 files changed, 62 insertions(+), 35 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 3fb26da..d18b646 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -666,3 +666,28 @@ class TestGenericUtils(ut_utils.BaseTestCase): result = generic_utils.check_commands_on_units(cmds, _units) self.assertIsNotNone(result) + + def test_systemctl(self): + self.patch_object(generic_utils.model, "get_unit_from_name") + self.patch_object(generic_utils.model, "run_on_unit") + _unit = mock.MagicMock() + _unit.entity_id = "unit/2" + _command = "stop" + _service = "servicename" + _systemctl = "/bin/systemctl {} {}".format(_command, _service) + self.run_on_unit.return_value = {"Code": 0} + self.get_unit_from_name.return_value = _unit + + # With Unit object + generic_utils.systemctl(_unit, _service, command=_command) + self.run_on_unit.assert_called_with(_unit.entity_id, _systemctl) + + # With string name unit + generic_utils.systemctl(_unit.entity_id, _service, command=_command) + self.run_on_unit.assert_called_with(_unit.entity_id, _systemctl) + + # Failed return code + self.run_on_unit.return_value = {"Code": 1} + with self.assertRaises(AssertionError): + generic_utils.systemctl( + _unit.entity_id, _service, command=_command) diff --git a/zaza/openstack/charm_tests/dragent/configure.py b/zaza/openstack/charm_tests/dragent/configure.py index 6d6e075..f509609 100644 --- a/zaza/openstack/charm_tests/dragent/configure.py +++ b/zaza/openstack/charm_tests/dragent/configure.py @@ -16,6 +16,8 @@ """Setup for BGP deployments.""" +import logging +import zaza.model from zaza.openstack.configure import ( network, bgp_speaker, @@ -86,6 +88,15 @@ def setup(): # Confugre the overcloud network network.setup_sdn(network_config, keystone_session=keystone_session) + + # LP Bugs #1784083 and #1841459, require a late restart of the + # neutron-bgp-dragent service + logging.warning("Due to LP Bugs #1784083 and #1841459, we require a late " + "restart of the neutron-bgp-dragent service before " + "setting up BGP.") + for unit in zaza.model.get_units("neutron-dynamic-routing"): + generic_utils.systemctl(unit, "neutron-bgp-dragent", command="restart") + # Configure BGP bgp_speaker.setup_bgp_speaker( peer_application_name=DEFAULT_PEER_APPLICATION_NAME, diff --git a/zaza/openstack/configure/bgp_speaker.py b/zaza/openstack/configure/bgp_speaker.py index 2f5dbb9..4fd9587 100755 --- a/zaza/openstack/configure/bgp_speaker.py +++ b/zaza/openstack/configure/bgp_speaker.py @@ -19,7 +19,6 @@ import argparse import logging import sys -import neutronclient from zaza.openstack.utilities import ( cli as cli_utils, openstack as openstack_utils, @@ -99,40 +98,6 @@ def setup_bgp_speaker(peer_application_name, keystone_session=None): "Advertised floating IP: {}".format( floating_ip["floating_ip_address"])) - # NOTE(fnordahl): As a workaround for LP: #1784083 remove BGP speaker from - # dragent and add it back. - logging.info( - "Waiting for Neutron agent 'neutron-bgp-dragent' to appear...") - keystone_session = openstack_utils.get_overcloud_keystone_session() - neutron_client = openstack_utils.get_neutron_session_client( - keystone_session) - agents = openstack_utils.neutron_agent_appears(neutron_client, - 'neutron-bgp-dragent') - agent_id = None - for agent in agents.get('agents', []): - agent_id = agent.get('id', None) - if agent_id is not None: - break - logging.info( - 'Waiting for BGP speaker to appear on agent "{}"...'.format(agent_id)) - bgp_speakers = openstack_utils.neutron_bgp_speaker_appears_on_agent( - neutron_client, agent_id) - logging.info( - "Removing and adding back bgp-speakers to agent (LP: #1784083)...") - while True: - try: - for bgp_speaker in bgp_speakers.get('bgp_speakers', []): - bgp_speaker_id = bgp_speaker.get('id', None) - logging.info('removing "{}" from "{}"' - ''.format(bgp_speaker_id, agent_id)) - neutron_client.remove_bgp_speaker_from_dragent( - agent_id, bgp_speaker_id) - except neutronclient.common.exceptions.NotFound as e: - logging.info('Exception: "{}"'.format(e)) - break - neutron_client.add_bgp_speaker_to_dragent( - agent_id, {'bgp_speaker_id': bgp_speaker_id}) - def run_from_cli(): """Run BGP Speaker setup from CLI. diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index dbeb7d3..7b673d0 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -824,3 +824,29 @@ def get_series(unit): result = model.run_on_unit(unit.entity_id, "lsb_release -cs") return result['Stdout'].strip() + + +def systemctl(unit, service, command="restart"): + """Run systemctl command on a unit. + + :param unit: Unit object or unit name + :type unit: Union[Unit,string] + :param service: Name of service to act on + :type service: string + :param command: Name of command. i.e. start, stop, restart + :type command: string + :raises: AssertionError if the command is unsuccessful + :returns: None if successful + """ + cmd = "/bin/systemctl {} {}".format(command, service) + + # Check if this is a unit object or string name of a unit + try: + unit.entity_id + except AttributeError: + unit = model.get_unit_from_name(unit) + + result = model.run_on_unit( + unit.entity_id, cmd) + assert int(result['Code']) == 0, ( + "{} of {} on {} failed".format(command, service, unit.entity_id)) From ca8e1e9d5b519d9f324b348a094358fa17c23329 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 21 Jan 2020 14:23:13 -0600 Subject: [PATCH 247/898] import functional tests keystone-ldap and zaza-ify --- zaza/openstack/charm_tests/keystone/tests.py | 96 ++++++++++++++++++-- 1 file changed, 87 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 66727cc..1d3a459 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -145,11 +145,11 @@ class CharmOperationTest(BaseKeystoneTest): if unit_repo != lead_repo: raise zaza_exceptions.KeystoneKeyRepositoryError( 'expect: "{}" actual({}): "{}"' - .format(pprint.pformat(lead_repo), unit.entity_id, - pprint.pformat(unit_repo))) + ''.format(pprint.pformat(lead_repo), unit.entity_id, + pprint.pformat(unit_repo))) logging.info('"{}" == "{}"' - .format(pprint.pformat(unit_repo), - pprint.pformat(lead_repo))) + ''.format(pprint.pformat(unit_repo), + pprint.pformat(lead_repo))) class AuthenticationAuthorizationTest(BaseKeystoneTest): @@ -212,7 +212,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): def test_end_user_domain_admin_access(self): """Verify that end-user domain admin does not have elevated privileges. - In additon to validating that the `policy.json` is written and the + In addition to validating that the `policy.json` is written and the service is restarted on config-changed, the test validates that our `policy.json` is correct. @@ -257,8 +257,9 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): 'allowed when it should not be.') logging.info('OK') - def test_end_user_acccess_and_token(self): - """Verify regular end-user access resources and validate token data. + def test_end_user_access_and_token(self): + """ + Verify regular end-user access resources and validate token data. In effect this also validates user creation, presence of standard roles (`_member_`, `Member`), effect of policy and configuration @@ -279,12 +280,12 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): if len(token) != 32: raise zaza_exceptions.KeystoneWrongTokenProvider( 'We expected a UUID token and got this: "{}"' - .format(token)) + ''.format(token)) else: if len(token) < 180: raise zaza_exceptions.KeystoneWrongTokenProvider( 'We expected a Fernet token and got this: "{}"' - .format(token)) + ''.format(token)) logging.info('token: "{}"'.format(pprint.pformat(token))) if (openstack_utils.get_os_release() < @@ -371,3 +372,80 @@ class SecurityTests(BaseKeystoneTest): expected_passes, expected_failures, expected_to_pass=False) + + +class LdapTests(BaseKeystoneTest): + """Keystone ldap tests tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Keystone ldap-tests.""" + super(LdapTests, cls).setUpClass() + + def _get_ldap_config(self): + ldap_ips = zaza.model.get_app_ips("ldap-server") + ldap_server_ip = ldap_ips[0] + + keystone_ldap_config = { + 'ldap-server': "ldap://{}".format(ldap_server_ip), + 'ldap-user': 'cn=admin,dc=test,dc=com', + 'ldap-password': 'crapper', + 'ldap-suffix': 'dc=test,dc=com', + 'domain-name': 'userdomain', + } + if all(keystone_ldap_config.values()): + self.ldap_configured = True + return keystone_ldap_config + else: + # NOTE(jamespage): Use mock values to check deployment only + # as no test fixture has been supplied + self.ldap_configured = False + return { + 'ldap-server': 'myserver', + 'ldap-user': 'myuser', + 'ldap-password': 'mypassword', + 'ldap-suffix': 'mysuffix', + 'domain-name': 'userdomain', + } + + def _find_keystone_v3_user(self, username, domain): + """Find a user within a specified keystone v3 domain.""" + client = self.admin_keystone_client + domain_users = client.users.list( + domain=client.domains.find(name=domain).id + ) + usernames = [] + for user in domain_users: + usernames.append(user.name) + if username.lower() == user.name.lower(): + return user + logging.debug("User {} was not in these users: {}. Returning None." + "".format(username, usernames)) + return None + + def test_100_keystone_ldap_users(self): + """Validate basic functionality of keystone API with ldap.""" + zaza.model.set_application_config('keystone-ldap', + self._get_ldap_config()) + + if not self.ldap_configured: + msg = 'Skipping API tests as no LDAP test fixture' + logging.info(msg) + return + + logging.info('Waiting for ldap config-changes to complete...') + states = { + 'keystone-ldap': { + 'workload-status': 'idle', + 'workload-status-messages': 'Unit is ready' + } + } + + zaza.model.wait_for_application_states(states=states) + + # NOTE(jamespage): Test fixture should have johndoe and janedoe + # accounts + johndoe = self._find_keystone_v3_user('john doe', 'userdomain') + assert johndoe is not None + janedoe = self._find_keystone_v3_user('jane doe', 'userdomain') + assert janedoe is not None From 0c9e00610b38adfb4c40b0951e928b1ab89a7032 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 21 Jan 2020 14:43:43 -0600 Subject: [PATCH 248/898] wait for keystone to settle also --- zaza/openstack/charm_tests/keystone/tests.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 1d3a459..7388237 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -17,7 +17,7 @@ import collections import json import logging import pprint - +import tenacity import keystoneauth1 import zaza.model @@ -408,6 +408,10 @@ class LdapTests(BaseKeystoneTest): 'domain-name': 'userdomain', } + # NOTE: intermittent authentication fails. Wrap in a retry loop. + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, + min=5, max=10), + reraise=True) def _find_keystone_v3_user(self, username, domain): """Find a user within a specified keystone v3 domain.""" client = self.admin_keystone_client @@ -438,9 +442,12 @@ class LdapTests(BaseKeystoneTest): 'keystone-ldap': { 'workload-status': 'idle', 'workload-status-messages': 'Unit is ready' + }, + 'keystone': { + 'workload-status': 'idle', + 'workload-status-messages': 'Unit is ready' } } - zaza.model.wait_for_application_states(states=states) # NOTE(jamespage): Test fixture should have johndoe and janedoe From b4f1201da256b71bae015fb7410fd6baa8389362 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 22 Jan 2020 09:30:48 -0600 Subject: [PATCH 249/898] Integrate suggestions from ajkavanagh --- zaza/openstack/charm_tests/keystone/tests.py | 101 ++++++++----------- zaza/openstack/charm_tests/test_utils.py | 34 +++++-- 2 files changed, 67 insertions(+), 68 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 7388237..bf9d652 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -17,7 +17,7 @@ import collections import json import logging import pprint -import tenacity +import unittest import keystoneauth1 import zaza.model @@ -145,11 +145,11 @@ class CharmOperationTest(BaseKeystoneTest): if unit_repo != lead_repo: raise zaza_exceptions.KeystoneKeyRepositoryError( 'expect: "{}" actual({}): "{}"' - ''.format(pprint.pformat(lead_repo), unit.entity_id, - pprint.pformat(unit_repo))) + .format(pprint.pformat(lead_repo), unit.entity_id, + pprint.pformat(unit_repo))) logging.info('"{}" == "{}"' - ''.format(pprint.pformat(unit_repo), - pprint.pformat(lead_repo))) + .format(pprint.pformat(unit_repo), + pprint.pformat(lead_repo))) class AuthenticationAuthorizationTest(BaseKeystoneTest): @@ -258,8 +258,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): logging.info('OK') def test_end_user_access_and_token(self): - """ - Verify regular end-user access resources and validate token data. + """Verify regular end-user access resources and validate token data. In effect this also validates user creation, presence of standard roles (`_member_`, `Member`), effect of policy and configuration @@ -280,7 +279,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): if len(token) != 32: raise zaza_exceptions.KeystoneWrongTokenProvider( 'We expected a UUID token and got this: "{}"' - ''.format(token)) + .format(token)) else: if len(token) < 180: raise zaza_exceptions.KeystoneWrongTokenProvider( @@ -382,36 +381,19 @@ class LdapTests(BaseKeystoneTest): """Run class setup for running Keystone ldap-tests.""" super(LdapTests, cls).setUpClass() - def _get_ldap_config(self): + @staticmethod + def _get_ldap_config(): ldap_ips = zaza.model.get_app_ips("ldap-server") - ldap_server_ip = ldap_ips[0] - - keystone_ldap_config = { - 'ldap-server': "ldap://{}".format(ldap_server_ip), - 'ldap-user': 'cn=admin,dc=test,dc=com', - 'ldap-password': 'crapper', - 'ldap-suffix': 'dc=test,dc=com', - 'domain-name': 'userdomain', - } - if all(keystone_ldap_config.values()): - self.ldap_configured = True - return keystone_ldap_config - else: - # NOTE(jamespage): Use mock values to check deployment only - # as no test fixture has been supplied - self.ldap_configured = False - return { - 'ldap-server': 'myserver', - 'ldap-user': 'myuser', - 'ldap-password': 'mypassword', - 'ldap-suffix': 'mysuffix', + if ldap_ips: + return True, { + 'ldap-server': "ldap://{}".format(ldap_ips[0]), + 'ldap-user': 'cn=admin,dc=test,dc=com', + 'ldap-password': 'crapper', + 'ldap-suffix': 'dc=test,dc=com', 'domain-name': 'userdomain', } + return False, {} - # NOTE: intermittent authentication fails. Wrap in a retry loop. - @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, - min=5, max=10), - reraise=True) def _find_keystone_v3_user(self, username, domain): """Find a user within a specified keystone v3 domain.""" client = self.admin_keystone_client @@ -423,36 +405,35 @@ class LdapTests(BaseKeystoneTest): usernames.append(user.name) if username.lower() == user.name.lower(): return user - logging.debug("User {} was not in these users: {}. Returning None." - "".format(username, usernames)) + logging.debug( + "User {} was not in these users: {}. Returning None." + .format(username, usernames) + ) return None def test_100_keystone_ldap_users(self): """Validate basic functionality of keystone API with ldap.""" - zaza.model.set_application_config('keystone-ldap', - self._get_ldap_config()) + application_name = 'keystone-ldap' + can_config, config = self._get_ldap_config() + if not can_config: + raise unittest.SkipTest("Skipping API tests as no LDAP test fixture") - if not self.ldap_configured: - msg = 'Skipping API tests as no LDAP test fixture' - logging.info(msg) - return - - logging.info('Waiting for ldap config-changes to complete...') - states = { - 'keystone-ldap': { - 'workload-status': 'idle', - 'workload-status-messages': 'Unit is ready' - }, - 'keystone': { - 'workload-status': 'idle', - 'workload-status-messages': 'Unit is ready' + with self.config_change( + self.config_current(application_name), + config, + application_name=application_name): + logging.info('Waiting for users to become available in keystone...') + states = { + 'keystone': { + 'workload-status': 'idle', + 'workload-status-messages': 'Unit is ready' + } } - } - zaza.model.wait_for_application_states(states=states) + zaza.model.wait_for_application_states(states=states) - # NOTE(jamespage): Test fixture should have johndoe and janedoe - # accounts - johndoe = self._find_keystone_v3_user('john doe', 'userdomain') - assert johndoe is not None - janedoe = self._find_keystone_v3_user('jane doe', 'userdomain') - assert janedoe is not None + # NOTE(jamespage): Test fixture should have johndoe and janedoe + # accounts + johndoe = self._find_keystone_v3_user('john doe', 'userdomain') + self.assertIsNotNone(johndoe, "user 'john doe' was unknown") + janedoe = self._find_keystone_v3_user('jane doe', 'userdomain') + self.assertIsNotNone(janedoe, "user 'jane doe' was unknown") diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 82df994..237ad12 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -133,6 +133,29 @@ class OpenStackBaseTest(unittest.TestCase): model_name=cls.model_name) logging.debug('Leader unit is {}'.format(cls.lead_unit)) + def config_current(self, application_name=None, keys=None): + """Get Current Config of an application normalized into key-values + + :param application_name: String application name for use when called + by a charm under test other than the object's + application. + :type application_name: str + :param keys: iterable of strs to index into the current config. If + None, return all keys from the config + :type keys: Union[iterable[str], None] + """ + if not application_name: + application_name = self.application_name + _app_config = model.get_application_config(application_name) + # convert the more elaborate config structure from libjuju to key-value + keys = keys or _app_config.keys() + # note that conversion to string for all values is due to + # attempting to set any config with other types lead to Traceback + return { + str(k): str(_app_config.get(k, {}).get('value', '')) + for k in keys + } + @contextlib.contextmanager def config_change(self, default_config, alternate_config, application_name=None): @@ -158,17 +181,12 @@ class OpenStackBaseTest(unittest.TestCase): """ if not application_name: application_name = self.application_name + # we need to compare config values to what is already applied before # attempting to set them. otherwise the model will behave differently # than we would expect while waiting for completion of the change - _app_config = model.get_application_config(application_name) - app_config = {} - # convert the more elaborate config structure from libjuju to something - # we can compare to what the caller supplies to this function - for k in alternate_config.keys(): - # note that conversion to string for all values is due to - # attempting to set any config with other types lead to Traceback - app_config[k] = str(_app_config.get(k, {}).get('value', '')) + app_config = self.config_current(application_name, keys=alternate_config.keys()) + if all(item in app_config.items() for item in alternate_config.items()): logging.debug('alternate_config equals what is already applied ' From 59b06a956b251de960a963c5a0654b0b6be1d25d Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 22 Jan 2020 09:32:17 -0600 Subject: [PATCH 250/898] resolve another suggestion from ajkavanagh --- zaza/openstack/charm_tests/keystone/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index bf9d652..c692255 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -284,7 +284,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): if len(token) < 180: raise zaza_exceptions.KeystoneWrongTokenProvider( 'We expected a Fernet token and got this: "{}"' - ''.format(token)) + .format(token)) logging.info('token: "{}"'.format(pprint.pformat(token))) if (openstack_utils.get_os_release() < From b75aaa79c58e631a5850c0be5b8c961156ce2f1b Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 22 Jan 2020 09:33:58 -0600 Subject: [PATCH 251/898] Resolve pep8 violations --- zaza/openstack/charm_tests/keystone/tests.py | 8 ++++++-- zaza/openstack/charm_tests/test_utils.py | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index c692255..7f6e845 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -416,13 +416,17 @@ class LdapTests(BaseKeystoneTest): application_name = 'keystone-ldap' can_config, config = self._get_ldap_config() if not can_config: - raise unittest.SkipTest("Skipping API tests as no LDAP test fixture") + raise unittest.SkipTest( + "Skipping API tests as no LDAP test fixture" + ) with self.config_change( self.config_current(application_name), config, application_name=application_name): - logging.info('Waiting for users to become available in keystone...') + logging.info( + 'Waiting for users to become available in keystone...' + ) states = { 'keystone': { 'workload-status': 'idle', diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 237ad12..3b26414 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -134,7 +134,7 @@ class OpenStackBaseTest(unittest.TestCase): logging.debug('Leader unit is {}'.format(cls.lead_unit)) def config_current(self, application_name=None, keys=None): - """Get Current Config of an application normalized into key-values + """Get Current Config of an application normalized into key-values. :param application_name: String application name for use when called by a charm under test other than the object's @@ -185,7 +185,9 @@ class OpenStackBaseTest(unittest.TestCase): # we need to compare config values to what is already applied before # attempting to set them. otherwise the model will behave differently # than we would expect while waiting for completion of the change - app_config = self.config_current(application_name, keys=alternate_config.keys()) + app_config = self.config_current( + application_name, keys=alternate_config.keys() + ) if all(item in app_config.items() for item in alternate_config.items()): From e2940340185acf3433a0d32b4d0db22e08e793da Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 22 Jan 2020 13:14:12 -0600 Subject: [PATCH 252/898] improve docstrings for config_current --- zaza/openstack/charm_tests/test_utils.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 3b26414..6c44b0c 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Module containg base class for implementing charm tests.""" +"""Module containing base class for implementing charm tests.""" import contextlib import logging import subprocess @@ -70,7 +70,7 @@ def audit_assertions(action, :param expected_passes: List of test names that are expected to pass :type expected_passes: List[str] :param expected_failures: List of test names that are expected to fail - :type expexted_failures: List[str] + :type expected_failures: List[str] :raises: AssertionError if the assertion fails. """ if expected_failures is None: @@ -115,7 +115,7 @@ class OpenStackBaseTest(unittest.TestCase): @classmethod def setUpClass(cls, application_name=None, model_alias=None): - """Run setup for test class to create common resourcea.""" + """Run setup for test class to create common resources.""" cls.model_aliases = model.get_juju_model_aliases() if model_alias: cls.model_name = cls.model_aliases[model_alias] @@ -136,13 +136,15 @@ class OpenStackBaseTest(unittest.TestCase): def config_current(self, application_name=None, keys=None): """Get Current Config of an application normalized into key-values. - :param application_name: String application name for use when called + @param application_name: String application name for use when called by a charm under test other than the object's application. - :type application_name: str - :param keys: iterable of strs to index into the current config. If + @type application_name: Optional[str] + @param keys: iterable of strs to index into the current config. If None, return all keys from the config - :type keys: Union[iterable[str], None] + @type keys: Optional[Iterable[str]] + @return: Dictionary of requested config from application + @rtype: Dict[str, str] """ if not application_name: application_name = self.application_name From c76b7d423a4f3efff6e26e06db0dec179880ac3d Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 22 Jan 2020 13:20:42 -0600 Subject: [PATCH 253/898] improved docstrings for ldap tests --- zaza/openstack/charm_tests/keystone/tests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 7f6e845..3b13f8c 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -395,7 +395,12 @@ class LdapTests(BaseKeystoneTest): return False, {} def _find_keystone_v3_user(self, username, domain): - """Find a user within a specified keystone v3 domain.""" + """Find a user within a specified keystone v3 domain. + @param str username: Username to search for in keystone + @param str domain: username selected from which domain + @return: return username if found + @rtype: Optional[str] + """ client = self.admin_keystone_client domain_users = client.users.list( domain=client.domains.find(name=domain).id @@ -417,7 +422,7 @@ class LdapTests(BaseKeystoneTest): can_config, config = self._get_ldap_config() if not can_config: raise unittest.SkipTest( - "Skipping API tests as no LDAP test fixture" + "Skipping API tests because there's no ldap-server" ) with self.config_change( From 873b64cde3ab14def097dbd958890ea7fe5f3962 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 22 Jan 2020 13:25:17 -0600 Subject: [PATCH 254/898] improve docstring --- zaza/openstack/charm_tests/keystone/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 3b13f8c..21a4740 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -396,6 +396,7 @@ class LdapTests(BaseKeystoneTest): def _find_keystone_v3_user(self, username, domain): """Find a user within a specified keystone v3 domain. + @param str username: Username to search for in keystone @param str domain: username selected from which domain @return: return username if found From f065050d170bc26138096d1a8783bc51b0ff17fa Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 23 Jan 2020 10:52:50 -0600 Subject: [PATCH 255/898] use ':' style argument definitions rather than '@' --- zaza/openstack/charm_tests/keystone/tests.py | 14 ++++++++++---- zaza/openstack/charm_tests/test_utils.py | 12 ++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 21a4740..c402a31 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -383,6 +383,12 @@ class LdapTests(BaseKeystoneTest): @staticmethod def _get_ldap_config(): + """Generate ldap config for current model. + + :return: tuple of whether ldap-server is running and if so, config + for the keystone-ldap application. + :rtype: Tuple[bool, Dict[str,str]] + """ ldap_ips = zaza.model.get_app_ips("ldap-server") if ldap_ips: return True, { @@ -397,10 +403,10 @@ class LdapTests(BaseKeystoneTest): def _find_keystone_v3_user(self, username, domain): """Find a user within a specified keystone v3 domain. - @param str username: Username to search for in keystone - @param str domain: username selected from which domain - @return: return username if found - @rtype: Optional[str] + :param str username: Username to search for in keystone + :param str domain: username selected from which domain + :return: return username if found + :rtype: Optional[str] """ client = self.admin_keystone_client domain_users = client.users.list( diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 6c44b0c..48835f5 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -136,15 +136,15 @@ class OpenStackBaseTest(unittest.TestCase): def config_current(self, application_name=None, keys=None): """Get Current Config of an application normalized into key-values. - @param application_name: String application name for use when called + :param application_name: String application name for use when called by a charm under test other than the object's application. - @type application_name: Optional[str] - @param keys: iterable of strs to index into the current config. If + :type application_name: Optional[str] + :param keys: iterable of strs to index into the current config. If None, return all keys from the config - @type keys: Optional[Iterable[str]] - @return: Dictionary of requested config from application - @rtype: Dict[str, str] + :type keys: Optional[Iterable[str]] + :return: Dictionary of requested config from application + :rtype: Dict[str, str] """ if not application_name: application_name = self.application_name From 70630568444c6691f8dba355b4dd2e87ec347ee8 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 23 Jan 2020 10:52:50 -0600 Subject: [PATCH 256/898] use ':' style argument definitions rather than '@' --- zaza/openstack/charm_tests/keystone/tests.py | 41 ++++++++++---------- zaza/openstack/charm_tests/test_utils.py | 12 +++--- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 21a4740..9959b2b 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -17,7 +17,6 @@ import collections import json import logging import pprint -import unittest import keystoneauth1 import zaza.model @@ -381,26 +380,30 @@ class LdapTests(BaseKeystoneTest): """Run class setup for running Keystone ldap-tests.""" super(LdapTests, cls).setUpClass() - @staticmethod - def _get_ldap_config(): + def _get_ldap_config(self): + """Generate ldap config for current model. + + :return: tuple of whether ldap-server is running and if so, config + for the keystone-ldap application. + :rtype: Tuple[bool, Dict[str,str]] + """ ldap_ips = zaza.model.get_app_ips("ldap-server") - if ldap_ips: - return True, { - 'ldap-server': "ldap://{}".format(ldap_ips[0]), - 'ldap-user': 'cn=admin,dc=test,dc=com', - 'ldap-password': 'crapper', - 'ldap-suffix': 'dc=test,dc=com', - 'domain-name': 'userdomain', - } - return False, {} + self.assertTrue(ldap_ips, "Should be at least one ldap server") + return { + 'ldap-server': "ldap://{}".format(ldap_ips[0]), + 'ldap-user': 'cn=admin,dc=test,dc=com', + 'ldap-password': 'crapper', + 'ldap-suffix': 'dc=test,dc=com', + 'domain-name': 'userdomain', + } def _find_keystone_v3_user(self, username, domain): """Find a user within a specified keystone v3 domain. - @param str username: Username to search for in keystone - @param str domain: username selected from which domain - @return: return username if found - @rtype: Optional[str] + :param str username: Username to search for in keystone + :param str domain: username selected from which domain + :return: return username if found + :rtype: Optional[str] """ client = self.admin_keystone_client domain_users = client.users.list( @@ -420,11 +423,7 @@ class LdapTests(BaseKeystoneTest): def test_100_keystone_ldap_users(self): """Validate basic functionality of keystone API with ldap.""" application_name = 'keystone-ldap' - can_config, config = self._get_ldap_config() - if not can_config: - raise unittest.SkipTest( - "Skipping API tests because there's no ldap-server" - ) + config = self._get_ldap_config() with self.config_change( self.config_current(application_name), diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 6c44b0c..48835f5 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -136,15 +136,15 @@ class OpenStackBaseTest(unittest.TestCase): def config_current(self, application_name=None, keys=None): """Get Current Config of an application normalized into key-values. - @param application_name: String application name for use when called + :param application_name: String application name for use when called by a charm under test other than the object's application. - @type application_name: Optional[str] - @param keys: iterable of strs to index into the current config. If + :type application_name: Optional[str] + :param keys: iterable of strs to index into the current config. If None, return all keys from the config - @type keys: Optional[Iterable[str]] - @return: Dictionary of requested config from application - @rtype: Dict[str, str] + :type keys: Optional[Iterable[str]] + :return: Dictionary of requested config from application + :rtype: Dict[str, str] """ if not application_name: application_name = self.application_name From 0614ff5ad60db931fef4a0ee0b95565ff1d457b9 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 24 Jan 2020 12:59:58 -0600 Subject: [PATCH 257/898] integrate comments from @thedac to improve waiting for stable, and some documentation issues --- zaza/openstack/charm_tests/keystone/tests.py | 13 +++++-------- zaza/openstack/charm_tests/test_utils.py | 4 ++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 9959b2b..fd81736 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -23,7 +23,7 @@ import zaza.model import zaza.openstack.utilities.exceptions as zaza_exceptions import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils - +import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.openstack.charm_tests.test_utils as test_utils from zaza.openstack.charm_tests.keystone import ( BaseKeystoneTest, @@ -432,13 +432,10 @@ class LdapTests(BaseKeystoneTest): logging.info( 'Waiting for users to become available in keystone...' ) - states = { - 'keystone': { - 'workload-status': 'idle', - 'workload-status-messages': 'Unit is ready' - } - } - zaza.model.wait_for_application_states(states=states) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) # NOTE(jamespage): Test fixture should have johndoe and janedoe # accounts diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 48835f5..c787055 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -151,8 +151,8 @@ class OpenStackBaseTest(unittest.TestCase): _app_config = model.get_application_config(application_name) # convert the more elaborate config structure from libjuju to key-value keys = keys or _app_config.keys() - # note that conversion to string for all values is due to - # attempting to set any config with other types lead to Traceback + # note the conversion to str for all values is due to + # attempting to set any config with other type leads to Traceback return { str(k): str(_app_config.get(k, {}).get('value', '')) for k in keys From 7e2febbde9a842a620323a904bcf553072e99b2a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 27 Jan 2020 09:11:43 +0000 Subject: [PATCH 258/898] Skip QoS Tests in OVN deploys The QoS tests do not work with OVN deploys, so skip them for the time being. --- zaza/openstack/charm_tests/neutron/tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index de80d87..02450e5 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -191,6 +191,16 @@ class NeutronApiTest(test_utils.OpenStackBaseTest): openstack_utils.get_os_release('trusty_mitaka')): logging.info('running qos check') + dhcp_agents = self.neutron_client.list_agents( + binary='neutron-dhcp-agent')['agents'] + if not dhcp_agents: + ovn_agents = self.neutron_client.list_agents( + binary='ovn-controller')['agents'] + if ovn_agents: + raise unittest.SkipTest( + "QoS tests are currently not supported on OVN " + "deployments") + with self.config_change( {'enable-qos': 'False'}, {'enable-qos': 'True'}, From 012e08c07714b08ccc7a38675392fbcc269542d4 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 27 Jan 2020 09:32:06 +0000 Subject: [PATCH 259/898] Add setup method for swift GR to check regions Add setup method for swift GR to check region count. This ensures the environment is ready before running GR tests. --- zaza/openstack/charm_tests/swift/setup.py | 38 +++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 zaza/openstack/charm_tests/swift/setup.py diff --git a/zaza/openstack/charm_tests/swift/setup.py b/zaza/openstack/charm_tests/swift/setup.py new file mode 100644 index 0000000..06dfb63 --- /dev/null +++ b/zaza/openstack/charm_tests/swift/setup.py @@ -0,0 +1,38 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for configuring octavia.""" + +import logging +import tenacity + +import zaza.openstack.utilities.openstack as openstack + + +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=10, max=300), + reraise=True, stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(AssertionError)) +def wait_for_region2(): + """Ensure two regions are present.""" + keystone_session = openstack.get_overcloud_keystone_session() + keystone_client = ( + openstack.get_keystone_session_client( + keystone_session, + client_api_version='3')) + swift_svc_id = keystone_client.services.find(name='swift').id + regions = set([ep.region + for ep in keystone_client.endpoints.list(swift_svc_id)]) + logging.info('Checking there are 2 regions. Current count is {}'.format( + len(regions))) + assert len(set(regions)) == 2, "Incorrect number of regions" From 4b9773bb5ecd94120083df1c86fd37e0d31d03d3 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 27 Jan 2020 10:10:43 +0000 Subject: [PATCH 260/898] Fix copy/pasta error --- zaza/openstack/charm_tests/swift/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/swift/setup.py b/zaza/openstack/charm_tests/swift/setup.py index 06dfb63..eb98806 100644 --- a/zaza/openstack/charm_tests/swift/setup.py +++ b/zaza/openstack/charm_tests/swift/setup.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Code for configuring octavia.""" +"""Code for configuring swift.""" import logging import tenacity From 827bb4bf1e4e23e1ada532e43bcfa65c85e1a344 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 27 Jan 2020 12:09:09 +0000 Subject: [PATCH 261/898] Retry get_ceph_pool_sample It sometime takes a short time for removal to be reflected in get_ceph_pool_sample so wrap check in tenacity decorator to retry. --- .../charm_tests/cinder_backup/tests.py | 54 +++++++++++-------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 2fa12db..087bbac 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -15,8 +15,11 @@ # limitations under the License. """Encapsulate cinder-backup testing.""" +import copy import logging +import tenacity + import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.ceph as ceph_utils @@ -169,26 +172,35 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): stop_after_attempt=15, msg="Volume") + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=10, max=300), + reraise=True, stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(AssertionError)) + def _check_get_ceph_pool_sample(obj_count_samples, pool_size_samples): + pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample( + unit_name, cinder_ceph_pool, self.model_name) + + _obj_count_samples = copy.deepcopy(obj_count_samples) + _pool_size_samples = copy.deepcopy(pool_size_samples) + _obj_count_samples.append(obj_count) + _pool_size_samples.append(kb_used) + # Validate ceph cinder pool object count samples over time + original, created, deleted = range(3) + self.assertFalse(_obj_count_samples[created] <= + _obj_count_samples[original]) + self.assertFalse(_obj_count_samples[deleted] >= + _obj_count_samples[created]) + + # Luminous (pike) ceph seems more efficient at disk usage so we + # cannot guarantee the ordering of kb_used + if (openstack_utils.get_os_release() < + openstack_utils.get_os_release('xenial_mitaka')): + self.assertFalse(_pool_size_samples[created] <= + _pool_size_samples[original]) + self.assertFalse(_pool_size_samples[deleted] >= + _pool_size_samples[created]) + # Final check, ceph cinder pool object count and disk usage logging.info('Checking ceph cinder pool after volume delete...') - pool_name, obj_count, kb_used = ceph_utils.get_ceph_pool_sample( - unit_name, cinder_ceph_pool, self.model_name) - - obj_count_samples.append(obj_count) - pool_size_samples.append(kb_used) - - # Validate ceph cinder pool object count samples over time - original, created, deleted = range(3) - self.assertFalse(obj_count_samples[created] <= - obj_count_samples[original]) - self.assertFalse(obj_count_samples[deleted] >= - obj_count_samples[created]) - - # Luminous (pike) ceph seems more efficient at disk usage so we cannot - # grantee the ordering of kb_used - if (openstack_utils.get_os_release() < - openstack_utils.get_os_release('xenial_mitaka')): - self.assertFalse(pool_size_samples[created] <= - pool_size_samples[original]) - self.assertFalse(pool_size_samples[deleted] >= - pool_size_samples[created]) + # It sometime takes a short time for removal to be reflected in + # get_ceph_pool_sample so wrap check in tenacity decorator to retry. + _check_get_ceph_pool_sample(obj_count_samples, pool_size_samples) From b080f5ff59deae035f747375d00a54bde3ee60a2 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Mon, 27 Jan 2020 11:46:23 -0600 Subject: [PATCH 262/898] config_current doesn't need to flatten values to , but applying config to zaza.model does required all values are strings --- zaza/openstack/charm_tests/keystone/tests.py | 2 +- zaza/openstack/charm_tests/test_utils.py | 31 ++++++++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index fd81736..9069775 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -426,7 +426,7 @@ class LdapTests(BaseKeystoneTest): config = self._get_ldap_config() with self.config_change( - self.config_current(application_name), + self.config_current(application_name, config.keys()), config, application_name=application_name): logging.info( diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index c787055..b81e95a 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -16,7 +16,6 @@ import contextlib import logging import subprocess import unittest -import zaza.model import zaza.model as model import zaza.charm_lifecycle.utils as lifecycle_utils @@ -28,7 +27,7 @@ def skipIfNotHA(service_name): """Run decorator to skip tests if application not in HA configuration.""" def _skipIfNotHA_inner_1(f): def _skipIfNotHA_inner_2(*args, **kwargs): - ips = zaza.model.get_app_ips( + ips = model.get_app_ips( service_name) if len(ips) > 1: return f(*args, **kwargs) @@ -144,20 +143,34 @@ class OpenStackBaseTest(unittest.TestCase): None, return all keys from the config :type keys: Optional[Iterable[str]] :return: Dictionary of requested config from application - :rtype: Dict[str, str] + :rtype: Dict[str, Any] """ if not application_name: application_name = self.application_name + _app_config = model.get_application_config(application_name) - # convert the more elaborate config structure from libjuju to key-value + keys = keys or _app_config.keys() - # note the conversion to str for all values is due to - # attempting to set any config with other type leads to Traceback return { - str(k): str(_app_config.get(k, {}).get('value', '')) + str(k): _app_config.get(k, {}).get('value', '') for k in keys } + @staticmethod + def _stringed_value_config(config): + """ + Workaround: + + libjuju refuses to accept data with types other than strings + through the zuzu.model.set_application_config + + :param config: Config dictionary with any typed values + :type config: Dict[str,Any] + :return: Config Dictionary with string-ly typed values + :rtype: Dict[str,str] + """ + return {k: str(v) for k, v in config.items()} + @contextlib.contextmanager def config_change(self, default_config, alternate_config, application_name=None): @@ -207,7 +220,7 @@ class OpenStackBaseTest(unittest.TestCase): .format(alternate_config)) model.set_application_config( application_name, - alternate_config, + self._stringed_value_config(alternate_config), model_name=self.model_name) logging.debug( @@ -227,7 +240,7 @@ class OpenStackBaseTest(unittest.TestCase): logging.debug('Restoring charm setting to {}'.format(default_config)) model.set_application_config( application_name, - default_config, + self._stringed_value_config(default_config), model_name=self.model_name) logging.debug( From 056b2552fd999b04d1c246db670b0d82643ddaf9 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Mon, 27 Jan 2020 11:48:58 -0600 Subject: [PATCH 263/898] resolve pep8 issues --- zaza/openstack/charm_tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index b81e95a..1f68c68 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -158,9 +158,9 @@ class OpenStackBaseTest(unittest.TestCase): @staticmethod def _stringed_value_config(config): - """ - Workaround: + """Stringify values in a dict. + Workaround: libjuju refuses to accept data with types other than strings through the zuzu.model.set_application_config From abfa96042fc845db57d9c087dede737e7766c48a Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Mon, 27 Jan 2020 12:30:08 -0600 Subject: [PATCH 264/898] also don't cast config keys to str --- zaza/openstack/charm_tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 1f68c68..eef3b77 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -152,7 +152,7 @@ class OpenStackBaseTest(unittest.TestCase): keys = keys or _app_config.keys() return { - str(k): _app_config.get(k, {}).get('value', '') + k: _app_config.get(k, {}).get('value', '') for k in keys } From b7890be342b3204300127a5d5c9fcf00b836bb97 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Mon, 27 Jan 2020 13:15:59 -0600 Subject: [PATCH 265/898] _stringified config value of --> '' --- zaza/openstack/charm_tests/test_utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index eef3b77..5903119 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -152,7 +152,7 @@ class OpenStackBaseTest(unittest.TestCase): keys = keys or _app_config.keys() return { - k: _app_config.get(k, {}).get('value', '') + k: _app_config.get(k, {}).get('value') for k in keys } @@ -169,7 +169,12 @@ class OpenStackBaseTest(unittest.TestCase): :return: Config Dictionary with string-ly typed values :rtype: Dict[str,str] """ - return {k: str(v) for k, v in config.items()} + # if v is None, stringify to '' + # otherwise use a strict cast with str(...) + return { + k: '' if v is None else str(v) + for k, v in config.items() + } @contextlib.contextmanager def config_change(self, default_config, alternate_config, From 416c85b08ff8e0ec690a0df2b9ae466bf5508d56 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 28 Jan 2020 14:03:06 -0600 Subject: [PATCH 266/898] attempt to force keystone_v3 for ldap tests --- zaza/openstack/charm_tests/keystone/tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 9069775..12a8313 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -405,7 +405,10 @@ class LdapTests(BaseKeystoneTest): :return: return username if found :rtype: Optional[str] """ - client = self.admin_keystone_client + client = openstack_utils.get_keystone_session_client( + self.admin_keystone_session, + client_api_version=3) + domain_users = client.users.list( domain=client.domains.find(name=domain).id ) From 51f7f2cfa76e6ad18c0cf289751b8fcffd91f618 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 29 Jan 2020 09:17:28 -0600 Subject: [PATCH 267/898] attempt to force ldap tests to use keystone v3 api --- .../charm_tests/keystone/__init__.py | 10 ++++ zaza/openstack/charm_tests/keystone/tests.py | 55 ++++++++++--------- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/__init__.py b/zaza/openstack/charm_tests/keystone/__init__.py index e244e16..432eac2 100644 --- a/zaza/openstack/charm_tests/keystone/__init__.py +++ b/zaza/openstack/charm_tests/keystone/__init__.py @@ -13,6 +13,7 @@ # limitations under the License. """Collection of code for setting up and testing keystone.""" +import contextlib import zaza import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -59,3 +60,12 @@ class BaseKeystoneTest(test_utils.OpenStackBaseTest): openstack_utils.get_keystone_session_client( cls.admin_keystone_session, client_api_version=cls.default_api_version)) + + @contextlib.contextmanager + def v3_keystone_preferred(self): + """Set the preferred keystone api to v3 within called context.""" + with self.config_change( + {'preferred-api-version': self.default_api_version}, + {'preferred-api-version': '3'}, + application_name="keystone"): + yield diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 12a8313..fec4c82 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -188,10 +188,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): openstack_utils.get_os_release('trusty_mitaka')): logging.info('skipping test < trusty_mitaka') return - with self.config_change( - {'preferred-api-version': self.default_api_version}, - {'preferred-api-version': '3'}, - application_name="keystone"): + with self.v3_keystone_preferred(): for ip in self.keystone_ips: try: logging.info('keystone IP {}'.format(ip)) @@ -221,10 +218,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): openstack_utils.get_os_release('xenial_ocata')): logging.info('skipping test < xenial_ocata') return - with self.config_change( - {'preferred-api-version': self.default_api_version}, - {'preferred-api-version': '3'}, - application_name="keystone"): + with self.v3_keystone_preferred(): for ip in self.keystone_ips: openrc = { 'API_VERSION': 3, @@ -405,18 +399,28 @@ class LdapTests(BaseKeystoneTest): :return: return username if found :rtype: Optional[str] """ - client = openstack_utils.get_keystone_session_client( - self.admin_keystone_session, - client_api_version=3) + for ip in self.keystone_ips: + try: + logging.info('keystone IP {}'.format(ip)) + ks_session = openstack_utils.get_keystone_session( + openstack_utils.get_overcloud_auth(address=ip)) + ks_client = openstack_utils.get_keystone_session_client( + ks_session) + + domain_users = ks_client.users.list( + domain=ks_client.domains.find(name=domain).id + ) + + usernames = [] + for user in domain_users: + usernames.append(user.name) + if username.lower() == user.name.lower(): + return user + except keystoneauth1.exceptions.http.HTTPError as e: + raise zaza_exceptions.KeystoneAuthorizationStrict( + 'Retrieve domain list as admin FAILED. ({})'.format(e) + ) - domain_users = client.users.list( - domain=client.domains.find(name=domain).id - ) - usernames = [] - for user in domain_users: - usernames.append(user.name) - if username.lower() == user.name.lower(): - return user logging.debug( "User {} was not in these users: {}. Returning None." .format(username, usernames) @@ -440,9 +444,10 @@ class LdapTests(BaseKeystoneTest): states=test_config.get("target_deploy_status", {}) ) - # NOTE(jamespage): Test fixture should have johndoe and janedoe - # accounts - johndoe = self._find_keystone_v3_user('john doe', 'userdomain') - self.assertIsNotNone(johndoe, "user 'john doe' was unknown") - janedoe = self._find_keystone_v3_user('jane doe', 'userdomain') - self.assertIsNotNone(janedoe, "user 'jane doe' was unknown") + with self.v3_keystone_preferred(): + # NOTE(jamespage): Test fixture should have johndoe and janedoe + # accounts + johndoe = self._find_keystone_v3_user('john doe', 'userdomain') + self.assertIsNotNone(johndoe, "user 'john doe' was unknown") + janedoe = self._find_keystone_v3_user('jane doe', 'userdomain') + self.assertIsNotNone(janedoe, "user 'jane doe' was unknown") From 7f82aa3a80ab41bfc82697388959946e1510e03e Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 29 Jan 2020 10:30:27 -0600 Subject: [PATCH 268/898] simplify looking for user from keystone --- zaza/openstack/charm_tests/keystone/tests.py | 31 +++++++------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index fec4c82..eda5441 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -400,30 +400,21 @@ class LdapTests(BaseKeystoneTest): :rtype: Optional[str] """ for ip in self.keystone_ips: - try: - logging.info('keystone IP {}'.format(ip)) - ks_session = openstack_utils.get_keystone_session( - openstack_utils.get_overcloud_auth(address=ip)) - ks_client = openstack_utils.get_keystone_session_client( - ks_session) + logging.info('Keystone IP {}'.format(ip)) + session = openstack_utils.get_keystone_session( + openstack_utils.get_overcloud_auth(address=ip)) + client = openstack_utils.get_keystone_session_client(session) - domain_users = ks_client.users.list( - domain=ks_client.domains.find(name=domain).id - ) + domain_users = client.users.list( + domain=client.domains.find(name=domain).id + ) - usernames = [] - for user in domain_users: - usernames.append(user.name) - if username.lower() == user.name.lower(): - return user - except keystoneauth1.exceptions.http.HTTPError as e: - raise zaza_exceptions.KeystoneAuthorizationStrict( - 'Retrieve domain list as admin FAILED. ({})'.format(e) - ) + usernames = [u.name.lower() for u in domain_users] + if username.lower() in usernames: + return username logging.debug( - "User {} was not in these users: {}. Returning None." - .format(username, usernames) + "User {} was not found. Returning None.".format(username) ) return None From fa1d77c15a7af6b4d678b1d19f7a85f754eb6ff9 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 30 Jan 2020 12:39:52 +0000 Subject: [PATCH 269/898] Fix hacluster pause/resume tests The hacluster tests used to pause the hacluster subordinate but then rely on the unintended side-effect of the prinicple going into a blocked state to confirm the action was complete. This is not what the old amulet tests used to do. Also, more importantly, the side-effect has been fixed so the tests no longer worrk. --- zaza/openstack/charm_tests/hacluster/tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 7ac67dc..fb9d8e2 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -86,11 +86,13 @@ class HaclusterTest(test_utils.OpenStackBaseTest): for subordinate in primary_status["units"][leader]["subordinates"]: logging.info("Pausing {}".format(subordinate)) zaza.model.run_action(subordinate, "pause") - zaza.model.block_until_unit_wl_status(leader, "blocked") + zaza.model.block_until_unit_wl_status( + subordinate, + "maintenance") logging.info("Resuming {}".format(subordinate)) zaza.model.run_action(subordinate, "resume") - zaza.model.block_until_unit_wl_status(leader, "active") + zaza.model.block_until_unit_wl_status(subordinate, "active") _states = {"hacluster": { "workload-status": "active", From 4ba9951510bb75d04f568ecabd0d0bfe397c05ac Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 30 Jan 2020 13:32:21 +0000 Subject: [PATCH 270/898] No ceilometer-collector restart check on trusty Due to Bug #1861321 ceilometer-collector does not reliably restart so do not check it until it is fixed. --- .../openstack/charm_tests/ceilometer/tests.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index 23f4bbd..dcd9147 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -16,6 +16,7 @@ """Encapsulate Ceilometer testing.""" +import copy import logging import zaza.openstack.utilities.openstack as openstack_utils @@ -46,20 +47,20 @@ class CeilometerTest(test_utils.OpenStackBaseTest): bug #1846390. https://bugs.launchpad.net/charms/+source/ceilometer/+bug/1846390 """ - current_release = openstack_utils.get_os_release() + self.current_release = openstack_utils.get_os_release() services = [] - if current_release >= CeilometerTest.XENIAL_PIKE: + if self.current_release >= CeilometerTest.XENIAL_PIKE: # services.append('ceilometer-polling: AgentManager worker(0)') services.append('ceilometer-agent-notification: ' 'NotificationService worker(0)') - elif current_release >= CeilometerTest.XENIAL_OCATA: + elif self.current_release >= CeilometerTest.XENIAL_OCATA: services.append('ceilometer-collector: CollectorService worker(0)') # services.append('ceilometer-polling: AgentManager worker(0)') services.append('ceilometer-agent-notification: ' 'NotificationService worker(0)') services.append('apache2') - elif current_release >= CeilometerTest.XENIAL_NEWTON: + elif self.current_release >= CeilometerTest.XENIAL_NEWTON: services.append('ceilometer-collector - CollectorService(0)') # services.append('ceilometer-polling - AgentManager(0)') services.append('ceilometer-agent-notification - ' @@ -70,11 +71,11 @@ class CeilometerTest(test_utils.OpenStackBaseTest): services.append('ceilometer-api') services.append('ceilometer-agent-notification') - if current_release < CeilometerTest.TRUSTY_MITAKA: + if self.current_release < CeilometerTest.TRUSTY_MITAKA: services.append('ceilometer-alarm-notifier') services.append('ceilometer-alarm-evaluator') - # if current_release >= CeilometerTest.TRUSTY_LIBERTY: + # if self.current_release >= CeilometerTest.TRUSTY_LIBERTY: # Liberty and later # services.append('ceilometer-polling') # else: @@ -87,6 +88,15 @@ class CeilometerTest(test_utils.OpenStackBaseTest): def test_900_restart_on_config_change(self): """Checking restart happens on config change.""" + _services = copy.deepcopy(self.services) + + # Due to Bug #1861321 ceilometer-collector does not reliably + # restart. + if self.current_release <= CeilometerTest.TRUSTY_MITAKA: + try: + _services.remove('ceilometer-collector') + except ValueError: + pass # Expected default and alternate values current_value = openstack_utils.get_application_config_option( 'ceilometer', 'debug' @@ -110,7 +120,7 @@ class CeilometerTest(test_utils.OpenStackBaseTest): set_alternate, default_entry, alternate_entry, - self.services) + _services) def test_901_pause_resume(self): """Run pause and resume tests. From 38c3ad77e0435ae317a3a241e9a59030785e9514 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 30 Jan 2020 10:19:03 -0500 Subject: [PATCH 271/898] Update CeilometerTest to test ceilometer-agent CeilometerTest is modified so that it can be used for both the charm-ceilometer and charm-ceilometer-agent functional tests. --- .../openstack/charm_tests/ceilometer/tests.py | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index dcd9147..56e3703 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -31,6 +31,7 @@ class CeilometerTest(test_utils.OpenStackBaseTest): XENIAL_PIKE = openstack_utils.get_os_release('xenial_pike') XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') XENIAL_NEWTON = openstack_utils.get_os_release('xenial_newton') + XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') TRUSTY_MITAKA = openstack_utils.get_os_release('trusty_mitaka') TRUSTY_LIBERTY = openstack_utils.get_os_release('trusty_liberty') @@ -41,15 +42,19 @@ class CeilometerTest(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list services for Openstack Release. - - Note: disabling ceilometer-polling and ceilometer-agent-central due to - bug #1846390. - https://bugs.launchpad.net/charms/+source/ceilometer/+bug/1846390 - """ + """Return a list services for Openstack Release.""" self.current_release = openstack_utils.get_os_release() services = [] + if self.application_name == 'ceilometer-agent': + if self.current_release <= CeilometerTest.XENIAL_MITAKA: + services.append('ceilometer-polling') + else: + services.append('ceilometer-polling: AgentManager worker(0)') + return services + + # Note: disabling ceilometer-polling and ceilometer-agent-central due + # to bug 1846390: https://bugs.launchpad.net/bugs/1846390 if self.current_release >= CeilometerTest.XENIAL_PIKE: # services.append('ceilometer-polling: AgentManager worker(0)') services.append('ceilometer-agent-notification: ' @@ -97,9 +102,15 @@ class CeilometerTest(test_utils.OpenStackBaseTest): _services.remove('ceilometer-collector') except ValueError: pass + + config_name = 'debug' + + if self.application_name == 'ceilometer-agent': + config_name = 'use-internal-endpoints' + # Expected default and alternate values current_value = openstack_utils.get_application_config_option( - 'ceilometer', 'debug' + self.application_name, config_name ) assert type(current_value) == bool new_value = not current_value @@ -108,11 +119,16 @@ class CeilometerTest(test_utils.OpenStackBaseTest): current_value = str(current_value) new_value = str(new_value) - set_default = {'debug': current_value} - set_alternate = {'debug': new_value} + set_default = {config_name: current_value} + set_alternate = {config_name: new_value} + default_entry = {'DEFAULT': {'debug': [current_value]}} alternate_entry = {'DEFAULT': {'debug': [new_value]}} + if self.application_name == 'ceilometer-agent': + default_entry = None + alternate_entry = {'service_credentials': {'interface': ['internalURL']}} + logging.info('changing config: {}'.format(set_alternate)) self.restart_on_changed( CeilometerTest.CONF_FILE, From 1a8fd43268d5739e0c1394852b9289bc494a340d Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 31 Jan 2020 17:34:17 -0500 Subject: [PATCH 272/898] Fix pep8 failure --- zaza/openstack/charm_tests/ceilometer/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index 56e3703..2741191 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -127,7 +127,9 @@ class CeilometerTest(test_utils.OpenStackBaseTest): if self.application_name == 'ceilometer-agent': default_entry = None - alternate_entry = {'service_credentials': {'interface': ['internalURL']}} + alternate_entry = { + 'service_credentials': {'interface': ['internalURL']} + } logging.info('changing config: {}'.format(set_alternate)) self.restart_on_changed( From b79900405b2ece5d70f83437555ae44392ef9a35 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 2 Feb 2020 18:38:08 +0000 Subject: [PATCH 273/898] Easyrsa does not support series upgrade so skip it Easyrsa does not currently support series upgrade so skip it in the openstack tests until Bug #1850121 is resolved. --- zaza/openstack/charm_tests/series_upgrade/tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 107ac80..fa5257c 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -56,6 +56,9 @@ class SeriesUpgradeTest(unittest.TestCase): # Skip subordinates if applications[application]["subordinate-to"]: continue + if "easyrsa" in applications[application]["charm"]: + logging.warn("Skipping series upgrade of easyrsa Bug #1850121") + continue if "percona-cluster" in applications[application]["charm"]: origin = "source" pause_non_leader_primary = True From c2e585ae4cdc90162e0e0e9921731eb9ad7dddb8 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 3 Feb 2020 09:34:13 +0100 Subject: [PATCH 274/898] octavia: Move LB creation API calls to helper --- zaza/openstack/charm_tests/octavia/tests.py | 213 ++++++++++---------- 1 file changed, 112 insertions(+), 101 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index c022b02..c3786f4 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -47,6 +47,115 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): """Run class setup for running LBaaSv2 service tests.""" super(LBAASv2Test, cls).setUpClass() + @staticmethod + @tenacity.retry(wait=tenacity.wait_fixed(1), + reraise=True, stop=tenacity.stop_after_delay(900)) + def wait_for_lb_resource(octavia_show_func, resource_id, + operating_status=None): + """Wait for loadbalancer resource to reach expected status.""" + resp = octavia_show_func(resource_id) + logging.info(resp['provisioning_status']) + assert resp['provisioning_status'] == 'ACTIVE', ( + 'load balancer resource has not reached ' + 'expected provisioning status: {}' + .format(resp)) + if operating_status: + logging.info(resp['operating_status']) + assert resp['operating_status'] == operating_status, ( + 'load balancer resource has not reached ' + 'expected operating status: {}'.format(resp)) + + return resp + + def _create_lb_resources(self, octavia_client, provider, vip_subnet_id, + member_subnet_id, payload_ips): + result = octavia_client.load_balancer_create( + json={ + 'loadbalancer': { + 'description': 'Created by Zaza', + 'admin_state_up': True, + 'vip_subnet_id': vip_subnet_id, + 'name': 'zaza-lb-0', + }}) + lb = result['loadbalancer'] + lb_id = lb['id'] + + logging.info('Awaiting loadbalancer to reach provisioning_status ' + '"ACTIVE"') + resp = self.wait_for_lb_resource( + octavia_client.load_balancer_show, lb_id) + logging.info(resp) + + result = octavia_client.listener_create( + json={ + 'listener': { + 'loadbalancer_id': lb_id, + 'name': 'listener1', + 'protocol': 'HTTP', + 'protocol_port': 80 + }, + }) + listener_id = result['listener']['id'] + logging.info('Awaiting listener to reach provisioning_status ' + '"ACTIVE"') + resp = self.wait_for_lb_resource( + octavia_client.listener_show, listener_id) + logging.info(resp) + + result = octavia_client.pool_create( + json={ + 'pool': { + 'listener_id': listener_id, + 'name': 'pool1', + 'lb_algorithm': 'ROUND_ROBIN', + 'protocol': 'HTTP', + }, + }) + pool_id = result['pool']['id'] + logging.info('Awaiting pool to reach provisioning_status ' + '"ACTIVE"') + resp = self.wait_for_lb_resource(octavia_client.pool_show, pool_id) + logging.info(resp) + + result = octavia_client.health_monitor_create( + json={ + 'healthmonitor': { + 'pool_id': pool_id, + 'delay': 5, + 'max_retries': 4, + 'timeout': 10, + 'type': 'HTTP', + 'url_path': '/', + }, + }) + healthmonitor_id = result['healthmonitor']['id'] + logging.info('Awaiting healthmonitor to reach provisioning_status ' + '"ACTIVE"') + resp = self.wait_for_lb_resource(octavia_client.health_monitor_show, + healthmonitor_id) + logging.info(resp) + + for ip in payload_ips: + result = octavia_client.member_create( + pool_id=pool_id, + json={ + 'member': { + 'subnet_id': member_subnet_id, + 'address': ip, + 'protocol_port': 80, + }, + }) + member_id = result['member']['id'] + logging.info('Awaiting member to reach provisioning_status ' + '"ACTIVE"') + resp = self.wait_for_lb_resource( + lambda x: octavia_client.member_show( + pool_id=pool_id, member_id=x), + member_id, + operating_status='ONLINE') + logging.info(resp) + return lb + def test_create_loadbalancer(self): """Create load balancer.""" keystone_session = openstack_utils.get_overcloud_keystone_session() @@ -70,109 +179,11 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): vip_subnet_id = subnet_id octavia_client = openstack_utils.get_octavia_session_client( keystone_session) - result = octavia_client.load_balancer_create( - json={ - 'loadbalancer': { - 'description': 'Created by Zaza', - 'admin_state_up': True, - 'vip_subnet_id': vip_subnet_id, - 'name': 'zaza-lb-0', - }}) - lb_id = result['loadbalancer']['id'] - lb_vip_port_id = result['loadbalancer']['vip_port_id'] - - @tenacity.retry(wait=tenacity.wait_fixed(1), - reraise=True, stop=tenacity.stop_after_delay(900)) - def wait_for_lb_resource(octavia_show_func, resource_id, - operating_status=None): - resp = octavia_show_func(resource_id) - logging.info(resp['provisioning_status']) - assert resp['provisioning_status'] == 'ACTIVE', ( - 'load balancer resource has not reached ' - 'expected provisioning status: {}' - .format(resp)) - if operating_status: - logging.info(resp['operating_status']) - assert resp['operating_status'] == operating_status, ( - 'load balancer resource has not reached ' - 'expected operating status: {}'.format(resp)) - - return resp - logging.info('Awaiting loadbalancer to reach provisioning_status ' - '"ACTIVE"') - resp = wait_for_lb_resource(octavia_client.load_balancer_show, lb_id) - logging.info(resp) - - result = octavia_client.listener_create( - json={ - 'listener': { - 'loadbalancer_id': lb_id, - 'name': 'listener1', - 'protocol': 'HTTP', - 'protocol_port': 80 - }, - }) - listener_id = result['listener']['id'] - logging.info('Awaiting listener to reach provisioning_status ' - '"ACTIVE"') - resp = wait_for_lb_resource(octavia_client.listener_show, listener_id) - logging.info(resp) - - result = octavia_client.pool_create( - json={ - 'pool': { - 'listener_id': listener_id, - 'name': 'pool1', - 'lb_algorithm': 'ROUND_ROBIN', - 'protocol': 'HTTP', - }, - }) - pool_id = result['pool']['id'] - logging.info('Awaiting pool to reach provisioning_status ' - '"ACTIVE"') - resp = wait_for_lb_resource(octavia_client.pool_show, pool_id) - logging.info(resp) - - result = octavia_client.health_monitor_create( - json={ - 'healthmonitor': { - 'pool_id': pool_id, - 'delay': 5, - 'max_retries': 4, - 'timeout': 10, - 'type': 'HTTP', - 'url_path': '/', - }, - }) - healthmonitor_id = result['healthmonitor']['id'] - logging.info('Awaiting healthmonitor to reach provisioning_status ' - '"ACTIVE"') - resp = wait_for_lb_resource(octavia_client.health_monitor_show, - healthmonitor_id) - logging.info(resp) - - for ip in payload_ips: - result = octavia_client.member_create( - pool_id=pool_id, - json={ - 'member': { - 'subnet_id': subnet_id, - 'address': ip, - 'protocol_port': 80, - }, - }) - member_id = result['member']['id'] - logging.info('Awaiting member to reach provisioning_status ' - '"ACTIVE"') - resp = wait_for_lb_resource( - lambda x: octavia_client.member_show( - pool_id=pool_id, member_id=x), - member_id, - operating_status='ONLINE') - logging.info(resp) + lb = self._create_lb_resources(octavia_client, 'amphora', + vip_subnet_id, subnet_id, payload_ips) lb_fp = openstack_utils.create_floating_ip( - neutron_client, 'ext_net', port={'id': lb_vip_port_id}) + neutron_client, 'ext_net', port={'id': lb['vip_port_id']}) @tenacity.retry(wait=tenacity.wait_fixed(1), reraise=True, stop=tenacity.stop_after_delay(900)) From 998a61a6a510ca26eb2d2e892b19657843253ea6 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 3 Feb 2020 10:03:49 +0100 Subject: [PATCH 275/898] octavia: Support testing multiple providers --- zaza/openstack/charm_tests/octavia/tests.py | 113 +++++++++++++------- 1 file changed, 75 insertions(+), 38 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index c3786f4..bd23cc9 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -67,15 +67,43 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): return resp + @staticmethod + def get_lb_providers(octavia_client): + """Retrieve loadbalancer providers. + + :param octavia_client: Octavia client object + :type octavia_client: OctaviaAPI + :returns: Dictionary with provider information, name as keys + :rtype: Dict[str,Dict[str,str]] + """ + providers = { + provider['name']: provider + for provider in octavia_client.provider_list().get('providers', []) + if provider['name'] != 'octavia' # alias for `amphora`, skip + } + return providers + def _create_lb_resources(self, octavia_client, provider, vip_subnet_id, member_subnet_id, payload_ips): + # The `amphora` provider is required for load balancing based on + # higher layer protocols + if provider == 'amphora': + protocol = 'HTTP' + algorithm = 'ROUND_ROBIN' + monitor = True + else: + protocol = 'TCP' + algorithm = 'SOURCE_IP_PORT' + monitor = False + result = octavia_client.load_balancer_create( json={ 'loadbalancer': { 'description': 'Created by Zaza', 'admin_state_up': True, 'vip_subnet_id': vip_subnet_id, - 'name': 'zaza-lb-0', + 'name': 'zaza-{}-0'.format(provider), + 'provider': provider, }}) lb = result['loadbalancer'] lb_id = lb['id'] @@ -91,7 +119,7 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): 'listener': { 'loadbalancer_id': lb_id, 'name': 'listener1', - 'protocol': 'HTTP', + 'protocol': protocol, 'protocol_port': 80 }, }) @@ -107,8 +135,8 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): 'pool': { 'listener_id': listener_id, 'name': 'pool1', - 'lb_algorithm': 'ROUND_ROBIN', - 'protocol': 'HTTP', + 'lb_algorithm': algorithm, + 'protocol': protocol, }, }) pool_id = result['pool']['id'] @@ -117,23 +145,25 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): resp = self.wait_for_lb_resource(octavia_client.pool_show, pool_id) logging.info(resp) - result = octavia_client.health_monitor_create( - json={ - 'healthmonitor': { - 'pool_id': pool_id, - 'delay': 5, - 'max_retries': 4, - 'timeout': 10, - 'type': 'HTTP', - 'url_path': '/', - }, - }) - healthmonitor_id = result['healthmonitor']['id'] - logging.info('Awaiting healthmonitor to reach provisioning_status ' - '"ACTIVE"') - resp = self.wait_for_lb_resource(octavia_client.health_monitor_show, - healthmonitor_id) - logging.info(resp) + if monitor: + result = octavia_client.health_monitor_create( + json={ + 'healthmonitor': { + 'pool_id': pool_id, + 'delay': 5, + 'max_retries': 4, + 'timeout': 10, + 'type': 'HTTP', + 'url_path': '/', + }, + }) + healthmonitor_id = result['healthmonitor']['id'] + logging.info('Awaiting healthmonitor to reach provisioning_status ' + '"ACTIVE"') + resp = self.wait_for_lb_resource( + octavia_client.health_monitor_show, + healthmonitor_id) + logging.info(resp) for ip in payload_ips: result = octavia_client.member_create( @@ -152,10 +182,19 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): lambda x: octavia_client.member_show( pool_id=pool_id, member_id=x), member_id, - operating_status='ONLINE') + operating_status='ONLINE' if monitor else '') logging.info(resp) return lb + @staticmethod + @tenacity.retry(wait=tenacity.wait_fixed(1), + reraise=True, stop=tenacity.stop_after_delay(900)) + def _get_payload(ip): + return subprocess.check_output( + ['wget', '-O', '-', + 'http://{}/'.format(ip)], + universal_newlines=True) + def test_create_loadbalancer(self): """Create load balancer.""" keystone_session = openstack_utils.get_overcloud_keystone_session() @@ -179,21 +218,19 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): vip_subnet_id = subnet_id octavia_client = openstack_utils.get_octavia_session_client( keystone_session) - lb = self._create_lb_resources(octavia_client, 'amphora', - vip_subnet_id, subnet_id, payload_ips) + for provider in self.get_lb_providers(octavia_client).keys(): + logging.info('Creating loadbalancer with provider {}' + .format(provider)) + lb = self._create_lb_resources(octavia_client, provider, + vip_subnet_id, subnet_id, + payload_ips) - lb_fp = openstack_utils.create_floating_ip( - neutron_client, 'ext_net', port={'id': lb['vip_port_id']}) + lb_fp = openstack_utils.create_floating_ip( + neutron_client, 'ext_net', port={'id': lb['vip_port_id']}) - @tenacity.retry(wait=tenacity.wait_fixed(1), - reraise=True, stop=tenacity.stop_after_delay(900)) - def get_payload(): - return subprocess.check_output( - ['wget', '-O', '-', - 'http://{}/'.format(lb_fp['floating_ip_address'])], - universal_newlines=True) - snippet = 'This is the default welcome page' - assert snippet in get_payload() - logging.info('Found "{}" in page retrieved through load balancer at ' - '"http://{}/"' - .format(snippet, lb_fp['floating_ip_address'])) + snippet = 'This is the default welcome page' + assert snippet in self._get_payload(lb_fp['floating_ip_address']) + logging.info('Found "{}" in page retrieved through load balancer ' + ' (provider="{}") at "http://{}/"' + .format(snippet, provider, + lb_fp['floating_ip_address'])) From 83982a7aec3da7813d0e4c8897d2ed4342d5f7cf Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 3 Feb 2020 13:39:17 +0000 Subject: [PATCH 276/898] Add unseal function for mojo use. * Add an unseal function for mojo to use. * Switch mojo to use new function after series upgrade reboot. * Add fallback if vip client is unavailable (the case if all hacluster units are paused for series upgrade). --- .../charm_tests/series_upgrade/tests.py | 2 +- zaza/openstack/charm_tests/vault/setup.py | 18 ++++++++++++++++++ zaza/openstack/charm_tests/vault/utils.py | 4 ++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index fa5257c..ab82b0d 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -85,7 +85,7 @@ class SeriesUpgradeTest(unittest.TestCase): if "vault" in applications[application]["charm"]: post_upgrade_functions = [ ('zaza.openstack.charm_tests.vault.setup.' - 'basic_setup_and_unseal')] + 'basic_unseal_mojo_cacert')] if ("mongodb" in applications[application]["charm"] or "vault" in applications[application]["charm"]): # Mongodb and vault need to run series upgrade diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 952bbcd..47623cd 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -15,6 +15,7 @@ """Run configuration phase.""" import functools +import os import requests import tempfile @@ -51,6 +52,23 @@ def basic_setup_and_unseal(cacert=None): zaza.model.run_on_unit(unit.name, './hooks/update-status') +def basic_unseal_mojo_cacert(): + """Unseal Vault and search for cacert to use. + + This is designed to be used from a mojo spec where certs are stored in the + $MOJO_LOCAL directory. + """ + try: + cert_dir = os.environ['MOJO_LOCAL'] + except KeyError: + raise Exception("Could not find cacert.pem, MOJO_LOCAL unset") + cacert = os.path.join(cert_dir, 'cacert.pem') + if os.path.exists(cacert): + basic_setup_and_unseal(cacert=cacert) + else: + raise Exception("Could not find cacert.pem") + + def auto_initialize(cacert=None, validation_application='keystone'): """Auto initialize vault for testing. diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index b6f4cf5..c0ac82e 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -55,6 +55,10 @@ class VaultFacade: self.vip_client = get_vip_client(cacert=cacert) if self.vip_client: self.unseal_client = self.vip_client + try: + self.unseal_client.hvac_client.is_initialized() + except requests.exceptions.ConnectionError: + self.unseal_client = self.clients[0] else: self.unseal_client = self.clients[0] self.initialized = is_initialized(self.unseal_client) From cac2f4c4a7906e3c50c8d35304a457e7109aa5b6 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 3 Feb 2020 16:03:42 +0000 Subject: [PATCH 277/898] Fix MOJO_LOCAL_DIR name --- zaza/openstack/charm_tests/vault/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 47623cd..6942393 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -59,7 +59,7 @@ def basic_unseal_mojo_cacert(): $MOJO_LOCAL directory. """ try: - cert_dir = os.environ['MOJO_LOCAL'] + cert_dir = os.environ['MOJO_LOCAL_DIR'] except KeyError: raise Exception("Could not find cacert.pem, MOJO_LOCAL unset") cacert = os.path.join(cert_dir, 'cacert.pem') From d66358ce41fc706128573ab464d1a9bd6567c4ea Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 4 Feb 2020 09:16:44 +0100 Subject: [PATCH 278/898] policyd/octavia: Use correct network for vips Which network to use for vips depend on deployment configuration. --- zaza/openstack/charm_tests/policyd/tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 532495f..f332765 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -679,7 +679,11 @@ class OctaviaTests(BasePolicydSpecialization): octavia_client = openstack_utils.get_octavia_session_client(sess) neutron_client = openstack_utils.get_neutron_session_client(sess) - resp = neutron_client.list_networks(name="private_lb_fip_network") + if openstack_utils.dvr_enabled(): + network_name = 'private_lb_fip_network' + else: + network_name = 'private' + resp = neutron_client.list_networks(name=network_name) vip_subnet_id = resp['networks'][0]['subnets'][0] From 5927878a0383fce4d6ff4f107d7aa5c3d5cfa1e8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 4 Feb 2020 10:10:50 +0000 Subject: [PATCH 279/898] Tidyup mojo cert unseal --- .../charm_tests/series_upgrade/tests.py | 10 ++++--- zaza/openstack/charm_tests/vault/setup.py | 28 ++++++++----------- zaza/openstack/charm_tests/vault/utils.py | 4 --- zaza/openstack/utilities/exceptions.py | 6 ++++ zaza/openstack/utilities/generic.py | 19 +++++++++++++ 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index ab82b0d..bd49c8a 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -83,12 +83,14 @@ class SeriesUpgradeTest(unittest.TestCase): pause_non_leader_primary = False pause_non_leader_subordinate = False if "vault" in applications[application]["charm"]: + origin = None + pause_non_leader_primary = False + pause_non_leader_subordinate = True post_upgrade_functions = [ ('zaza.openstack.charm_tests.vault.setup.' - 'basic_unseal_mojo_cacert')] - if ("mongodb" in applications[application]["charm"] or - "vault" in applications[application]["charm"]): - # Mongodb and vault need to run series upgrade + 'mojo_unseal_by_unit')] + if "mongodb" in applications[application]["charm"]: + # Mongodb needs to run series upgrade # on its secondaries first. generic_utils.series_upgrade_non_leaders_first( application, diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 6942393..9ea409d 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -15,7 +15,6 @@ """Run configuration phase.""" import functools -import os import requests import tempfile @@ -24,6 +23,7 @@ import zaza.openstack.charm_tests.vault.utils as vault_utils import zaza.model import zaza.openstack.utilities.cert import zaza.openstack.utilities.openstack +import zaza.openstack.utilities.generic def basic_setup(cacert=None, unseal_and_authorize=False): @@ -52,21 +52,17 @@ def basic_setup_and_unseal(cacert=None): zaza.model.run_on_unit(unit.name, './hooks/update-status') -def basic_unseal_mojo_cacert(): - """Unseal Vault and search for cacert to use. - - This is designed to be used from a mojo spec where certs are stored in the - $MOJO_LOCAL directory. - """ - try: - cert_dir = os.environ['MOJO_LOCAL_DIR'] - except KeyError: - raise Exception("Could not find cacert.pem, MOJO_LOCAL unset") - cacert = os.path.join(cert_dir, 'cacert.pem') - if os.path.exists(cacert): - basic_setup_and_unseal(cacert=cacert) - else: - raise Exception("Could not find cacert.pem") +def mojo_unseal_by_unit(): + """Unseal any units reported as sealed using mojo cacert.""" + cacert = zaza.openstack.utilities.generic.get_mojo_cacert() + vault_creds = vault_utils.get_credentails() + for client in vault_utils.get_clients(cacert=cacert): + if client.hvac_client.is_sealed(): + client.hvac_client.unseal(vault_creds['keys'][0]) + unit_name = zaza.utilities.juju.get_unit_name_from_ip_address( + client.addr, + 'vault') + zaza.model.run_on_unit(unit_name, './hooks/update-status') def auto_initialize(cacert=None, validation_application='keystone'): diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index c0ac82e..b6f4cf5 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -55,10 +55,6 @@ class VaultFacade: self.vip_client = get_vip_client(cacert=cacert) if self.vip_client: self.unseal_client = self.vip_client - try: - self.unseal_client.hvac_client.is_initialized() - except requests.exceptions.ConnectionError: - self.unseal_client = self.clients[0] else: self.unseal_client = self.clients[0] self.initialized = is_initialized(self.unseal_client) diff --git a/zaza/openstack/utilities/exceptions.py b/zaza/openstack/utilities/exceptions.py index f59b4c1..e9e5d23 100644 --- a/zaza/openstack/utilities/exceptions.py +++ b/zaza/openstack/utilities/exceptions.py @@ -184,3 +184,9 @@ class PolicydError(Exception): """Policyd override failed.""" pass + + +class CACERTNotFound(Exception): + """Could not find cacert.""" + + pass diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 7b673d0..c3a7afb 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -850,3 +850,22 @@ def systemctl(unit, service, command="restart"): unit.entity_id, cmd) assert int(result['Code']) == 0, ( "{} of {} on {} failed".format(command, service, unit.entity_id)) + + +def get_mojo_cacert(): + """Retrieve cacert from Mojo storage location. + + :returns: Pathh to cacert + :rtype: str + :raises: zaza_exceptions.CACERTNotFound + """ + try: + cert_dir = os.environ['MOJO_LOCAL_DIR'] + except KeyError: + raise zaza_exceptions.CACERTNotFound( + "Could not find cacert.pem, MOJO_LOCAL_DIR unset") + cacert = os.path.join(cert_dir, 'cacert.pem') + if os.path.exists(cacert): + return cacert + else: + raise zaza_exceptions.CACERTNotFound("Could not find cacert.pem") From 9c8a18b66fa3e5b9775e2cf53a427d9dd39748b0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 4 Feb 2020 11:05:23 +0000 Subject: [PATCH 280/898] Fix path to get_unit_name_from_ip_address --- zaza/openstack/charm_tests/vault/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 9ea409d..6f93c0d 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -24,6 +24,7 @@ import zaza.model import zaza.openstack.utilities.cert import zaza.openstack.utilities.openstack import zaza.openstack.utilities.generic +import zaza.utilities.juju as juju_utils def basic_setup(cacert=None, unseal_and_authorize=False): @@ -59,7 +60,7 @@ def mojo_unseal_by_unit(): for client in vault_utils.get_clients(cacert=cacert): if client.hvac_client.is_sealed(): client.hvac_client.unseal(vault_creds['keys'][0]) - unit_name = zaza.utilities.juju.get_unit_name_from_ip_address( + unit_name = juju_utils.get_unit_name_from_ip_address( client.addr, 'vault') zaza.model.run_on_unit(unit_name, './hooks/update-status') From 5fdde89be073f4f04e3c029c1fc98aa41a182b1f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 4 Feb 2020 12:48:00 +0000 Subject: [PATCH 281/898] Tidyup docstring and method name --- test.py | 13 +++++++++++++ zaza/openstack/charm_tests/vault/setup.py | 2 +- zaza/openstack/utilities/generic.py | 5 +++-- 3 files changed, 17 insertions(+), 3 deletions(-) create mode 100755 test.py diff --git a/test.py b/test.py new file mode 100755 index 0000000..6c94b60 --- /dev/null +++ b/test.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +from zaza.openstack.utilities import ( + cli as cli_utils, + openstack as openstack_utils, + openstack_upgrade as upgrade_utils, +) + + +applications = [] +for group in ['Core Identity', 'Storage', 'Control Plane', 'Compute']: + applications.extend(upgrade_utils.SERVICE_GROUPS[group]) +print(applications) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 6f93c0d..96c45cb 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -55,7 +55,7 @@ def basic_setup_and_unseal(cacert=None): def mojo_unseal_by_unit(): """Unseal any units reported as sealed using mojo cacert.""" - cacert = zaza.openstack.utilities.generic.get_mojo_cacert() + cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path() vault_creds = vault_utils.get_credentails() for client in vault_utils.get_clients(cacert=cacert): if client.hvac_client.is_sealed(): diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index c3a7afb..c0566b0 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -852,12 +852,13 @@ def systemctl(unit, service, command="restart"): "{} of {} on {} failed".format(command, service, unit.entity_id)) -def get_mojo_cacert(): +def get_mojo_cacert_path(): """Retrieve cacert from Mojo storage location. - :returns: Pathh to cacert + :returns: Path to cacert :rtype: str :raises: zaza_exceptions.CACERTNotFound + :raises: :class:`zaza_exceptions.CACERTNotfound` """ try: cert_dir = os.environ['MOJO_LOCAL_DIR'] From 00af513cf4efb5c297f7a2111dc490f1b01ccf70 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 4 Feb 2020 12:55:19 +0000 Subject: [PATCH 282/898] Skip series upgrade of easyrsa (Bug #1850124) --- zaza/openstack/charm_tests/series_upgrade/tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index bd49c8a..bad4b1d 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -59,6 +59,9 @@ class SeriesUpgradeTest(unittest.TestCase): if "easyrsa" in applications[application]["charm"]: logging.warn("Skipping series upgrade of easyrsa Bug #1850121") continue + if "etcd" in applications[application]["charm"]: + logging.warn("Skipping series upgrade of easyrsa Bug #1850124") + continue if "percona-cluster" in applications[application]["charm"]: origin = "source" pause_non_leader_primary = True From ee21886a8cdf7a89d9d1192a57f8bf24c26dc896 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 4 Feb 2020 13:33:28 +0000 Subject: [PATCH 283/898] Fix order in UPGRADE_SERVICES --- test.py | 13 ------------- zaza/openstack/utilities/openstack.py | 15 ++++++++++----- 2 files changed, 10 insertions(+), 18 deletions(-) delete mode 100755 test.py diff --git a/test.py b/test.py deleted file mode 100755 index 6c94b60..0000000 --- a/test.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python3 - -from zaza.openstack.utilities import ( - cli as cli_utils, - openstack as openstack_utils, - openstack_upgrade as upgrade_utils, -) - - -applications = [] -for group in ['Core Identity', 'Storage', 'Control Plane', 'Compute']: - applications.extend(upgrade_utils.SERVICE_GROUPS[group]) -print(applications) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 8b578e6..6d114b8 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -108,18 +108,23 @@ CHARM_TYPES = { 'origin_setting': 'openstack-origin' }, } + +# Older tests use the order the services appear in the list to imply +# they shoould be upgraded in. This approache has been superceded and +# zaza.openstack.utilities.openstack_upgrade.get_upgrade_groups should be used +# instead UPGRADE_SERVICES = [ {'name': 'keystone', 'type': CHARM_TYPES['keystone']}, - {'name': 'nova-cloud-controller', 'type': CHARM_TYPES['nova']}, - {'name': 'nova-compute', 'type': CHARM_TYPES['nova']}, {'name': 'neutron-api', 'type': CHARM_TYPES['neutron']}, - {'name': 'neutron-gateway', 'type': CHARM_TYPES['neutron']}, + {'name': 'nova-cloud-controller', 'type': CHARM_TYPES['nova']}, {'name': 'glance', 'type': CHARM_TYPES['glance']}, {'name': 'cinder', 'type': CHARM_TYPES['cinder']}, - {'name': 'openstack-dashboard', - 'type': CHARM_TYPES['openstack-dashboard']}, + {'name': 'neutron-gateway', 'type': CHARM_TYPES['neutron']}, {'name': 'ceilometer', 'type': CHARM_TYPES['ceilometer']}, {'name': 'designate', 'type': CHARM_TYPES['designate']}, + {'name': 'nova-compute', 'type': CHARM_TYPES['nova']}, + {'name': 'openstack-dashboard', + 'type': CHARM_TYPES['openstack-dashboard']}, ] From 51c2ec89b0c4342be5b211674ffa8e8a050792f9 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 4 Feb 2020 13:36:44 +0000 Subject: [PATCH 284/898] Fix spelling --- zaza/openstack/utilities/openstack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 6d114b8..7e48a98 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -110,9 +110,9 @@ CHARM_TYPES = { } # Older tests use the order the services appear in the list to imply -# they shoould be upgraded in. This approache has been superceded and +# the order they should be upgraded in. This approach has been superceded and # zaza.openstack.utilities.openstack_upgrade.get_upgrade_groups should be used -# instead +# instead. UPGRADE_SERVICES = [ {'name': 'keystone', 'type': CHARM_TYPES['keystone']}, {'name': 'neutron-api', 'type': CHARM_TYPES['neutron']}, From 9d2369f5760dff8d7230fc48029193dfb004052c Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 4 Feb 2020 12:58:55 -0600 Subject: [PATCH 285/898] designate-bind zaza tests --- .../charm_tests/designate/__init__.py | 73 ++++++++ zaza/openstack/charm_tests/designate/tests.py | 175 ++++++++++++++++++ zaza/openstack/utilities/openstack.py | 14 ++ 3 files changed, 262 insertions(+) create mode 100644 zaza/openstack/charm_tests/designate/__init__.py create mode 100644 zaza/openstack/charm_tests/designate/tests.py diff --git a/zaza/openstack/charm_tests/designate/__init__.py b/zaza/openstack/charm_tests/designate/__init__.py new file mode 100644 index 0000000..5ae540b --- /dev/null +++ b/zaza/openstack/charm_tests/designate/__init__.py @@ -0,0 +1,73 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing designate.""" +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +class BaseDesignateTest(test_utils.OpenStackBaseTest): + """Base for Designate charm tests.""" + + @classmethod + def setUpClass(cls, application_name=None, model_alias=None): + """Run class setup for running Designate charm operation tests.""" + application_name = application_name or "designate" + model_alias = model_alias or "" + super(BaseDesignateTest, cls).setUpClass(application_name, model_alias) + os_release = openstack_utils.get_os_release + + if os_release() >= os_release('bionic_rocky'): + cls.designate_svcs = [ + 'designate-agent', 'designate-api', 'designate-central', + 'designate-mdns', 'designate-worker', 'designate-sink', + 'designate-producer', + ] + else: + cls.designate_svcs = [ + 'designate-agent', 'designate-api', 'designate-central', + 'designate-mdns', 'designate-pool-manager', 'designate-sink', + 'designate-zone-manager', + ] + + # Get keystone session + keystone_api = 3 if (os_release() >= os_release('xenial_queens')) else 2 + cls.keystone_session = openstack_utils.get_overcloud_keystone_session() + cls.keystone = openstack_utils.get_keystone_session_client( + cls.keystone_session, keystone_api + ) + + if os_release() >= os_release('xenial_queens'): + cls.designate = openstack_utils.get_designate_session_client( + session=cls.keystone_session + ) + cls.zones_list = cls.designate.zones.list + cls.zones_delete = cls.designate.zones.delete + else: + # Authenticate admin with designate endpoint + designate_ep = cls.keystone.service_catalog.url_for( + service_type='dns', + interface='publicURL') + keystone_ep = cls.keystone.service_catalog.url_for( + service_type='identity', + interface='publicURL') + cls.designate = openstack_utils.get_designate_session_client( + version=1, + auth_url=keystone_ep, + username="admin", + password="openstack", + tenant_name="admin", + endpoint=designate_ep) + cls.zones_list = cls.designate.domains.list + cls.zones_delete = cls.designate.domains.delete diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py new file mode 100644 index 0000000..22a669e --- /dev/null +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -0,0 +1,175 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate designate testing.""" +import logging +import tenacity +import subprocess +import designateclient.v1.domains as domains +import designateclient.v1.records as records +import zaza.model as model +import zaza.openstack.utilities.juju as zaza_juju +from zaza.openstack.charm_tests.designate import ( + BaseDesignateTest, +) +import zaza.openstack.utilities.openstack as openstack_utils + + +class BaseTests(BaseDesignateTest): + def test_100_services(self): + """Verify the expected services are running on the corresponding + service units.""" + logging.debug('Checking system services on units...') + + model.block_until_service_status( + self.application_name, + self.designate_svcs, + 'running', + self.model_name, + timeout=30 + ) + + logging.debug('OK') + + +class BindTests(BaseDesignateTest): + TEST_DOMAIN = 'amuletexample.com.' + TEST_WWW_RECORD = "www.{}".format(TEST_DOMAIN) + TEST_RECORD = {TEST_WWW_RECORD: '10.0.0.23'} + + def _get_domain_id(self, domain_name): + domain_id = None + for dom in self.zones_list(): + if isinstance(dom, dict): + if dom['name'] == domain_name: + domain_id = dom['name'] + break + else: + if dom.name == domain_name: + domain_id = dom.id + break + return domain_id + + def _get_test_domain_id(self): + return self._get_domain_id(self.TEST_DOMAIN) + + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), + reraise=True + ) + def _wait_on_domain_gone(self): + logging.debug('Waiting for domain to disappear') + if self._get_test_domain_id(): + raise Exception("Domain Exists") + + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), + reraise=True + ) + def _wait_to_resolve_test_record(self, dns_ip): + logging.debug('Waiting for dns record to propagate') + lookup_cmd = [ + 'dig', '+short', '@{}'.format(dns_ip), + self.TEST_WWW_RECORD] + cmd_out = subprocess.check_output( + lookup_cmd, universal_newlines=True).rstrip() + if not self.TEST_RECORD[self.TEST_WWW_RECORD] == cmd_out: + raise Exception("Record Doesn't Exist") + + def test_205_designate_designate_bind_relation(self): + """Verify the designate to designate-bind dns-backend relation data""" + logging.debug('Checking designate:designate-bind dns-backend relation' + 'data...') + + unit_name = 'designate/0' + remote_unit_name = 'designate-bind/0' + relation_name = 'dns-backend' + remote_unit = model.get_unit_from_name(remote_unit_name) + remote_ip = remote_unit.public_address + relation = zaza_juju.get_relation_from_unit( + unit_name, + remote_unit_name, + relation_name + ) + # Get private-address in relation + rel_private_ip = relation.get('private-address') + # The private address in relation should match designate-bind/0 address + self.assertEqual(rel_private_ip, remote_ip) + + def test_206_designate_bind_designate_relation(self): + """Verify the designate_bind to designate dns-backend relation data""" + logging.debug('Checking designate-bind:designate dns-backend relation' + 'data...') + + unit_name = 'designate-bind/0' + remote_unit_name = 'designate/0' + relation_name = 'dns-backend' + remote_unit = model.get_unit_from_name(remote_unit_name) + remote_ip = remote_unit.public_address + relation = zaza_juju.get_relation_from_unit( + unit_name, + remote_unit_name, + relation_name + ) + # Get private-address in relation + rel_private_ip = relation.get('private-address') + # The private address in relation should match designate-bind/0 address + self.assertEqual(rel_private_ip, remote_ip) + + def test_400_domain_creation(self): + """Simple api calls to create domain""" + logging.debug('Checking if domain exists before trying to create it') + old_dom_id = self._get_test_domain_id() + if old_dom_id: + logging.debug('Deleting old domain') + self.zones_delete(old_dom_id) + self._wait_on_domain_gone() + + logging.debug('Creating new domain') + domain = domains.Domain( + name=self.TEST_DOMAIN, + email="fred@amuletexample.com") + + os_release = openstack_utils.get_os_release + if os_release() >= os_release('xenial_queens'): + new_domain = self.designate.zones.create( + name=domain.name, email=domain.email) + else: + new_domain = self.designate.domains.create(domain) + self.assertIsNotNone(new_domain) + + logging.debug('Creating new test record') + _record = records.Record( + name=self.TEST_WWW_RECORD, + type="A", + data=self.TEST_RECORD[self.TEST_WWW_RECORD]) + + if os_release() >= os_release('xenial_queens'): + _domain_id = new_domain['id'] + self.designate.recordsets.create( + _domain_id, _record.name, _record.type, [_record.data]) + else: + _domain_id = new_domain.id + self.designate.records.create(_domain_id, _record) + + dns_ip = zaza_juju.get_relation_from_unit( + 'designate-bind/0', + 'designate/0', + 'dns-backend' + ).get('private-address') + self._wait_to_resolve_test_record(dns_ip) + + logging.debug('Tidy up delete test record') + self.zones_delete(_domain_id) + logging.debug('OK') diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 7e48a98..83167e8 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -29,6 +29,7 @@ from aodhclient.v2 import client as aodh_client from cinderclient import client as cinderclient from heatclient import client as heatclient from glanceclient import Client as GlanceClient +from designateclient.client import Client as DesignateClient from keystoneclient.v2_0 import client as keystoneclient_v2 from keystoneclient.v3 import client as keystoneclient_v3 @@ -208,6 +209,19 @@ def get_glance_session_client(session): return GlanceClient('2', session=session) +def get_designate_session_client(**kwargs): + """Return designateclient authenticated by keystone session. + + :param kwargs: Designate Client Arguments + :returns: Authenticated designateclient + :rtype: DesignateClient + """ + version = kwargs.pop('version', None) or 2 + return DesignateClient(session=session, + version=str(version), + **kwargs) + + def get_nova_session_client(session): """Return novaclient authenticated by keystone session. From 3d58e9661eb241d315e2d791b35af85fc4c459e7 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 4 Feb 2020 13:17:54 -0600 Subject: [PATCH 286/898] include designateclient into zaza requirements --- setup.py | 1 + zaza/openstack/charm_tests/designate/__init__.py | 2 +- zaza/openstack/charm_tests/designate/tests.py | 15 +++++++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 7f2d6a4..745731d 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ install_require = [ 'tenacity', 'oslo.config', 'aodhclient', + 'python-designateclient', 'python-heatclient', 'python-glanceclient', 'python-keystoneclient', diff --git a/zaza/openstack/charm_tests/designate/__init__.py b/zaza/openstack/charm_tests/designate/__init__.py index 5ae540b..7be4c14 100644 --- a/zaza/openstack/charm_tests/designate/__init__.py +++ b/zaza/openstack/charm_tests/designate/__init__.py @@ -42,7 +42,7 @@ class BaseDesignateTest(test_utils.OpenStackBaseTest): ] # Get keystone session - keystone_api = 3 if (os_release() >= os_release('xenial_queens')) else 2 + keystone_api = 3 if os_release() >= os_release('xenial_queens') else 2 cls.keystone_session = openstack_utils.get_overcloud_keystone_session() cls.keystone = openstack_utils.get_keystone_session_client( cls.keystone_session, keystone_api diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 22a669e..30f202a 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -27,9 +27,10 @@ import zaza.openstack.utilities.openstack as openstack_utils class BaseTests(BaseDesignateTest): + """Base Designate charm tests.""" + def test_100_services(self): - """Verify the expected services are running on the corresponding - service units.""" + """Verify expected services are running.""" logging.debug('Checking system services on units...') model.block_until_service_status( @@ -44,6 +45,8 @@ class BaseTests(BaseDesignateTest): class BindTests(BaseDesignateTest): + """Designate Bind charm tests.""" + TEST_DOMAIN = 'amuletexample.com.' TEST_WWW_RECORD = "www.{}".format(TEST_DOMAIN) TEST_RECORD = {TEST_WWW_RECORD: '10.0.0.23'} @@ -88,7 +91,7 @@ class BindTests(BaseDesignateTest): raise Exception("Record Doesn't Exist") def test_205_designate_designate_bind_relation(self): - """Verify the designate to designate-bind dns-backend relation data""" + """Verify the designate to designate-bind dns-backend relation data.""" logging.debug('Checking designate:designate-bind dns-backend relation' 'data...') @@ -108,7 +111,7 @@ class BindTests(BaseDesignateTest): self.assertEqual(rel_private_ip, remote_ip) def test_206_designate_bind_designate_relation(self): - """Verify the designate_bind to designate dns-backend relation data""" + """Verify the designate_bind to designate dns-backend relation data.""" logging.debug('Checking designate-bind:designate dns-backend relation' 'data...') @@ -126,9 +129,9 @@ class BindTests(BaseDesignateTest): rel_private_ip = relation.get('private-address') # The private address in relation should match designate-bind/0 address self.assertEqual(rel_private_ip, remote_ip) - + def test_400_domain_creation(self): - """Simple api calls to create domain""" + """Simple api calls to create domain.""" logging.debug('Checking if domain exists before trying to create it') old_dom_id = self._get_test_domain_id() if old_dom_id: From 39fb848452bf786fac2289aa60eb31726ee0e396 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 4 Feb 2020 13:39:13 -0600 Subject: [PATCH 287/898] pin python-designateclient to 2.x --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 745731d..d212ae8 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_require = [ 'tenacity', 'oslo.config', 'aodhclient', - 'python-designateclient', + 'python-designateclient>=1.5,<3.0.0', 'python-heatclient', 'python-glanceclient', 'python-keystoneclient', From b88bccd526bdde9a1ab473057f954a6e4df036e6 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 4 Feb 2020 14:04:42 -0600 Subject: [PATCH 288/898] Successful tests --- zaza/openstack/charm_tests/designate/tests.py | 9 +++++---- zaza/openstack/utilities/openstack.py | 4 +++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 30f202a..65ae3c0 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -81,7 +81,7 @@ class BindTests(BaseDesignateTest): reraise=True ) def _wait_to_resolve_test_record(self, dns_ip): - logging.debug('Waiting for dns record to propagate') + logging.info('Waiting for dns record to propagate @ {}'.format(dns_ip)) lookup_cmd = [ 'dig', '+short', '@{}'.format(dns_ip), self.TEST_WWW_RECORD] @@ -145,7 +145,8 @@ class BindTests(BaseDesignateTest): email="fred@amuletexample.com") os_release = openstack_utils.get_os_release - if os_release() >= os_release('xenial_queens'): + post_xenial_queens = os_release() >= os_release('xenial_queens') + if post_xenial_queens: new_domain = self.designate.zones.create( name=domain.name, email=domain.email) else: @@ -158,7 +159,7 @@ class BindTests(BaseDesignateTest): type="A", data=self.TEST_RECORD[self.TEST_WWW_RECORD]) - if os_release() >= os_release('xenial_queens'): + if post_xenial_queens: _domain_id = new_domain['id'] self.designate.recordsets.create( _domain_id, _record.name, _record.type, [_record.data]) @@ -167,8 +168,8 @@ class BindTests(BaseDesignateTest): self.designate.records.create(_domain_id, _record) dns_ip = zaza_juju.get_relation_from_unit( - 'designate-bind/0', 'designate/0', + 'designate-bind/0', 'dns-backend' ).get('private-address') self._wait_to_resolve_test_record(dns_ip) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 83167e8..077e50e 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -209,9 +209,11 @@ def get_glance_session_client(session): return GlanceClient('2', session=session) -def get_designate_session_client(**kwargs): +def get_designate_session_client(session, **kwargs): """Return designateclient authenticated by keystone session. + :param session: Keystone session object + :type session: keystoneauth1.session.Session object :param kwargs: Designate Client Arguments :returns: Authenticated designateclient :rtype: DesignateClient From 2f5ef8f027515fe18d700d800ec4e9f911298fa3 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 4 Feb 2020 14:25:05 -0600 Subject: [PATCH 289/898] eliminate multiple checks for post xenial-queens --- zaza/openstack/charm_tests/designate/__init__.py | 5 +++-- zaza/openstack/charm_tests/designate/tests.py | 6 ++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/__init__.py b/zaza/openstack/charm_tests/designate/__init__.py index 7be4c14..66e2373 100644 --- a/zaza/openstack/charm_tests/designate/__init__.py +++ b/zaza/openstack/charm_tests/designate/__init__.py @@ -42,13 +42,14 @@ class BaseDesignateTest(test_utils.OpenStackBaseTest): ] # Get keystone session - keystone_api = 3 if os_release() >= os_release('xenial_queens') else 2 + cls.post_xenial_queens = os_release() >= os_release('xenial_queens') + keystone_api = 3 if cls.post_xenial_queens else 2 cls.keystone_session = openstack_utils.get_overcloud_keystone_session() cls.keystone = openstack_utils.get_keystone_session_client( cls.keystone_session, keystone_api ) - if os_release() >= os_release('xenial_queens'): + if cls.post_xenial_queens: cls.designate = openstack_utils.get_designate_session_client( session=cls.keystone_session ) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 65ae3c0..af9099d 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -144,9 +144,7 @@ class BindTests(BaseDesignateTest): name=self.TEST_DOMAIN, email="fred@amuletexample.com") - os_release = openstack_utils.get_os_release - post_xenial_queens = os_release() >= os_release('xenial_queens') - if post_xenial_queens: + if self.post_xenial_queens: new_domain = self.designate.zones.create( name=domain.name, email=domain.email) else: @@ -159,7 +157,7 @@ class BindTests(BaseDesignateTest): type="A", data=self.TEST_RECORD[self.TEST_WWW_RECORD]) - if post_xenial_queens: + if self.post_xenial_queens: _domain_id = new_domain['id'] self.designate.recordsets.create( _domain_id, _record.name, _record.type, [_record.data]) From 2f395314efeb735d85e0db893e23d24f80400e2f Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 4 Feb 2020 14:32:11 -0600 Subject: [PATCH 290/898] pep8 issues --- zaza/openstack/charm_tests/designate/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index af9099d..9949dc1 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -23,7 +23,6 @@ import zaza.openstack.utilities.juju as zaza_juju from zaza.openstack.charm_tests.designate import ( BaseDesignateTest, ) -import zaza.openstack.utilities.openstack as openstack_utils class BaseTests(BaseDesignateTest): From d5ee2106d41b09998eb6c59560301cf1a3cab1a5 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 4 Feb 2020 16:14:35 -0600 Subject: [PATCH 291/898] port designate amulet tests to zaza --- zaza/openstack/charm_tests/designate/tests.py | 403 ++++++++++++++++-- zaza/openstack/utilities/openstack.py | 7 +- 2 files changed, 359 insertions(+), 51 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 9949dc1..a96e931 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -13,11 +13,13 @@ # limitations under the License. """Encapsulate designate testing.""" +import re import logging import tenacity import subprocess import designateclient.v1.domains as domains import designateclient.v1.records as records +import designateclient.v1.servers as servers import zaza.model as model import zaza.openstack.utilities.juju as zaza_juju from zaza.openstack.charm_tests.designate import ( @@ -43,51 +45,223 @@ class BaseTests(BaseDesignateTest): logging.debug('OK') -class BindTests(BaseDesignateTest): - """Designate Bind charm tests.""" +class ApiTests(BaseDesignateTest): + """Designate charm api tests.""" - TEST_DOMAIN = 'amuletexample.com.' - TEST_WWW_RECORD = "www.{}".format(TEST_DOMAIN) - TEST_RECORD = {TEST_WWW_RECORD: '10.0.0.23'} + VALID_URL_TOKENS = { + 'adminURL' + 'publicURL' + 'internalURL' + } - def _get_domain_id(self, domain_name): - domain_id = None - for dom in self.zones_list(): - if isinstance(dom, dict): - if dom['name'] == domain_name: - domain_id = dom['name'] - break - else: - if dom.name == domain_name: - domain_id = dom.id - break - return domain_id + VALID_URL = re.compile( + r'^(?:http|ftp)s?://' + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa + r'localhost|' + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' + r'(?::\d+)?' + r'(?:/?|[/?]\S+)$', + re.IGNORECASE) - def _get_test_domain_id(self): - return self._get_domain_id(self.TEST_DOMAIN) + def test_110_service_catalog(self): + """Verify that the service catalog endpoint data is valid.""" + logging.debug('Checking keystone service catalog data...') + actual = self.keystone.service_catalog.get_endpoints() + self.assertIsNotNone(actual['dns'].get('id')) + self.assertEqual(actual['dns'].get('region'), "RegionOne") + for token in self.VALID_URL_TOKENS: + validate = actual['dns'].get(token) + self.assertIsNotNone( + validate, "token {} doesn't exist".format(token)) + self.assertRegexpMatches(validate, self.VALID_URL) - @tenacity.retry( - wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), - reraise=True - ) - def _wait_on_domain_gone(self): - logging.debug('Waiting for domain to disappear') - if self._get_test_domain_id(): - raise Exception("Domain Exists") + def test_114_designate_api_endpoint(self): + """Verify the designate api endpoint data.""" + logging.debug('Checking designate api endpoint data...') + endpoints = self.keystone.endpoints.list() + logging.debug(endpoints) - @tenacity.retry( - wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), - reraise=True - ) - def _wait_to_resolve_test_record(self, dns_ip): - logging.info('Waiting for dns record to propagate @ {}'.format(dns_ip)) - lookup_cmd = [ - 'dig', '+short', '@{}'.format(dns_ip), - self.TEST_WWW_RECORD] - cmd_out = subprocess.check_output( - lookup_cmd, universal_newlines=True).rstrip() - if not self.TEST_RECORD[self.TEST_WWW_RECORD] == cmd_out: - raise Exception("Record Doesn't Exist") + self.assertIsNotNone(endpoints.get('id')) + self.assertEqual(endpoints.get('region'), "RegionOne") + self.assertIsNotNone(endpoints.get('server_id')) + for token in self.VALID_URL_TOKENS: + validate = endpoints.get(token) + self.assertIsNotNone( + validate, "token {} doesn't exist".format(token)) + self.assertRegexpMatches(validate, self.VALID_URL) + + +class KeystoneIdentityRelationTests(BaseDesignateTest): + """Designate Keystone identity relations charm tests.""" + + def test_200_designate_identity_relation(self): + """Verify the designate to keystone identity-service relation data.""" + logging.debug('Checking designate to keystone identity-service ' + 'relation data...') + + unit_name = 'designate/0' + remote_unit_name = 'keystone/0' + relation_name = 'identity-service' + remote_unit = model.get_unit_from_name(remote_unit_name) + remote_ip = remote_unit.public_address + relation = zaza_juju.get_relation_from_unit( + unit_name, + remote_unit_name, + relation_name + ) + + designate_endpoint = "http://{}:9001".format(remote_ip) + expected = { + 'admin_url': designate_endpoint, + 'internal_url': designate_endpoint, + 'private-address': remote_ip, + 'public_url': designate_endpoint, + 'region': 'RegionOne', + 'service': 'designate', + } + + for token, value in expected.items(): + r_val = relation.get(token) + self.assertEqual( + r_val, value, "token({}) doesn't match".format(token) + ) + + def test_201_keystone_designate_identity_relation(self): + """Verify the keystone to designate identity-service relation data.""" + logging.debug('Checking keystone:designate identity relation data...') + + unit_name = 'keystone/0' + remote_unit_name = 'designate/0' + relation_name = 'identity-service' + remote_unit = model.get_unit_from_name(remote_unit_name) + remote_ip = remote_unit.public_address + relation = zaza_juju.get_relation_from_unit( + unit_name, + remote_unit_name, + relation_name + ) + expected = { + 'admin_token': 'ubuntutesting', + 'auth_host': remote_ip, + 'auth_port': "35357", + 'auth_protocol': 'http', + 'private-address': remote_ip, + 'service_host': remote_ip, + 'service_port': "5000", + 'service_protocol': 'http', + 'service_tenant': 'services', + 'service_username': 'designate', + } + + for token, value in expected.items(): + r_val = relation.get(token) + self.assertEqual( + r_val, value, "token({}) doesn't match".format(token) + ) + + self.assertIsNotNone( + relation.get('service_password'), + "service_password missing" + ) + + self.assertIsNotNone( + relation.get('service_tenant_id'), + "service_tenant_id missing" + ) + + +class AmqpRelationTests(BaseDesignateTest): + """Designate Rabbit MQ relations charm tests.""" + + def test_203_designate_amqp_relation(self): + """Verify the designate to rabbitmq-server amqp relation data.""" + logging.debug('Checking designate:amqp rabbitmq relation data...') + + unit_name = 'designate/0' + remote_unit_name = 'rabbitmq-server/0' + relation_name = 'amqp' + remote_unit = model.get_unit_from_name(remote_unit_name) + remote_ip = remote_unit.public_address + relation = zaza_juju.get_relation_from_unit( + unit_name, + remote_unit_name, + relation_name + ) + # Get private-address in relation + rel_private_ip = relation.get('private-address') + username = relation.get('username') + vhost = relation.get('vhost') + + self.assertEqual(rel_private_ip, remote_ip) + self.assertEqual(username, 'designate') + self.assertEqual(vhost, 'openstack') + + def test_204_amqp_designate_relation(self): + """Verify the rabbitmq-server to designate amqp relation data.""" + logging.debug('Checking rabbitmq:amqp designate relation data...') + + unit_name = 'rabbitmq-server/0' + remote_unit_name = 'designate/0' + relation_name = 'amqp' + remote_unit = model.get_unit_from_name(remote_unit_name) + remote_ip = remote_unit.public_address + relation = zaza_juju.get_relation_from_unit( + unit_name, + remote_unit_name, + relation_name + ) + # Get private-address in relation + rel_private_ip = relation.get('private-address') + username = relation.get('username') + vhost = relation.get('vhost') + + self.assertEqual(rel_private_ip, remote_ip) + self.assertEqual(username, 'designate') + self.assertEqual(vhost, 'openstack') + + +class NeutronApiRelationTests(BaseDesignateTest): + """Designate Neutron API relations charm tests.""" + + def test_207_designate_neutron_api_relation(self): + """Verify the designate to neutron-api external-dns relation data.""" + logging.debug('Checking designate:dnsaas relation data...') + unit_name = 'designate/0' + remote_unit_name = 'neutron-api/0' + relation_name = 'dnsaas' + remote_unit = model.get_unit_from_name(remote_unit_name) + remote_ip = remote_unit.public_address + relation = zaza_juju.get_relation_from_unit( + unit_name, + remote_unit_name, + relation_name + ) + # Get private-address in relation + rel_private_ip = relation.get('private-address') + # The private address in relation should match designate-bind/0 address + self.assertEqual(rel_private_ip, remote_ip) + + def test_208_neutron_api_designate_relation(self): + """Verify the neutron-api to designate dnsaas relation data.""" + logging.debug('Checking neutron-api:external-dns relation data...') + unit_name = 'neutron-api/0' + remote_unit_name = 'designate/0' + relation_name = 'external-dns' + remote_unit = model.get_unit_from_name(remote_unit_name) + remote_ip = remote_unit.public_address + relation = zaza_juju.get_relation_from_unit( + unit_name, + remote_unit_name, + relation_name + ) + # Get private-address in relation + rel_private_ip = relation.get('private-address') + # The private address in relation should match designate-bind/0 address + self.assertEqual(rel_private_ip, remote_ip) + + +class BindRelationTests(BaseDesignateTest): + """Designate Bind relations charm tests.""" def test_205_designate_designate_bind_relation(self): """Verify the designate to designate-bind dns-backend relation data.""" @@ -129,6 +303,148 @@ class BindTests(BaseDesignateTest): # The private address in relation should match designate-bind/0 address self.assertEqual(rel_private_ip, remote_ip) + +class ResetAndPauseTests(BaseDesignateTest): + """Designate charm restart and pause tests.""" + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change. + + Change disk format and assert then change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + set_default = {'debug': 'False'} + set_alternate = {'debug': 'True'} + + # Services which are expected to restart upon config change, + # and corresponding config files affected by the change + conf_file = '/etc/designate/designate.conf' + + # Make config change, check for service restarts + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + {'DEFAULT': {'debug': ['False']}}, + {'DEFAULT': {'debug': ['True']}}, + self.designate_svcs) + + def test_910_pause_and_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + with self.pause_resume( + self.designate_svcs, + pgrep_full=False): + logging.info("Testing pause resume") + + +class ServerCreationTest(BaseDesignateTest): + """Designate charm server creation tests.""" + + TEST_NS1_RECORD = 'ns1.amuletexample.com.' + TEST_NS2_RECORD = 'ns2.amuletexample.com.' + + def _get_server_id(self, server_name): + server_id = None + for server in self.designate.servers.list(): + if server.name == server_name: + server_id = server.id + break + return server_id + + def _get_test_server_id(self): + return self._get_server_id(self.TEST_NS2_RECORD) + + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), + reraise=True + ) + def _wait_on_server_gone(self): + logging.debug('Waiting for server to disappear') + return not self._get_test_server_id() + + def test_400_server_creation(self): + """Simple api calls to create domain.""" + # Designate does not allow the last server to be delete so ensure ns1 + # always present + if self.post_xenial_queens: + logging.info('Skipping server creation tests for Queens and above') + return + + if not self._get_server_id(self.TEST_NS1_RECORD): + server = servers.Server(name=self.TEST_NS1_RECORD) + new_server = self.designate.servers.create(server) + self.assertIsNotNone(new_server) + + logging.debug('Checking if server exists before trying to create it') + old_server_id = self._get_test_server_id() + if old_server_id: + logging.debug('Deleting old server') + self.designate.servers.delete(old_server_id) + self._wait_on_server_gone() + + logging.debug('Creating new server') + server = servers.Server(name=self.TEST_NS2_RECORD) + new_server = self.designate.servers.create(server) + self.assertIsNotNone(new_server) + + +class DomainCreationTests(BaseDesignateTest): + """Designate charm domain creation tests.""" + + TEST_DOMAIN = 'amuletexample.com.' + TEST_WWW_RECORD = "www.{}".format(TEST_DOMAIN) + TEST_RECORD = {TEST_WWW_RECORD: '10.0.0.23'} + + def _get_domain_id(self, domain_name): + domain_id = None + for dom in self.zones_list(): + if isinstance(dom, dict): + if dom['name'] == domain_name: + domain_id = dom['name'] + break + else: + if dom.name == domain_name: + domain_id = dom.id + break + return domain_id + + def _get_test_domain_id(self): + return self._get_domain_id(self.TEST_DOMAIN) + + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), + reraise=True + ) + def _wait_on_domain_gone(self): + logging.debug('Waiting for domain to disappear') + if self._get_test_domain_id(): + raise Exception("Domain Exists") + + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), + reraise=True + ) + def _wait_to_resolve_test_record(self): + dns_ip = zaza_juju.get_relation_from_unit( + 'designate/0', + 'designate-bind/0', + 'dns-backend' + ).get('private-address') + + logging.info('Waiting for dns record to propagate @ {}'.format(dns_ip)) + lookup_cmd = [ + 'dig', '+short', '@{}'.format(dns_ip), + self.TEST_WWW_RECORD] + cmd_out = subprocess.check_output( + lookup_cmd, universal_newlines=True).rstrip() + if not self.TEST_RECORD[self.TEST_WWW_RECORD] == cmd_out: + raise Exception("Record Doesn't Exist") + def test_400_domain_creation(self): """Simple api calls to create domain.""" logging.debug('Checking if domain exists before trying to create it') @@ -164,12 +480,7 @@ class BindTests(BaseDesignateTest): _domain_id = new_domain.id self.designate.records.create(_domain_id, _record) - dns_ip = zaza_juju.get_relation_from_unit( - 'designate/0', - 'designate-bind/0', - 'dns-backend' - ).get('private-address') - self._wait_to_resolve_test_record(dns_ip) + self._wait_to_resolve_test_record() logging.debug('Tidy up delete test record') self.zones_delete(_domain_id) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 077e50e..90a928a 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -209,18 +209,15 @@ def get_glance_session_client(session): return GlanceClient('2', session=session) -def get_designate_session_client(session, **kwargs): +def get_designate_session_client(**kwargs): """Return designateclient authenticated by keystone session. - :param session: Keystone session object - :type session: keystoneauth1.session.Session object :param kwargs: Designate Client Arguments :returns: Authenticated designateclient :rtype: DesignateClient """ version = kwargs.pop('version', None) or 2 - return DesignateClient(session=session, - version=str(version), + return DesignateClient(version=str(version), **kwargs) From 06a34cd831b541ab5bde996872e2cd1eaa48836b Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 4 Feb 2020 16:56:40 -0600 Subject: [PATCH 292/898] designate-bind tests passing --- zaza/openstack/charm_tests/designate/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index a96e931..5b6e8e9 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -35,7 +35,7 @@ class BaseTests(BaseDesignateTest): logging.debug('Checking system services on units...') model.block_until_service_status( - self.application_name, + self.lead_unit, self.designate_svcs, 'running', self.model_name, From a86f7affd465a928afc0077765d3f26822a0e668 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 4 Feb 2020 22:30:29 -0600 Subject: [PATCH 293/898] retrieve keystone client through simpler api --- zaza/openstack/charm_tests/designate/__init__.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/__init__.py b/zaza/openstack/charm_tests/designate/__init__.py index 66e2373..22683cc 100644 --- a/zaza/openstack/charm_tests/designate/__init__.py +++ b/zaza/openstack/charm_tests/designate/__init__.py @@ -43,15 +43,13 @@ class BaseDesignateTest(test_utils.OpenStackBaseTest): # Get keystone session cls.post_xenial_queens = os_release() >= os_release('xenial_queens') - keystone_api = 3 if cls.post_xenial_queens else 2 - cls.keystone_session = openstack_utils.get_overcloud_keystone_session() - cls.keystone = openstack_utils.get_keystone_session_client( - cls.keystone_session, keystone_api - ) + overcloud_auth = openstack_utils.get_overcloud_auth() + cls.keystone = openstack_utils.get_keystone_client(overcloud_auth) if cls.post_xenial_queens: + keystone_session = openstack_utils.get_overcloud_keystone_session() cls.designate = openstack_utils.get_designate_session_client( - session=cls.keystone_session + session=keystone_session ) cls.zones_list = cls.designate.zones.list cls.zones_delete = cls.designate.zones.delete From a3f2751fdce5330c6c49f4ad72b067290d4e09fa Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Tue, 4 Feb 2020 23:00:53 -0600 Subject: [PATCH 294/898] Resolve Designate API tests --- zaza/openstack/charm_tests/designate/tests.py | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 5b6e8e9..2992a55 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -48,11 +48,11 @@ class BaseTests(BaseDesignateTest): class ApiTests(BaseDesignateTest): """Designate charm api tests.""" - VALID_URL_TOKENS = { - 'adminURL' - 'publicURL' - 'internalURL' - } + VALID_INTERFACE = [ + 'admin', + 'public', + 'internal' + ] VALID_URL = re.compile( r'^(?:http|ftp)s?://' @@ -67,28 +67,27 @@ class ApiTests(BaseDesignateTest): """Verify that the service catalog endpoint data is valid.""" logging.debug('Checking keystone service catalog data...') actual = self.keystone.service_catalog.get_endpoints() - self.assertIsNotNone(actual['dns'].get('id')) - self.assertEqual(actual['dns'].get('region'), "RegionOne") - for token in self.VALID_URL_TOKENS: - validate = actual['dns'].get(token) - self.assertIsNotNone( - validate, "token {} doesn't exist".format(token)) - self.assertRegexpMatches(validate, self.VALID_URL) + dns_endpoints = actual['dns'] + + for ep in dns_endpoints: + logging.debug(ep) + self.assertIsNotNone(ep.get('id')) + self.assertEqual(ep.get('region'), "RegionOne") + self.assertIn(ep.get('interface'), self.VALID_INTERFACE) + self.assertRegexpMatches(ep.get('url'), self.VALID_URL) def test_114_designate_api_endpoint(self): """Verify the designate api endpoint data.""" logging.debug('Checking designate api endpoint data...') endpoints = self.keystone.endpoints.list() - logging.debug(endpoints) - self.assertIsNotNone(endpoints.get('id')) - self.assertEqual(endpoints.get('region'), "RegionOne") - self.assertIsNotNone(endpoints.get('server_id')) - for token in self.VALID_URL_TOKENS: - validate = endpoints.get(token) - self.assertIsNotNone( - validate, "token {} doesn't exist".format(token)) - self.assertRegexpMatches(validate, self.VALID_URL) + for ep in endpoints: + logging.debug(ep) + self.assertIsNotNone(ep.id) + self.assertEqual(ep.region, "RegionOne") + self.assertIsNotNone(ep.service_id) + self.assertIn(ep.interface, self.VALID_INTERFACE) + self.assertRegexpMatches(ep.url, self.VALID_URL) class KeystoneIdentityRelationTests(BaseDesignateTest): From 1398583e07aab24ceebaa084a6f2878e4b3b278d Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 5 Feb 2020 13:20:55 +0100 Subject: [PATCH 295/898] ovn: Update bridge configuration code After power user feedback we want to change how interfaces are attached to bridges on the OVN charms. Ref: openstack-charmers/charm-layer-ovn#6 --- zaza/openstack/utilities/openstack.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 7e48a98..2c27b7d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -672,7 +672,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, except KeyError: # ovn-dedicated-chassis not in deployment pass - port_config_key = 'interface-bridge-mappings' + port_config_key = 'bridge-interface-mappings' config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) add_dataport_to_netplan = True else: @@ -713,8 +713,6 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, if 'ext-port' in port['name']: if deprecated_extnet_mode: ext_br_macs.append(port['mac_address']) - elif ovn_present(): - ext_br_macs.append('{}:br-ex'.format(port['mac_address'])) else: ext_br_macs.append('br-ex:{}'.format(port['mac_address'])) ext_br_macs.sort() From c3c00308d5b87a057ba79b2d34e8d3c2322753a5 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 5 Feb 2020 08:19:52 -0600 Subject: [PATCH 296/898] successful designate tests --- zaza/openstack/charm_tests/designate/tests.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 2992a55..bcb9b95 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -98,8 +98,8 @@ class KeystoneIdentityRelationTests(BaseDesignateTest): logging.debug('Checking designate to keystone identity-service ' 'relation data...') - unit_name = 'designate/0' - remote_unit_name = 'keystone/0' + unit_name = 'keystone/0' + remote_unit_name = 'designate/0' relation_name = 'identity-service' remote_unit = model.get_unit_from_name(remote_unit_name) remote_ip = remote_unit.public_address @@ -129,8 +129,8 @@ class KeystoneIdentityRelationTests(BaseDesignateTest): """Verify the keystone to designate identity-service relation data.""" logging.debug('Checking keystone:designate identity relation data...') - unit_name = 'keystone/0' - remote_unit_name = 'designate/0' + unit_name = 'designate/0' + remote_unit_name = 'keystone/0' relation_name = 'identity-service' remote_unit = model.get_unit_from_name(remote_unit_name) remote_ip = remote_unit.public_address @@ -211,12 +211,12 @@ class AmqpRelationTests(BaseDesignateTest): ) # Get private-address in relation rel_private_ip = relation.get('private-address') - username = relation.get('username') - vhost = relation.get('vhost') + hostname = relation.get('hostname') + password = relation.get('password') self.assertEqual(rel_private_ip, remote_ip) - self.assertEqual(username, 'designate') - self.assertEqual(vhost, 'openstack') + self.assertEqual(hostname, remote_ip) + self.assertIsNotNone(password) class NeutronApiRelationTests(BaseDesignateTest): @@ -225,8 +225,8 @@ class NeutronApiRelationTests(BaseDesignateTest): def test_207_designate_neutron_api_relation(self): """Verify the designate to neutron-api external-dns relation data.""" logging.debug('Checking designate:dnsaas relation data...') - unit_name = 'designate/0' - remote_unit_name = 'neutron-api/0' + unit_name = 'neutron-api/0' + remote_unit_name = 'designate/0' relation_name = 'dnsaas' remote_unit = model.get_unit_from_name(remote_unit_name) remote_ip = remote_unit.public_address @@ -243,8 +243,8 @@ class NeutronApiRelationTests(BaseDesignateTest): def test_208_neutron_api_designate_relation(self): """Verify the neutron-api to designate dnsaas relation data.""" logging.debug('Checking neutron-api:external-dns relation data...') - unit_name = 'neutron-api/0' - remote_unit_name = 'designate/0' + unit_name = 'designate/0' + remote_unit_name = 'neutron-api/0' relation_name = 'external-dns' remote_unit = model.get_unit_from_name(remote_unit_name) remote_ip = remote_unit.public_address From 30f752bbfc48094ab46dee6dee80baa48e5f56b9 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 5 Feb 2020 08:43:39 -0600 Subject: [PATCH 297/898] repair AmqpRelation test --- zaza/openstack/charm_tests/designate/tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index bcb9b95..7bad77e 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -176,8 +176,8 @@ class AmqpRelationTests(BaseDesignateTest): """Verify the designate to rabbitmq-server amqp relation data.""" logging.debug('Checking designate:amqp rabbitmq relation data...') - unit_name = 'designate/0' - remote_unit_name = 'rabbitmq-server/0' + unit_name = 'rabbitmq-server/0' + remote_unit_name = 'designate/0' relation_name = 'amqp' remote_unit = model.get_unit_from_name(remote_unit_name) remote_ip = remote_unit.public_address @@ -199,8 +199,8 @@ class AmqpRelationTests(BaseDesignateTest): """Verify the rabbitmq-server to designate amqp relation data.""" logging.debug('Checking rabbitmq:amqp designate relation data...') - unit_name = 'rabbitmq-server/0' - remote_unit_name = 'designate/0' + unit_name = 'designate/0' + remote_unit_name = 'rabbitmq-server/0' relation_name = 'amqp' remote_unit = model.get_unit_from_name(remote_unit_name) remote_ip = remote_unit.public_address From e444f51b49d1bfd3379c438504a9f65c727bd837 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 5 Feb 2020 09:59:25 -0600 Subject: [PATCH 298/898] endpoint URLs different before and after xenial_queens --- zaza/openstack/charm_tests/designate/tests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 7bad77e..732c354 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -86,8 +86,13 @@ class ApiTests(BaseDesignateTest): self.assertIsNotNone(ep.id) self.assertEqual(ep.region, "RegionOne") self.assertIsNotNone(ep.service_id) - self.assertIn(ep.interface, self.VALID_INTERFACE) - self.assertRegexpMatches(ep.url, self.VALID_URL) + if self.post_xenial_queens: + self.assertIn(ep.interface, self.VALID_INTERFACE) + self.assertRegexpMatches(ep.url, self.VALID_URL) + else: + self.assertRegexpMatches(ep.adminURL, self.VALID_URL) + self.assertRegexpMatches(ep.internalURL, self.VALID_URL) + self.assertRegexpMatches(ep.publicURL, self.VALID_URL) class KeystoneIdentityRelationTests(BaseDesignateTest): From 89b5b585bc25d5aada74b05eda416ef80a3e1cc5 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 5 Feb 2020 10:30:58 -0600 Subject: [PATCH 299/898] Fix issue with service endpoint tests --- zaza/openstack/charm_tests/designate/tests.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 732c354..3264cd7 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -73,8 +73,13 @@ class ApiTests(BaseDesignateTest): logging.debug(ep) self.assertIsNotNone(ep.get('id')) self.assertEqual(ep.get('region'), "RegionOne") - self.assertIn(ep.get('interface'), self.VALID_INTERFACE) - self.assertRegexpMatches(ep.get('url'), self.VALID_URL) + if self.post_xenial_queens: + self.assertIn(ep.get('interface'), self.VALID_INTERFACE) + self.assertRegexpMatches(ep.get('url'), self.VALID_URL) + else: + self.assertRegexpMatches(ep.get('adminURL'), self.VALID_URL) + self.assertRegexpMatches(ep.get('internalURL'), self.VALID_URL) + self.assertRegexpMatches(ep.get('publicURL'), self.VALID_URL) def test_114_designate_api_endpoint(self): """Verify the designate api endpoint data.""" @@ -86,13 +91,8 @@ class ApiTests(BaseDesignateTest): self.assertIsNotNone(ep.id) self.assertEqual(ep.region, "RegionOne") self.assertIsNotNone(ep.service_id) - if self.post_xenial_queens: - self.assertIn(ep.interface, self.VALID_INTERFACE) - self.assertRegexpMatches(ep.url, self.VALID_URL) - else: - self.assertRegexpMatches(ep.adminURL, self.VALID_URL) - self.assertRegexpMatches(ep.internalURL, self.VALID_URL) - self.assertRegexpMatches(ep.publicURL, self.VALID_URL) + self.assertIn(ep.interface, self.VALID_INTERFACE) + self.assertRegexpMatches(ep.url, self.VALID_URL) class KeystoneIdentityRelationTests(BaseDesignateTest): From 6672fe934d6559dab9a0e1a724a152df150855eb Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 24 Jan 2020 16:21:36 -0800 Subject: [PATCH 300/898] Test ovs-use-veth setting Validate settings for existing and configured ovs-use-veth settings. Verify the charm goes into a blocked state if they conflict. Partial-Bug: #1831935 --- zaza/openstack/charm_tests/neutron/tests.py | 89 ++++++++++++++++----- 1 file changed, 69 insertions(+), 20 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 02450e5..694b1fb 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -17,6 +17,7 @@ """Encapsulating `neutron-openvswitch` testing.""" +import copy import logging import tenacity import unittest @@ -29,21 +30,79 @@ import zaza.openstack.configure.guest as guest import zaza.openstack.utilities.openstack as openstack_utils -class NeutronGatewayTest(test_utils.OpenStackBaseTest): +class NeutronPluginApiSharedTests(test_utils.OpenStackBaseTest): + """Shared tests for Neutron Plugin API Charms.""" + + def setUpClass(cls): + """Run class setup for running Neutron Openvswitch tests.""" + super(NeutronPluginApiSharedTests, cls).setUpClass() + + cls.current_os_release = openstack_utils.get_os_release() + cls.bionic_stein = openstack_utils.get_os_release('bionic_stein') + cls.trusty_mitaka = openstack_utils.get_os_release('trusty_mitaka') + + if cls.current_os_release >= cls.bionic_stein: + cls.pgrep_full = True + else: + cls.pgrep_full = False + + def test_211_ovs_use_veth(self): + """Verify proper handling of ovs-use-veth setting.""" + conf_file = "/etc/neutron/dhcp_agent.ini" + expected = {"DEFAULT": {"ovs_use_veth": ["False"]}} + test_config = zaza.charm_lifecycle.utils.get_charm_config(fatal=False) + states = test_config.get("target_deploy_status", {}) + alt_states = copy.deepcopy(states) + alt_states[self.application_name] = { + "workload-status": "blocked", + "workload-status-message": + "Mismatched existing and configured ovs-use-veth. See log."} + + if "neutron-openvswitch" in self.application_name: + logging.info("Turning on DHCP and metadata") + zaza.model.set_application_config( + self.application_name, + {"enable-local-dhcp-and-metadata": "True"}) + zaza.model.wait_for_application_states(states=states) + + logging.info("Check for expected default ovs-use-veth setting of " + "False") + zaza.model.block_until_oslo_config_entries_match( + self.application_name, + conf_file, + expected, + ) + logging.info("Setting conflicting ovs-use-veth to True") + zaza.model.set_application_config( + self.application_name, + {"ovs-use-veth": "True"}) + logging.info("Wait to go into a blocked workload status") + zaza.model.wait_for_application_states(states=alt_states) + # Check the value stayed the same + logging.info("Check that the value of ovs-use-veth setting " + "remained False") + zaza.model.block_until_oslo_config_entries_match( + self.application_name, + conf_file, + expected, + ) + logging.info("Setting ovs-use-veth to match existing.") + zaza.model.set_application_config( + self.application_name, + {"ovs-use-veth": "False"}) + logging.info("Wait to go into unit ready workload status") + zaza.model.wait_for_application_states(states=states) + + +class NeutronGatewayTest(NeutronPluginApiSharedTests): """Test basic Neutron Gateway Charm functionality.""" @classmethod def setUpClass(cls): """Run class setup for running Neutron Gateway tests.""" - super(NeutronGatewayTest, cls).setUpClass() - cls.current_os_release = openstack_utils.get_os_release() + super(NeutronGatewayTest, cls).setUpClass(cls) cls.services = cls._get_services() - bionic_stein = openstack_utils.get_os_release('bionic_stein') - - cls.pgrep_full = (True if cls.current_os_release >= bionic_stein - else False) - def test_900_restart_on_config_change(self): """Checking restart happens on config change. @@ -328,28 +387,18 @@ class SecurityTest(test_utils.OpenStackBaseTest): expected_to_pass=expected_to_pass) -class NeutronOpenvSwitchTest(test_utils.OpenStackBaseTest): +class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): """Test basic Neutron Openvswitch Charm functionality.""" @classmethod def setUpClass(cls): """Run class setup for running Neutron Openvswitch tests.""" - super(NeutronOpenvSwitchTest, cls).setUpClass() - - cls.current_os_release = openstack_utils.get_os_release() + super(NeutronOpenvSwitchTest, cls).setUpClass(cls) cls.compute_unit = zaza.model.get_units('nova-compute')[0] cls.neutron_api_unit = zaza.model.get_units('neutron-api')[0] cls.n_ovs_unit = zaza.model.get_units('neutron-openvswitch')[0] - cls.bionic_stein = openstack_utils.get_os_release('bionic_stein') - cls.trusty_mitaka = openstack_utils.get_os_release('trusty_mitaka') - - if cls.current_os_release >= cls.bionic_stein: - cls.pgrep_full = True - else: - cls.pgrep_full = False - # set up client cls.neutron_client = ( openstack_utils.get_neutron_session_client(cls.keystone_session)) From 9f67c8e032b0ae32cd601f951b215bddfa42c296 Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 29 Jan 2020 16:19:56 -0800 Subject: [PATCH 301/898] Do not test Trusty --- zaza/openstack/charm_tests/neutron/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 694b1fb..6161c88 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -48,6 +48,13 @@ class NeutronPluginApiSharedTests(test_utils.OpenStackBaseTest): def test_211_ovs_use_veth(self): """Verify proper handling of ovs-use-veth setting.""" + current_release = openstack_utils.get_os_release() + xenial_mitaka = openstack_utils.get_os_release('xenial_mitaka') + if current_release < xenial_mitaka: + logging.info( + "Skipping OVS use veth test. ovs_use_veth is always True on " + "Trusty.") + return conf_file = "/etc/neutron/dhcp_agent.ini" expected = {"DEFAULT": {"ovs_use_veth": ["False"]}} test_config = zaza.charm_lifecycle.utils.get_charm_config(fatal=False) From 437aa4ab112c4e267d1d976be3f4b6dcdc42817d Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 5 Feb 2020 21:54:25 -0600 Subject: [PATCH 302/898] simply and reduce tests --- .../charm_tests/designate/__init__.py | 2 +- zaza/openstack/charm_tests/designate/tests.py | 311 +----------------- 2 files changed, 10 insertions(+), 303 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/__init__.py b/zaza/openstack/charm_tests/designate/__init__.py index 22683cc..cc68938 100644 --- a/zaza/openstack/charm_tests/designate/__init__.py +++ b/zaza/openstack/charm_tests/designate/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2018 Canonical Ltd. +# Copyright 2020 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 3264cd7..0a7d520 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -1,4 +1,4 @@ -# Copyright 2018 Canonical Ltd. +# Copyright 2020 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,297 +20,19 @@ import subprocess import designateclient.v1.domains as domains import designateclient.v1.records as records import designateclient.v1.servers as servers -import zaza.model as model import zaza.openstack.utilities.juju as zaza_juju -from zaza.openstack.charm_tests.designate import ( - BaseDesignateTest, -) +from zaza.openstack.charm_tests.designate import BaseDesignateTest -class BaseTests(BaseDesignateTest): - """Base Designate charm tests.""" - - def test_100_services(self): - """Verify expected services are running.""" - logging.debug('Checking system services on units...') - - model.block_until_service_status( - self.lead_unit, - self.designate_svcs, - 'running', - self.model_name, - timeout=30 - ) - - logging.debug('OK') - - -class ApiTests(BaseDesignateTest): - """Designate charm api tests.""" - - VALID_INTERFACE = [ - 'admin', - 'public', - 'internal' - ] - - VALID_URL = re.compile( - r'^(?:http|ftp)s?://' - r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa - r'localhost|' - r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' - r'(?::\d+)?' - r'(?:/?|[/?]\S+)$', - re.IGNORECASE) - - def test_110_service_catalog(self): - """Verify that the service catalog endpoint data is valid.""" - logging.debug('Checking keystone service catalog data...') - actual = self.keystone.service_catalog.get_endpoints() - dns_endpoints = actual['dns'] - - for ep in dns_endpoints: - logging.debug(ep) - self.assertIsNotNone(ep.get('id')) - self.assertEqual(ep.get('region'), "RegionOne") - if self.post_xenial_queens: - self.assertIn(ep.get('interface'), self.VALID_INTERFACE) - self.assertRegexpMatches(ep.get('url'), self.VALID_URL) - else: - self.assertRegexpMatches(ep.get('adminURL'), self.VALID_URL) - self.assertRegexpMatches(ep.get('internalURL'), self.VALID_URL) - self.assertRegexpMatches(ep.get('publicURL'), self.VALID_URL) - - def test_114_designate_api_endpoint(self): - """Verify the designate api endpoint data.""" - logging.debug('Checking designate api endpoint data...') - endpoints = self.keystone.endpoints.list() - - for ep in endpoints: - logging.debug(ep) - self.assertIsNotNone(ep.id) - self.assertEqual(ep.region, "RegionOne") - self.assertIsNotNone(ep.service_id) - self.assertIn(ep.interface, self.VALID_INTERFACE) - self.assertRegexpMatches(ep.url, self.VALID_URL) - - -class KeystoneIdentityRelationTests(BaseDesignateTest): - """Designate Keystone identity relations charm tests.""" - - def test_200_designate_identity_relation(self): - """Verify the designate to keystone identity-service relation data.""" - logging.debug('Checking designate to keystone identity-service ' - 'relation data...') - - unit_name = 'keystone/0' - remote_unit_name = 'designate/0' - relation_name = 'identity-service' - remote_unit = model.get_unit_from_name(remote_unit_name) - remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( - unit_name, - remote_unit_name, - relation_name - ) - - designate_endpoint = "http://{}:9001".format(remote_ip) - expected = { - 'admin_url': designate_endpoint, - 'internal_url': designate_endpoint, - 'private-address': remote_ip, - 'public_url': designate_endpoint, - 'region': 'RegionOne', - 'service': 'designate', - } - - for token, value in expected.items(): - r_val = relation.get(token) - self.assertEqual( - r_val, value, "token({}) doesn't match".format(token) - ) - - def test_201_keystone_designate_identity_relation(self): - """Verify the keystone to designate identity-service relation data.""" - logging.debug('Checking keystone:designate identity relation data...') - - unit_name = 'designate/0' - remote_unit_name = 'keystone/0' - relation_name = 'identity-service' - remote_unit = model.get_unit_from_name(remote_unit_name) - remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( - unit_name, - remote_unit_name, - relation_name - ) - expected = { - 'admin_token': 'ubuntutesting', - 'auth_host': remote_ip, - 'auth_port': "35357", - 'auth_protocol': 'http', - 'private-address': remote_ip, - 'service_host': remote_ip, - 'service_port': "5000", - 'service_protocol': 'http', - 'service_tenant': 'services', - 'service_username': 'designate', - } - - for token, value in expected.items(): - r_val = relation.get(token) - self.assertEqual( - r_val, value, "token({}) doesn't match".format(token) - ) - - self.assertIsNotNone( - relation.get('service_password'), - "service_password missing" - ) - - self.assertIsNotNone( - relation.get('service_tenant_id'), - "service_tenant_id missing" - ) - - -class AmqpRelationTests(BaseDesignateTest): - """Designate Rabbit MQ relations charm tests.""" - - def test_203_designate_amqp_relation(self): - """Verify the designate to rabbitmq-server amqp relation data.""" - logging.debug('Checking designate:amqp rabbitmq relation data...') - - unit_name = 'rabbitmq-server/0' - remote_unit_name = 'designate/0' - relation_name = 'amqp' - remote_unit = model.get_unit_from_name(remote_unit_name) - remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( - unit_name, - remote_unit_name, - relation_name - ) - # Get private-address in relation - rel_private_ip = relation.get('private-address') - username = relation.get('username') - vhost = relation.get('vhost') - - self.assertEqual(rel_private_ip, remote_ip) - self.assertEqual(username, 'designate') - self.assertEqual(vhost, 'openstack') - - def test_204_amqp_designate_relation(self): - """Verify the rabbitmq-server to designate amqp relation data.""" - logging.debug('Checking rabbitmq:amqp designate relation data...') - - unit_name = 'designate/0' - remote_unit_name = 'rabbitmq-server/0' - relation_name = 'amqp' - remote_unit = model.get_unit_from_name(remote_unit_name) - remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( - unit_name, - remote_unit_name, - relation_name - ) - # Get private-address in relation - rel_private_ip = relation.get('private-address') - hostname = relation.get('hostname') - password = relation.get('password') - - self.assertEqual(rel_private_ip, remote_ip) - self.assertEqual(hostname, remote_ip) - self.assertIsNotNone(password) - - -class NeutronApiRelationTests(BaseDesignateTest): - """Designate Neutron API relations charm tests.""" - - def test_207_designate_neutron_api_relation(self): - """Verify the designate to neutron-api external-dns relation data.""" - logging.debug('Checking designate:dnsaas relation data...') - unit_name = 'neutron-api/0' - remote_unit_name = 'designate/0' - relation_name = 'dnsaas' - remote_unit = model.get_unit_from_name(remote_unit_name) - remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( - unit_name, - remote_unit_name, - relation_name - ) - # Get private-address in relation - rel_private_ip = relation.get('private-address') - # The private address in relation should match designate-bind/0 address - self.assertEqual(rel_private_ip, remote_ip) - - def test_208_neutron_api_designate_relation(self): - """Verify the neutron-api to designate dnsaas relation data.""" - logging.debug('Checking neutron-api:external-dns relation data...') - unit_name = 'designate/0' - remote_unit_name = 'neutron-api/0' - relation_name = 'external-dns' - remote_unit = model.get_unit_from_name(remote_unit_name) - remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( - unit_name, - remote_unit_name, - relation_name - ) - # Get private-address in relation - rel_private_ip = relation.get('private-address') - # The private address in relation should match designate-bind/0 address - self.assertEqual(rel_private_ip, remote_ip) - - -class BindRelationTests(BaseDesignateTest): - """Designate Bind relations charm tests.""" - - def test_205_designate_designate_bind_relation(self): - """Verify the designate to designate-bind dns-backend relation data.""" - logging.debug('Checking designate:designate-bind dns-backend relation' - 'data...') - - unit_name = 'designate/0' - remote_unit_name = 'designate-bind/0' - relation_name = 'dns-backend' - remote_unit = model.get_unit_from_name(remote_unit_name) - remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( - unit_name, - remote_unit_name, - relation_name - ) - # Get private-address in relation - rel_private_ip = relation.get('private-address') - # The private address in relation should match designate-bind/0 address - self.assertEqual(rel_private_ip, remote_ip) - - def test_206_designate_bind_designate_relation(self): - """Verify the designate_bind to designate dns-backend relation data.""" - logging.debug('Checking designate-bind:designate dns-backend relation' - 'data...') - - unit_name = 'designate-bind/0' - remote_unit_name = 'designate/0' - relation_name = 'dns-backend' - remote_unit = model.get_unit_from_name(remote_unit_name) - remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( - unit_name, - remote_unit_name, - relation_name - ) - # Get private-address in relation - rel_private_ip = relation.get('private-address') - # The private address in relation should match designate-bind/0 address - self.assertEqual(rel_private_ip, remote_ip) - - -class ResetAndPauseTests(BaseDesignateTest): +class DesignateTests(BaseDesignateTest): """Designate charm restart and pause tests.""" + TEST_DOMAIN = 'amuletexample.com.' + TEST_NS1_RECORD = 'ns1.{}'.format(TEST_DOMAIN) + TEST_NS2_RECORD = 'ns2.{}'.format(TEST_DOMAIN) + TEST_WWW_RECORD = "www.{}".format(TEST_DOMAIN) + TEST_RECORD = {TEST_WWW_RECORD: '10.0.0.23'} + def test_900_restart_on_config_change(self): """Checking restart happens on config change. @@ -345,13 +67,6 @@ class ResetAndPauseTests(BaseDesignateTest): pgrep_full=False): logging.info("Testing pause resume") - -class ServerCreationTest(BaseDesignateTest): - """Designate charm server creation tests.""" - - TEST_NS1_RECORD = 'ns1.amuletexample.com.' - TEST_NS2_RECORD = 'ns2.amuletexample.com.' - def _get_server_id(self, server_name): server_id = None for server in self.designate.servers.list(): @@ -396,14 +111,6 @@ class ServerCreationTest(BaseDesignateTest): new_server = self.designate.servers.create(server) self.assertIsNotNone(new_server) - -class DomainCreationTests(BaseDesignateTest): - """Designate charm domain creation tests.""" - - TEST_DOMAIN = 'amuletexample.com.' - TEST_WWW_RECORD = "www.{}".format(TEST_DOMAIN) - TEST_RECORD = {TEST_WWW_RECORD: '10.0.0.23'} - def _get_domain_id(self, domain_name): domain_id = None for dom in self.zones_list(): From 89d72667347eff08befa1ae8759114ded3f4f5e7 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Wed, 5 Feb 2020 22:09:38 -0600 Subject: [PATCH 303/898] Remove regular expression import --- zaza/openstack/charm_tests/designate/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 0a7d520..5155fe8 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -13,7 +13,6 @@ # limitations under the License. """Encapsulate designate testing.""" -import re import logging import tenacity import subprocess From cdc0008bd347dff86928edba0c750ecaa067b1b0 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 6 Feb 2020 08:39:21 -0600 Subject: [PATCH 304/898] incorporate team comments --- .../charm_tests/designate/__init__.py | 57 ----------------- zaza/openstack/charm_tests/designate/tests.py | 63 ++++++++++++++++++- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/__init__.py b/zaza/openstack/charm_tests/designate/__init__.py index cc68938..78c15a3 100644 --- a/zaza/openstack/charm_tests/designate/__init__.py +++ b/zaza/openstack/charm_tests/designate/__init__.py @@ -13,60 +13,3 @@ # limitations under the License. """Collection of code for setting up and testing designate.""" -import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.utilities.openstack as openstack_utils - - -class BaseDesignateTest(test_utils.OpenStackBaseTest): - """Base for Designate charm tests.""" - - @classmethod - def setUpClass(cls, application_name=None, model_alias=None): - """Run class setup for running Designate charm operation tests.""" - application_name = application_name or "designate" - model_alias = model_alias or "" - super(BaseDesignateTest, cls).setUpClass(application_name, model_alias) - os_release = openstack_utils.get_os_release - - if os_release() >= os_release('bionic_rocky'): - cls.designate_svcs = [ - 'designate-agent', 'designate-api', 'designate-central', - 'designate-mdns', 'designate-worker', 'designate-sink', - 'designate-producer', - ] - else: - cls.designate_svcs = [ - 'designate-agent', 'designate-api', 'designate-central', - 'designate-mdns', 'designate-pool-manager', 'designate-sink', - 'designate-zone-manager', - ] - - # Get keystone session - cls.post_xenial_queens = os_release() >= os_release('xenial_queens') - overcloud_auth = openstack_utils.get_overcloud_auth() - cls.keystone = openstack_utils.get_keystone_client(overcloud_auth) - - if cls.post_xenial_queens: - keystone_session = openstack_utils.get_overcloud_keystone_session() - cls.designate = openstack_utils.get_designate_session_client( - session=keystone_session - ) - cls.zones_list = cls.designate.zones.list - cls.zones_delete = cls.designate.zones.delete - else: - # Authenticate admin with designate endpoint - designate_ep = cls.keystone.service_catalog.url_for( - service_type='dns', - interface='publicURL') - keystone_ep = cls.keystone.service_catalog.url_for( - service_type='identity', - interface='publicURL') - cls.designate = openstack_utils.get_designate_session_client( - version=1, - auth_url=keystone_ep, - username="admin", - password="openstack", - tenant_name="admin", - endpoint=designate_ep) - cls.zones_list = cls.designate.domains.list - cls.zones_delete = cls.designate.domains.delete diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 5155fe8..babfe9e 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -20,7 +20,63 @@ import designateclient.v1.domains as domains import designateclient.v1.records as records import designateclient.v1.servers as servers import zaza.openstack.utilities.juju as zaza_juju -from zaza.openstack.charm_tests.designate import BaseDesignateTest +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +class BaseDesignateTest(test_utils.OpenStackBaseTest): + """Base for Designate charm tests.""" + + @classmethod + def setUpClass(cls, application_name=None, model_alias=None): + """Run class setup for running Designate charm operation tests.""" + application_name = application_name or "designate" + model_alias = model_alias or "" + super(BaseDesignateTest, cls).setUpClass(application_name, model_alias) + os_release = openstack_utils.get_os_release + + if os_release() >= os_release('bionic_rocky'): + cls.designate_svcs = [ + 'designate-agent', 'designate-api', 'designate-central', + 'designate-mdns', 'designate-worker', 'designate-sink', + 'designate-producer', + ] + else: + cls.designate_svcs = [ + 'designate-agent', 'designate-api', 'designate-central', + 'designate-mdns', 'designate-pool-manager', 'designate-sink', + 'designate-zone-manager', + ] + + # Get keystone session + cls.post_xenial_queens = os_release() >= os_release('xenial_queens') + overcloud_auth = openstack_utils.get_overcloud_auth() + keystone = openstack_utils.get_keystone_client(overcloud_auth) + + if cls.post_xenial_queens: + keystone_session = openstack_utils.get_overcloud_keystone_session() + cls.designate = openstack_utils.get_designate_session_client( + session=keystone_session + ) + cls.zones_list = cls.designate.zones.list + cls.zones_delete = cls.designate.zones.delete + else: + # Authenticate admin with designate endpoint + designate_ep = keystone.service_catalog.url_for( + service_type='dns', + interface='publicURL') + keystone_ep = keystone.service_catalog.url_for( + service_type='identity', + interface='publicURL') + cls.designate = openstack_utils.get_designate_session_client( + version=1, + auth_url=keystone_ep, + username="admin", + password="openstack", + tenant_name="admin", + endpoint=designate_ep) + cls.zones_list = cls.designate.domains.list + cls.zones_delete = cls.designate.domains.delete class DesignateTests(BaseDesignateTest): @@ -87,8 +143,8 @@ class DesignateTests(BaseDesignateTest): def test_400_server_creation(self): """Simple api calls to create domain.""" - # Designate does not allow the last server to be delete so ensure ns1 - # always present + # Designate does not allow the last server to be deleted so ensure + # that ns1 is always present if self.post_xenial_queens: logging.info('Skipping server creation tests for Queens and above') return @@ -194,4 +250,5 @@ class DesignateTests(BaseDesignateTest): logging.debug('Tidy up delete test record') self.zones_delete(_domain_id) + self._wait_on_domain_gone() logging.debug('OK') From 8794298bd058c8bd9d3a5e10e31c1021fde72a03 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 6 Feb 2020 10:19:28 -0600 Subject: [PATCH 305/898] properly wait for zones to delete after successful tests --- zaza/openstack/charm_tests/designate/tests.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index babfe9e..029fdec 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -191,6 +191,25 @@ class DesignateTests(BaseDesignateTest): if self._get_test_domain_id(): raise Exception("Domain Exists") + def _get_test_zone_id(self): + zone_id = None + for zone in self.designate.zones.list(): + if zone['name'] == self.TEST_DOMAIN: + zone_id = zone['id'] + break + return zone_id + + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), + reraise=True + ) + def _wait_on_zone_gone(self): + self._wait_on_domain_gone() + if self.post_xenial_queens: + logging.debug('Waiting for zone to disappear') + if self._get_test_zone_id(): + raise Exception("Zone Exists") + @tenacity.retry( wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), reraise=True @@ -218,7 +237,7 @@ class DesignateTests(BaseDesignateTest): if old_dom_id: logging.debug('Deleting old domain') self.zones_delete(old_dom_id) - self._wait_on_domain_gone() + self._wait_on_zone_gone() logging.debug('Creating new domain') domain = domains.Domain( @@ -250,5 +269,5 @@ class DesignateTests(BaseDesignateTest): logging.debug('Tidy up delete test record') self.zones_delete(_domain_id) - self._wait_on_domain_gone() + self._wait_on_zone_gone() logging.debug('OK') From 827032b1c6ee00b98bff4fcc6506f3ebbb205990 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 6 Feb 2020 12:06:52 -0800 Subject: [PATCH 306/898] Standardize preferred-api-version as an int --- zaza/openstack/charm_tests/keystone/__init__.py | 9 ++++++--- zaza/openstack/charm_tests/keystone/setup.py | 2 +- zaza/openstack/charm_tests/keystone/tests.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/__init__.py b/zaza/openstack/charm_tests/keystone/__init__.py index 432eac2..6c4cca6 100644 --- a/zaza/openstack/charm_tests/keystone/__init__.py +++ b/zaza/openstack/charm_tests/keystone/__init__.py @@ -34,6 +34,9 @@ class BaseKeystoneTest(test_utils.OpenStackBaseTest): def setUpClass(cls, application_name=None): """Run class setup for running Keystone charm operation tests.""" super(BaseKeystoneTest, cls).setUpClass(application_name='keystone') + # Standardize v2 and v3 as ints + cls.api_v2 = 2 + cls.api_v3 = 3 # Check if we are related to Vault TLS certificates cls.tls_rid = zaza.model.get_relation_id( 'keystone', 'vault', remote_interface_name='certificates') @@ -51,9 +54,9 @@ class BaseKeystoneTest(test_utils.OpenStackBaseTest): cls.keystone_ips.append(cls.vip) if (openstack_utils.get_os_release() < openstack_utils.get_os_release('xenial_queens')): - cls.default_api_version = '2' + cls.default_api_version = cls.api_v2 else: - cls.default_api_version = '3' + cls.default_api_version = cls.api_v3 cls.admin_keystone_session = ( openstack_utils.get_overcloud_keystone_session()) cls.admin_keystone_client = ( @@ -66,6 +69,6 @@ class BaseKeystoneTest(test_utils.OpenStackBaseTest): """Set the preferred keystone api to v3 within called context.""" with self.config_change( {'preferred-api-version': self.default_api_version}, - {'preferred-api-version': '3'}, + {'preferred-api-version': self.api_v3}, application_name="keystone"): yield diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index c2cc4ac..688e3cd 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -110,7 +110,7 @@ def add_demo_user(): # under test other than keystone. with _singleton.config_change( {'preferred-api-version': _singleton.default_api_version}, - {'preferred-api-version': '3'}, application_name="keystone"): + {'preferred-api-version': 3}, application_name="keystone"): _v3() else: # create only V3 user diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index eda5441..e057428 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -321,7 +321,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): } with self.config_change( {'preferred-api-version': self.default_api_version}, - {'preferred-api-version': '3'}, + {'preferred-api-version': self.api_v3}, application_name="keystone"): for ip in self.keystone_ips: openrc.update( From 1919000fa4ea465ebee66b797465c65f2c253f6a Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 6 Feb 2020 20:13:34 +0000 Subject: [PATCH 307/898] Fix the vault pause/resume test The pause/resume test paused the lead unit, but then checked the first unit. In an HA scenario, the lead unit may not be the first unit. This PR changes the test to check the lead unit after the pause/resume test. Depends On: https://github.com/openstack-charmers/zaza/pull/327 --- zaza/openstack/charm_tests/vault/tests.py | 4 ++- zaza/openstack/charm_tests/vault/utils.py | 31 +++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 4719c4b..5c93046 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -248,9 +248,11 @@ class VaultTest(BaseVaultTest): if 'pause' not in vault_actions or 'resume' not in vault_actions: raise unittest.SkipTest("The version of charm-vault tested does " "not have pause/resume actions") + # this pauses and resumes the LEAD unit with self.pause_resume(['vault']): logging.info("Testing pause resume") - self.assertTrue(self.clients[0].hvac_client.seal_status['sealed']) + lead_client = vault_utils.extract_lead_unit_client(self.clients) + self.assertTrue(lead_client.hvac_client.seal_status['sealed']) if __name__ == '__main__': diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index b6f4cf5..c05fcf6 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -173,6 +173,37 @@ def get_clients(units=None, cacert=None): return clients +def extract_lead_unit_client( + clients=None, application_name='vault', cacert=None): + """Find the lead unit client. + + This returns the lead unit client from a list of clients. If no clients + are passed, then the clients are resolved using the cacert (if needed) and + the application_name. The client is then matched to the lead unit. If + clients are passed, but no leader is found in them, then the function + raises a RuntimeError. + + :param clients: List of CharmVaultClient + :type clients: List[CharmVaultClient] + :param application_name: The application name + :type application_name: str + :param cacert: Path to CA cert used for vaults api cert. + :type cacert: str + :returns: The leader client + :rtype: CharmVaultClient + :raises: RuntimeError if the lead unit cannot be found + """ + if clients is None: + units = zaza.model.get_app_ips('vault') + clients = get_clients(units, cacert) + lead_ip = zaza.model.get_lead_unit_ip(application_name) + for client in clients: + if client.addr == lead_ip: + return client + raise RuntimeError("Leader client not found for application: {}" + .format(application_name)) + + def is_initialized(client): """Check if vault is initialized. From c5e020d11b81367d93e50983cfbaf4363b541a25 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 7 Feb 2020 13:41:01 -0600 Subject: [PATCH 308/898] Refactor tests to provide readibility and sanity --- zaza/openstack/charm_tests/designate/tests.py | 149 ++++++++---------- 1 file changed, 65 insertions(+), 84 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 029fdec..47db587 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -53,13 +53,14 @@ class BaseDesignateTest(test_utils.OpenStackBaseTest): overcloud_auth = openstack_utils.get_overcloud_auth() keystone = openstack_utils.get_keystone_client(overcloud_auth) + keystone_session = openstack_utils.get_overcloud_keystone_session() if cls.post_xenial_queens: - keystone_session = openstack_utils.get_overcloud_keystone_session() cls.designate = openstack_utils.get_designate_session_client( session=keystone_session ) - cls.zones_list = cls.designate.zones.list - cls.zones_delete = cls.designate.zones.delete + cls.domain_list = cls.designate.zones.list + cls.domain_delete = cls.designate.zones.delete + cls.domain_create = cls.designate.zones.create else: # Authenticate admin with designate endpoint designate_ep = keystone.service_catalog.url_for( @@ -71,12 +72,15 @@ class BaseDesignateTest(test_utils.OpenStackBaseTest): cls.designate = openstack_utils.get_designate_session_client( version=1, auth_url=keystone_ep, - username="admin", - password="openstack", + token=keystone_session.get_token(), tenant_name="admin", endpoint=designate_ep) - cls.zones_list = cls.designate.domains.list - cls.zones_delete = cls.designate.domains.delete + cls.domain_list = cls.designate.domains.list + cls.domain_delete = cls.designate.domains.delete + cls.domain_create = cls.designate.domains.create + cls.server_list = cls.designate.servers.list + cls.server_create = cls.designate.servers.create + cls.server_delete = cls.designate.servers.delete class DesignateTests(BaseDesignateTest): @@ -122,93 +126,72 @@ class DesignateTests(BaseDesignateTest): pgrep_full=False): logging.info("Testing pause resume") - def _get_server_id(self, server_name): - server_id = None - for server in self.designate.servers.list(): - if server.name == server_name: - server_id = server.id - break - return server_id + def _get_server_id(self, server_name=None, server_id=None): + for srv in self.server_list(): + if isinstance(srv, dict): + if srv['id'] == server_id or srv['name'] == server_name: + return srv['id'] + elif srv.name == server_name or srv.id == server_id: + return srv.id + return None - def _get_test_server_id(self): - return self._get_server_id(self.TEST_NS2_RECORD) - - @tenacity.retry( - wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), - reraise=True - ) - def _wait_on_server_gone(self): - logging.debug('Waiting for server to disappear') - return not self._get_test_server_id() + def _wait_on_server_gone(self, server_id): + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), + reraise=True + ) + def wait(): + logging.debug('Waiting for server %s to disappear', server_id) + if self._get_server_id(server_id=server_id): + raise Exception("Server Exists") + self.server_delete(server_id) + return wait() def test_400_server_creation(self): - """Simple api calls to create domain.""" + """Simple api calls to create a server.""" # Designate does not allow the last server to be deleted so ensure # that ns1 is always present if self.post_xenial_queens: logging.info('Skipping server creation tests for Queens and above') return - if not self._get_server_id(self.TEST_NS1_RECORD): + if not self._get_server_id(server_name=self.TEST_NS1_RECORD): server = servers.Server(name=self.TEST_NS1_RECORD) - new_server = self.designate.servers.create(server) + new_server = self.server_create(server) self.assertIsNotNone(new_server) logging.debug('Checking if server exists before trying to create it') - old_server_id = self._get_test_server_id() + old_server_id = self._get_server_id(server_name=self.TEST_NS2_RECORD) if old_server_id: logging.debug('Deleting old server') - self.designate.servers.delete(old_server_id) - self._wait_on_server_gone() + self._wait_on_server_gone(old_server_id) logging.debug('Creating new server') server = servers.Server(name=self.TEST_NS2_RECORD) - new_server = self.designate.servers.create(server) - self.assertIsNotNone(new_server) + new_server = self.server_create(server) + self.assertIsNotNone(new_server, "Failed to Create Server") + self._wait_on_server_gone(self._get_server_id(self.TEST_NS2_RECORD)) - def _get_domain_id(self, domain_name): - domain_id = None - for dom in self.zones_list(): + def _get_domain_id(self, domain_name=None, domain_id=None): + for dom in self.domain_list(): if isinstance(dom, dict): - if dom['name'] == domain_name: - domain_id = dom['name'] - break - else: - if dom.name == domain_name: - domain_id = dom.id - break - return domain_id + if dom['id'] == domain_id or dom['name'] == domain_name: + return dom['id'] + elif dom.id == domain_id or dom.name == domain_name: + return dom.id + return None - def _get_test_domain_id(self): - return self._get_domain_id(self.TEST_DOMAIN) - - @tenacity.retry( - wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), - reraise=True - ) - def _wait_on_domain_gone(self): - logging.debug('Waiting for domain to disappear') - if self._get_test_domain_id(): - raise Exception("Domain Exists") - - def _get_test_zone_id(self): - zone_id = None - for zone in self.designate.zones.list(): - if zone['name'] == self.TEST_DOMAIN: - zone_id = zone['id'] - break - return zone_id - - @tenacity.retry( - wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), - reraise=True - ) - def _wait_on_zone_gone(self): - self._wait_on_domain_gone() - if self.post_xenial_queens: - logging.debug('Waiting for zone to disappear') - if self._get_test_zone_id(): - raise Exception("Zone Exists") + def _wait_on_domain_gone(self, domain_id): + @tenacity.retry( + wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), + reraise=True + ) + def wait(): + logging.debug('Waiting for domain %s to disappear', domain_id) + if self._get_domain_id(domain_id=domain_id): + raise Exception("Domain Exists") + self.domain_delete(domain_id) + wait() @tenacity.retry( wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), @@ -233,11 +216,10 @@ class DesignateTests(BaseDesignateTest): def test_400_domain_creation(self): """Simple api calls to create domain.""" logging.debug('Checking if domain exists before trying to create it') - old_dom_id = self._get_test_domain_id() + old_dom_id = self._get_domain_id(domain_name=self.TEST_DOMAIN) if old_dom_id: logging.debug('Deleting old domain') - self.zones_delete(old_dom_id) - self._wait_on_zone_gone() + self._wait_on_domain_gone(old_dom_id) logging.debug('Creating new domain') domain = domains.Domain( @@ -245,10 +227,10 @@ class DesignateTests(BaseDesignateTest): email="fred@amuletexample.com") if self.post_xenial_queens: - new_domain = self.designate.zones.create( + new_domain = self.domain_create( name=domain.name, email=domain.email) else: - new_domain = self.designate.domains.create(domain) + new_domain = self.domain_create(domain) self.assertIsNotNone(new_domain) logging.debug('Creating new test record') @@ -258,16 +240,15 @@ class DesignateTests(BaseDesignateTest): data=self.TEST_RECORD[self.TEST_WWW_RECORD]) if self.post_xenial_queens: - _domain_id = new_domain['id'] + domain_id = new_domain['id'] self.designate.recordsets.create( - _domain_id, _record.name, _record.type, [_record.data]) + domain_id, _record.name, _record.type, [_record.data]) else: - _domain_id = new_domain.id - self.designate.records.create(_domain_id, _record) + domain_id = new_domain.id + self.designate.records.create(domain_id, _record) self._wait_to_resolve_test_record() logging.debug('Tidy up delete test record') - self.zones_delete(_domain_id) - self._wait_on_zone_gone() + self._wait_on_domain_gone(domain_id) logging.debug('OK') From f224e614a8d6469a8b83d7a988a6d1eba557052c Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 7 Feb 2020 15:34:06 -0600 Subject: [PATCH 309/898] only use server api pre xenial-queens --- zaza/openstack/charm_tests/designate/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 47db587..8e57273 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -78,9 +78,9 @@ class BaseDesignateTest(test_utils.OpenStackBaseTest): cls.domain_list = cls.designate.domains.list cls.domain_delete = cls.designate.domains.delete cls.domain_create = cls.designate.domains.create - cls.server_list = cls.designate.servers.list - cls.server_create = cls.designate.servers.create - cls.server_delete = cls.designate.servers.delete + cls.server_list = cls.designate.servers.list + cls.server_create = cls.designate.servers.create + cls.server_delete = cls.designate.servers.delete class DesignateTests(BaseDesignateTest): From 19fb4c07826c4ba70713c02c45fabd3e9cac6853 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 10 Feb 2020 10:24:10 +0000 Subject: [PATCH 310/898] Upgrade complete action for percona. The Percona charm requires that an action is run once all the units have performed their series upgrade. Longer term test_200_run_series_upgrade needs refactoring but this is a tactical fix in the meantime. --- zaza/openstack/charm_tests/series_upgrade/tests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index bad4b1d..eb7add6 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -126,6 +126,15 @@ class SeriesUpgradeTest(unittest.TestCase): action_params={}) model.block_until_all_units_idle() + if "percona-cluster" in applications[application]["charm"]: + logging.info( + "Running complete-cluster-series-upgrade action on leader") + model.run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) + model.block_until_all_units_idle() + class OpenStackSeriesUpgrade(SeriesUpgradeTest): """OpenStack Series Upgrade. From 871c3e133ebd5d3475bf1938f495c08f664919bb Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 6 Feb 2020 17:10:20 +0100 Subject: [PATCH 311/898] Add new CephFS tests --- .../openstack/charm_tests/ceph/fs/__init__.py | 15 +++ zaza/openstack/charm_tests/ceph/fs/tests.py | 126 ++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 zaza/openstack/charm_tests/ceph/fs/__init__.py create mode 100644 zaza/openstack/charm_tests/ceph/fs/tests.py diff --git a/zaza/openstack/charm_tests/ceph/fs/__init__.py b/zaza/openstack/charm_tests/ceph/fs/__init__.py new file mode 100644 index 0000000..b9ae308 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/fs/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ceph-fs.""" diff --git a/zaza/openstack/charm_tests/ceph/fs/tests.py b/zaza/openstack/charm_tests/ceph/fs/tests.py new file mode 100644 index 0000000..63cc492 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/fs/tests.py @@ -0,0 +1,126 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate CephFS testing.""" + +from tenacity import Retrying, stop_after_attempt, wait_exponential + +import zaza.model as model +import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.charm_tests.neutron.tests as neutron_tests +import zaza.openstack.charm_tests.nova.utils as nova_utils +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.configure.guest as guest +import zaza.openstack.utilities.openstack as openstack_utils + + +class CephFSTests(test_utils.OpenStackBaseTest): + """Encapsulate CephFS tests.""" + + RESOURCE_PREFIX = 'zaza-cephfstests' + INSTANCE_USERDATA = """#cloud-config +packages: +- ceph-fuse +- python +mounts: + - [ 'none', '/mnt/cephfs', 'fuse.ceph', 'ceph.id=admin,ceph.conf=/etc/ceph/ceph.conf,_netdev,defaults', '0', '0' ] +write_files: +- content: | +{} + path: /etc/ceph/ceph.conf +- content: | +{} + path: /etc/ceph/ceph.client.admin.keyring +""" # noqa + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(CephFSTests, cls).setUpClass() + + def test_cephfs_share(self): + """Test that CephFS shares can be accessed on two instances. + + 1. Spawn two servers + 2. mount it on both + 3. write a file on one + 4. read it on the other + 5. profit + """ + keyring = model.run_on_leader( + 'ceph-mon', 'cat /etc/ceph/ceph.client.admin.keyring')['Stdout'] + conf = model.run_on_leader( + 'ceph-mon', 'cat /etc/ceph/ceph.conf')['Stdout'] + # Spawn Servers + for attempt in Retrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=10)): + with attempt: + instance_1 = guest.launch_instance( + glance_setup.LTS_IMAGE_NAME, + vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX), + userdata=self.INSTANCE_USERDATA.format( + _indent(conf, 8), + _indent(keyring, 8))) + + for attempt in Retrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=10)): + with attempt: + instance_2 = guest.launch_instance( + glance_setup.LTS_IMAGE_NAME, + vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX), + userdata=self.INSTANCE_USERDATA.format( + _indent(conf, 8), + _indent(keyring, 8))) + + # Write a file on instance_1 + def verify_setup(stdin, stdout, stderr): + status = stdout.channel.recv_exit_status() + self.assertEqual(status, 0) + + fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] + fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] + username = guest.boot_tests['bionic']['username'] + password = guest.boot_tests['bionic'].get('password') + privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) + + for attempt in Retrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=2, max=10)): + with attempt: + openstack_utils.ssh_command( + username, fip_1, 'instance-1', + 'sudo mount -a && ' + 'echo "test" | sudo tee /mnt/cephfs/test', + password=password, privkey=privkey, verify=verify_setup) + + def verify(stdin, stdout, stderr): + status = stdout.channel.recv_exit_status() + self.assertEqual(status, 0) + out = "" + for line in iter(stdout.readline, ""): + out += line + self.assertEqual(out, "test\n") + + openstack_utils.ssh_command( + username, fip_2, 'instance-2', + 'sudo mount -a && ' + 'sudo cat /mnt/cephfs/test', + password=password, privkey=privkey, verify=verify) + + +def _indent(text, amount, ch=' '): + padding = amount * ch + return ''.join(padding+line for line in text.splitlines(True)) From 6aa252ab64dd5c5f178602b880a6e1246d40c733 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 18 Feb 2020 10:21:31 +0100 Subject: [PATCH 312/898] Enable testing of Bionic/Ussuri and Focal --- zaza/openstack/utilities/os_versions.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index eb5093c..34f8bf2 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -34,6 +34,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('cosmic', 'rocky'), ('disco', 'stein'), ('eoan', 'train'), + ('focal', 'ussuri'), ]) @@ -55,6 +56,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2018.2', 'rocky'), ('2019.1', 'stein'), ('2019.2', 'train'), + ('2020.1', 'ussuri'), ]) OPENSTACK_RELEASES_PAIRS = [ @@ -64,7 +66,7 @@ OPENSTACK_RELEASES_PAIRS = [ 'xenial_pike', 'artful_pike', 'xenial_queens', 'bionic_queens', 'bionic_rocky', 'cosmic_rocky', 'bionic_stein', 'disco_stein', 'bionic_train', - 'eoan_train'] + 'eoan_train', 'focal_ussuri'] # The ugly duckling - must list releases oldest to newest SWIFT_CODENAMES = OrderedDict([ @@ -102,6 +104,8 @@ SWIFT_CODENAMES = OrderedDict([ ['2.20.0', '2.21.0']), ('train', ['2.22.0']), + ('ussuri', + ['2.24.0']), ]) # >= Liberty version->codename mapping @@ -116,6 +120,7 @@ PACKAGE_CODENAMES = { ('18', 'rocky'), ('19', 'stein'), ('20', 'train'), + ('21', 'ussuri'), ]), 'neutron-common': OrderedDict([ ('7', 'liberty'), @@ -127,6 +132,7 @@ PACKAGE_CODENAMES = { ('13', 'rocky'), ('14', 'stein'), ('15', 'train'), + ('16', 'ussuri'), ]), 'cinder-common': OrderedDict([ ('7', 'liberty'), @@ -138,6 +144,7 @@ PACKAGE_CODENAMES = { ('13', 'rocky'), ('14', 'stein'), ('15', 'train'), + ('16', 'ussuri'), ]), 'keystone': OrderedDict([ ('8', 'liberty'), @@ -149,6 +156,7 @@ PACKAGE_CODENAMES = { ('14', 'rocky'), ('15', 'stein'), ('16', 'train'), + ('17', 'ussuri'), ]), 'horizon-common': OrderedDict([ ('8', 'liberty'), @@ -160,6 +168,7 @@ PACKAGE_CODENAMES = { ('14', 'rocky'), ('15', 'stein'), ('16', 'train'), + ('17', 'ussuri'), ]), 'ceilometer-common': OrderedDict([ ('5', 'liberty'), @@ -171,6 +180,7 @@ PACKAGE_CODENAMES = { ('11', 'rocky'), ('12', 'stein'), ('13', 'train'), + ('14', 'ussuri'), ]), 'heat-common': OrderedDict([ ('5', 'liberty'), @@ -182,6 +192,7 @@ PACKAGE_CODENAMES = { ('11', 'rocky'), ('12', 'stein'), ('13', 'train'), + ('14', 'ussuri'), ]), 'glance-common': OrderedDict([ ('11', 'liberty'), @@ -193,6 +204,7 @@ PACKAGE_CODENAMES = { ('17', 'rocky'), ('18', 'stein'), ('19', 'train'), + ('20', 'ussuri'), ]), 'openstack-dashboard': OrderedDict([ ('8', 'liberty'), @@ -204,6 +216,7 @@ PACKAGE_CODENAMES = { ('14', 'rocky'), ('15', 'stein'), ('16', 'train'), + ('17', 'ussuri'), ]), 'designate-common': OrderedDict([ ('1', 'liberty'), @@ -215,5 +228,6 @@ PACKAGE_CODENAMES = { ('7', 'rocky'), ('8', 'stein'), ('9', 'train'), + ('10', 'ussuri'), ]), } From a3f6135908aad60591de2620835f435a393f5591 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 18 Feb 2020 12:03:56 +0100 Subject: [PATCH 313/898] Actually add bionic/ussuri release combination --- zaza/openstack/utilities/os_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 34f8bf2..4ffd445 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -66,7 +66,7 @@ OPENSTACK_RELEASES_PAIRS = [ 'xenial_pike', 'artful_pike', 'xenial_queens', 'bionic_queens', 'bionic_rocky', 'cosmic_rocky', 'bionic_stein', 'disco_stein', 'bionic_train', - 'eoan_train', 'focal_ussuri'] + 'eoan_train', 'bionic_ussuri', 'focal_ussuri'] # The ugly duckling - must list releases oldest to newest SWIFT_CODENAMES = OrderedDict([ From 6b110335a544093b0843db486bd1a9ee8d561ac1 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 14 Feb 2020 11:41:18 -0800 Subject: [PATCH 314/898] Percona to MySQL InnoDB Cluster migration Validate the migration from percona-cluster to mysql-innodb-cluster by dumping databases from percona and restoring them in mysql8. --- zaza/openstack/charm_tests/mysql/tests.py | 102 ++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 7603392..2344514 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -18,6 +18,7 @@ import json import logging import os import re +import tempfile import time import zaza.charm_lifecycle.utils as lifecycle_utils @@ -550,3 +551,104 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) + + +class MySQL8MigrationTests(MySQLBaseTest): + """Percona Cluster to MySQL InnoDB Cluster Tests.""" + + def test_999_migrate_percona_to_mysql(self): + """Migrate DBs from percona-cluster to mysql-innodb-cluster. + + Do not rely on self.application_name or other pre-set class values as + we will be pointing to both percona-cluster and mysql-innodb-cluster. + """ + # Current set of Databases and application names + DBS = ["keystone", "glance"] + percona_application = "percona-cluster" + mysql_application = "mysql-innodb-cluster" + percona_leader = zaza.model.get_unit_from_name( + zaza.model.get_lead_unit_name(percona_application)) + mysql_leader = zaza.model.get_unit_from_name( + zaza.model.get_lead_unit_name(mysql_application)) + logging.info("Remove percona-cluster:shared-db relations ...") + for client in DBS: + # Remove relations + zaza.model.remove_relation( + percona_application, + "{}:shared-db".format(percona_application), + "{}:shared-db".format(client)) + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + # Set PXC Strict Mode to MASTER + logging.info("Set PXC Strict Mode MASTER ...") + action = zaza.model.run_action_on_leader( + percona_application, + "set-pxc-strict-mode", + action_params={"mode": "MASTER"}) + assert "failed" not in action.data["status"], ( + "Set PXC Strict Mode MASTER action failed: {}" + .format(action.data)) + # Dump the percona db + logging.info("mysqldump percona-cluster DBs ...") + action = zaza.model.run_action_on_leader( + percona_application, + "mysqldump", + action_params={ + "databases": ",".join(DBS)}) + assert "failed" not in action.data["status"], ( + "mysqldump action failed: {}" + .format(action.data)) + remote_file = action.data["results"]["mysqldump-file"] + remote_backup_dir = "/var/backups/mysql" + # Permissions for ubuntu user to read + logging.info("Set permissions to read percona-cluster:{} ..." + .format(remote_backup_dir)) + zaza.model.run_on_leader( + percona_application, + "chmod 755 {}".format(remote_backup_dir)) + + # SCP back and forth + dump_file = "dump.sql.gz" + logging.info("SCP percona-cluster:{} to mysql-innodb-cluster:{} ..." + .format(remote_file, dump_file)) + with tempfile.TemporaryDirectory() as tmpdirname: + tmp_file = "{}/{}".format(tmpdirname, dump_file) + zaza.model.scp_from_unit( + percona_leader.name, + remote_file, + tmp_file) + zaza.model.scp_to_unit( + mysql_leader.name, + tmp_file, + dump_file) + # Restore mysqldump to mysql-innodb-cluster + logging.info("restore-mysqldump DBs onto mysql-innodb-cluster ...") + action = zaza.model.run_action_on_leader( + mysql_application, + "restore-mysqldump", + action_params={ + "dump-file": "/home/ubuntu/{}".format(dump_file)}) + assert "failed" not in action.data["status"], ( + "restore-mysqldump action failed: {}" + .format(action.data)) + # Add db router relations + logging.info("Add mysql-router:shared-db relations ...") + for client in DBS: + # add relations + zaza.model.add_relation( + mysql_application, + "{}:shared-db".format(client), + "{}-mysql-router:shared-db".format(client)) + # Set PXC Strict Mode back to ENFORCING + logging.info("Set PXC Strict Mode ENFORCING ...") + action = zaza.model.run_action_on_leader( + percona_application, + "set-pxc-strict-mode", + action_params={"mode": "ENFORCING"}) + assert "failed" not in action.data["status"], ( + "Set PXC Strict Mode ENFORCING action failed: {}" + .format(action.data)) + logging.info("Wait for application states ...") + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {})) From 0dc16b289a5919aca6deea2887ee0058c18f723f Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 18 Feb 2020 15:57:13 -0800 Subject: [PATCH 315/898] Lint fixes --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 2344514..04d7561 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -558,7 +558,7 @@ class MySQL8MigrationTests(MySQLBaseTest): def test_999_migrate_percona_to_mysql(self): """Migrate DBs from percona-cluster to mysql-innodb-cluster. - + Do not rely on self.application_name or other pre-set class values as we will be pointing to both percona-cluster and mysql-innodb-cluster. """ From 2136213c8362e84f238053f99e865306bba15b7d Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 19 Feb 2020 10:24:07 +0100 Subject: [PATCH 316/898] Manila tests do not need Nova client --- zaza/openstack/charm_tests/manila/tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index a3365de..d81e030 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -21,7 +21,6 @@ from tenacity import Retrying, stop_after_attempt, wait_exponential from manilaclient import client as manilaclient import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.utilities.openstack as openstack_utils class ManilaTests(test_utils.OpenStackBaseTest): @@ -31,8 +30,6 @@ class ManilaTests(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run class setup for running tests.""" super(ManilaTests, cls).setUpClass() - cls.nova_client = ( - openstack_utils.get_nova_session_client(cls.keystone_session)) cls.manila_client = manilaclient.Client( session=cls.keystone_session, client_version='2') From e22d55d966aed7f45568820a313d6b532e839809 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 25 Feb 2020 18:47:20 +0000 Subject: [PATCH 317/898] Enable for full stack --- zaza/openstack/charm_tests/mysql/tests.py | 26 ++++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 04d7561..fde698c 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -562,8 +562,18 @@ class MySQL8MigrationTests(MySQLBaseTest): Do not rely on self.application_name or other pre-set class values as we will be pointing to both percona-cluster and mysql-innodb-cluster. """ - # Current set of Databases and application names - DBS = ["keystone", "glance"] + # Map application name to db name + apps_to_dbs = { + "keystone": ["keystone"], + "glance": ["glance"], + "cinder": ["cinder"], + "nova-cloud-controller": ["nova", "nova_api", "nova_cell0"], + "neutron-api": ["neutron"], + "openstack-dashboard": ["horizon"], + "placement": ["placement"], + "vault": ["vault"]} + # TODO: This could do an automated check of what is actually deployed + dbs = [db for mapped_dbs in apps_to_dbs.values() for db in mapped_dbs] percona_application = "percona-cluster" mysql_application = "mysql-innodb-cluster" percona_leader = zaza.model.get_unit_from_name( @@ -571,12 +581,12 @@ class MySQL8MigrationTests(MySQLBaseTest): mysql_leader = zaza.model.get_unit_from_name( zaza.model.get_lead_unit_name(mysql_application)) logging.info("Remove percona-cluster:shared-db relations ...") - for client in DBS: + for app in apps_to_dbs.keys(): # Remove relations zaza.model.remove_relation( percona_application, "{}:shared-db".format(percona_application), - "{}:shared-db".format(client)) + "{}:shared-db".format(app)) logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() # Set PXC Strict Mode to MASTER @@ -594,7 +604,7 @@ class MySQL8MigrationTests(MySQLBaseTest): percona_application, "mysqldump", action_params={ - "databases": ",".join(DBS)}) + "databases": ",".join(dbs)}) assert "failed" not in action.data["status"], ( "mysqldump action failed: {}" .format(action.data)) @@ -633,12 +643,12 @@ class MySQL8MigrationTests(MySQLBaseTest): .format(action.data)) # Add db router relations logging.info("Add mysql-router:shared-db relations ...") - for client in DBS: + for app in apps_to_dbs.keys(): # add relations zaza.model.add_relation( mysql_application, - "{}:shared-db".format(client), - "{}-mysql-router:shared-db".format(client)) + "{}:shared-db".format(app), + "{}-mysql-router:shared-db".format(app)) # Set PXC Strict Mode back to ENFORCING logging.info("Set PXC Strict Mode ENFORCING ...") action = zaza.model.run_action_on_leader( From 27407998a9673b5a3d2726655567fbf48f75b5d2 Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 26 Feb 2020 22:18:56 +0000 Subject: [PATCH 318/898] Allow for no charm_name setting --- zaza/openstack/charm_tests/mysql/tests.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index fde698c..f337777 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -33,9 +33,9 @@ class MySQLBaseTest(test_utils.OpenStackBaseTest): """Base for mysql charm tests.""" @classmethod - def setUpClass(cls): + def setUpClass(cls, application_name=None): """Run class setup for running mysql tests.""" - super(MySQLBaseTest, cls).setUpClass() + super().setUpClass(application_name=application_name) cls.application = "mysql" cls.services = ["mysqld"] # Config file affected by juju set config change @@ -556,6 +556,13 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" + @classmethod + def setUpClass(cls): + """Run class setup for running migration tests.""" + # Having application_name set avoids breakage in the OpenStackBaseTest class + # when running bundle tests without charm_name specified + super().setUpClass(application_name="mysql-innodb-cluster") + def test_999_migrate_percona_to_mysql(self): """Migrate DBs from percona-cluster to mysql-innodb-cluster. From 1b3b33d83a1a5e5bf3d4365bb1e7801ad753b182 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 21 Feb 2020 15:39:44 +0100 Subject: [PATCH 319/898] Ensure that the Manila tests exit --- zaza/openstack/charm_tests/manila/tests.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index d81e030..8a1fdee 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -16,7 +16,7 @@ """Encapsulate Manila testing.""" -from tenacity import Retrying, stop_after_attempt, wait_exponential +import tenacity from manilaclient import client as manilaclient @@ -35,8 +35,10 @@ class ManilaTests(test_utils.OpenStackBaseTest): def test_manila_api(self): """Test that the Manila API is working.""" - # now just try a list the shares - for attempt in Retrying( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10)): - self.manila_client.shares.list() + self.assertEqual([], self._list_shares()) + + @tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) + def _list_shares(self): + return self.manila_client.shares.list() From 23cc3e2a9aedd8f85cb5bb796d3ae10497177384 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 27 Feb 2020 16:18:19 +0000 Subject: [PATCH 320/898] Lint fix --- zaza/openstack/charm_tests/mysql/tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index f337777..a800712 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -559,8 +559,9 @@ class MySQL8MigrationTests(MySQLBaseTest): @classmethod def setUpClass(cls): """Run class setup for running migration tests.""" - # Having application_name set avoids breakage in the OpenStackBaseTest class - # when running bundle tests without charm_name specified + # Having application_name set avoids breakage in the + # OpenStackBaseTest class when running bundle tests without + # charm_name specified super().setUpClass(application_name="mysql-innodb-cluster") def test_999_migrate_percona_to_mysql(self): From 707b36c0f805ea50daebfa56aff8c929c6f1aa62 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 3 Mar 2020 14:06:59 +0100 Subject: [PATCH 321/898] Add a ceilometer test case and fix existing ones for trusty/mitaka https://bugs.launchpad.net/charm-ceilometer/+bug/1828424 --- setup.py | 1 + .../openstack/charm_tests/ceilometer/tests.py | 44 ++++++++++++++----- zaza/openstack/charm_tests/test_utils.py | 4 +- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index d212ae8..6873ef9 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ install_require = [ 'python-novaclient', 'python-neutronclient', 'python-octaviaclient', + 'python-ceilometerclient', 'python-cinderclient', 'python-swiftclient', 'zaza@git+https://github.com/openstack-charmers/zaza.git#egg=zaza', diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index 2741191..5701cfb 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -19,8 +19,9 @@ import copy import logging -import zaza.openstack.utilities.openstack as openstack_utils +import ceilometerclient.v2.client as ceilo_client import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils class CeilometerTest(test_utils.OpenStackBaseTest): @@ -39,11 +40,11 @@ class CeilometerTest(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run class setup for running Ceilometer tests.""" super(CeilometerTest, cls).setUpClass() + cls.current_release = openstack_utils.get_os_release() @property def services(self): - """Return a list services for Openstack Release.""" - self.current_release = openstack_utils.get_os_release() + """Return a list of services for Openstack Release.""" services = [] if self.application_name == 'ceilometer-agent': @@ -89,20 +90,43 @@ class CeilometerTest(test_utils.OpenStackBaseTest): return services - # NOTE(beisner): need to add more functional tests - - def test_900_restart_on_config_change(self): - """Checking restart happens on config change.""" - _services = copy.deepcopy(self.services) + @property + def restartable_services(self): + """Return a list of services that are known to be restartable. + For Openstack Release these services are known to be able to be stopped + and started with no issues. + """ # Due to Bug #1861321 ceilometer-collector does not reliably # restart. + _services = copy.deepcopy(self.services) if self.current_release <= CeilometerTest.TRUSTY_MITAKA: try: _services.remove('ceilometer-collector') except ValueError: pass + return _services + def test_400_api_connection(self): + """Simple api calls to check service is up and responding.""" + if self.current_release >= CeilometerTest.XENIAL_PIKE: + logging.info('Skipping API checks as ceilometer api has been ' + 'removed') + return + + logging.info('Instantiating ceilometer client...') + ceil = ceilo_client.Client( + session=openstack_utils.get_overcloud_keystone_session() + ) + + logging.info('Checking api functionality...') + assert(ceil.samples.list() == []) + assert(ceil.meters.list() == []) + + # NOTE(beisner): need to add more functional tests + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change.""" config_name = 'debug' if self.application_name == 'ceilometer-agent': @@ -138,7 +162,7 @@ class CeilometerTest(test_utils.OpenStackBaseTest): set_alternate, default_entry, alternate_entry, - _services) + self.restartable_services) def test_901_pause_resume(self): """Run pause and resume tests. @@ -146,5 +170,5 @@ class CeilometerTest(test_utils.OpenStackBaseTest): Pause service and check services are stopped then resume and check they are started. """ - with self.pause_resume(self.services): + with self.pause_resume(self.restartable_services): logging.info("Testing pause and resume") diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 5903119..e33bd69 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -335,8 +335,8 @@ class OpenStackBaseTest(unittest.TestCase): Pause and then resume a unit checking that services are in the required state after each action - :param services: Services expected to be restarted when config_file is - changed. + :param services: Services expected to be restarted when the unit is + paused/resumed. :type services: list :param pgrep_full: Should pgrep be used rather than pidof to identify a service. From 0cee4c90bbba10c10fe921282c3e0b21b0b1a53a Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 5 Mar 2020 12:12:08 +0100 Subject: [PATCH 322/898] Fix typo --- zaza/openstack/utilities/openstack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 59f8cd6..661d107 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1534,7 +1534,7 @@ def get_undercloud_auth(): else: logging.error('Missing OS authentication setting: OS_AUTH_URL') raise exceptions.MissingOSAthenticationException( - 'One or more OpenStack authetication variables could ' + 'One or more OpenStack authentication variables could ' 'be found in the environment. Please export the OS_* ' 'settings into the environment.') @@ -1581,7 +1581,7 @@ def get_undercloud_auth(): logging.error('Missing OS authentication setting: {}' ''.format(key)) raise exceptions.MissingOSAthenticationException( - 'One or more OpenStack authetication variables could ' + 'One or more OpenStack authentication variables could ' 'be found in the environment. Please export the OS_* ' 'settings into the environment.') From d389e357e2d66646ffdf07ddca5821cdfc392188 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 5 Mar 2020 12:37:40 +0100 Subject: [PATCH 323/898] Remove obsolete comment --- zaza/openstack/charm_tests/ceilometer/tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index 5701cfb..a7f2618 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -123,8 +123,6 @@ class CeilometerTest(test_utils.OpenStackBaseTest): assert(ceil.samples.list() == []) assert(ceil.meters.list() == []) - # NOTE(beisner): need to add more functional tests - def test_900_restart_on_config_change(self): """Checking restart happens on config change.""" config_name = 'debug' From 3f1ec453daa74da7b8c231daa92395eef1b5a4c6 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Fri, 6 Mar 2020 09:40:32 +0100 Subject: [PATCH 324/898] add lxd container support to get_machine_status --- zaza/openstack/utilities/juju.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index fd7abc1..27107fa 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -137,7 +137,11 @@ def get_machine_status(machine, key=None, model_name=None): :rtype: dict """ status = get_full_juju_status(model_name=model_name) - status = status.machines.get(machine) + if "lxd" in machine: + host = machine.split('/')[0] + status = status.machines.get(host)['containers'][machine] + else: + status = status.machines.get(machine) if key: status = status.get(key) return status From 72fb1cf7e703f01fd1221996cb032c4a72994255 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 6 Mar 2020 10:35:31 +0100 Subject: [PATCH 325/898] Break out Neutron create network through the API test This is broken out into a separate class as it can be useful as standalone tests for Neutron plugin subordinate charms. --- zaza/openstack/charm_tests/neutron/tests.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 6161c88..2ae92f6 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -208,13 +208,17 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): return services -class NeutronApiTest(test_utils.OpenStackBaseTest): - """Test basic Neutron API Charm functionality.""" +class NeutronCreateNetworkTest(test_utils.OpenStackBaseTest): + """Test creating a Neutron network through the API. + + This is broken out into a separate class as it can be useful as standalone + tests for Neutron plugin subordinate charms. + """ @classmethod def setUpClass(cls): """Run class setup for running Neutron Gateway tests.""" - super(NeutronApiTest, cls).setUpClass() + super(NeutronCreateNetworkTest, cls).setUpClass() cls.current_os_release = openstack_utils.get_os_release() # set up clients @@ -251,6 +255,10 @@ class NeutronApiTest(test_utils.OpenStackBaseTest): logging.debug('Deleting neutron network...') self.neutron_client.delete_network(network['id']) + +class NeutronApiTest(NeutronCreateNetworkTest): + """Test basic Neutron API Charm functionality.""" + def test_401_enable_qos(self): """Check qos settings set via neutron-api charm.""" if (self.current_os_release >= From 9e3b88678cb83f934f65023b4cd36f0337c768a3 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 6 Mar 2020 13:58:07 +0100 Subject: [PATCH 326/898] Add Gnocchi tests https://bugs.launchpad.net/charm-gnocchi/+bug/1828424 --- requirements.txt | 1 + setup.py | 1 + .../openstack/charm_tests/gnocchi/__init__.py | 15 +++++ zaza/openstack/charm_tests/gnocchi/tests.py | 60 +++++++++++++++++++ 4 files changed, 77 insertions(+) create mode 100644 zaza/openstack/charm_tests/gnocchi/__init__.py create mode 100644 zaza/openstack/charm_tests/gnocchi/tests.py diff --git a/requirements.txt b/requirements.txt index 3b28f23..e43c4e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ dnspython>=1.12.0 psutil>=1.1.1,<2.0.0 python-openstackclient>=3.14.0 aodhclient +gnocchiclient>=7.0.5,<8.0.0 python-designateclient python-ceilometerclient python-cinderclient diff --git a/setup.py b/setup.py index 6873ef9..3090bd5 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ install_require = [ 'tenacity', 'oslo.config', 'aodhclient', + 'gnocchiclient>=7.0.5,<8.0.0', 'python-designateclient>=1.5,<3.0.0', 'python-heatclient', 'python-glanceclient', diff --git a/zaza/openstack/charm_tests/gnocchi/__init__.py b/zaza/openstack/charm_tests/gnocchi/__init__.py new file mode 100644 index 0000000..3225ba0 --- /dev/null +++ b/zaza/openstack/charm_tests/gnocchi/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing gnocchi.""" diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py new file mode 100644 index 0000000..6811880 --- /dev/null +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Gnocchi testing.""" + +import logging + +from gnocchiclient.v1 import client as gnocchi_client +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +class GnocchiTest(test_utils.OpenStackBaseTest): + """Encapsulate Gnocchi tests.""" + + @property + def services(self): + """Return a list of services for Openstack Release.""" + return ['haproxy', 'gnocchi-metricd', 'apache2'] + + def test_200_api_connection(self): + """Simple api calls to check service is up and responding.""" + logging.info('Instantiating gnocchi client...') + overcloud_auth = openstack_utils.get_overcloud_auth() + keystone = openstack_utils.get_keystone_client(overcloud_auth) + gnocchi_ep = keystone.service_catalog.url_for( + service_type='metric', + interface='publicURL' + ) + gnocchi = gnocchi_client.Client( + session=openstack_utils.get_overcloud_keystone_session(), + adapter_options={ + 'endpoint_override': gnocchi_ep, + } + ) + + logging.info('Checking api functionality...') + assert(gnocchi.status.get() != []) + + def test_910_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started. + """ + with self.pause_resume(self.services): + logging.info("Testing pause and resume") From f1168aa36d149ff5a61639f8fbaeee07e28ffdda Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 10 Mar 2020 10:21:14 +0000 Subject: [PATCH 327/898] Fixes for masakari tests * Deal with hypervisors now using fqdn * Skip instant restart tests until bug is resolved --- zaza/openstack/charm_tests/masakari/tests.py | 2 ++ zaza/openstack/configure/masakari.py | 2 +- zaza/openstack/utilities/juju.py | 5 +++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/masakari/tests.py b/zaza/openstack/charm_tests/masakari/tests.py index 274119a..52d9c01 100644 --- a/zaza/openstack/charm_tests/masakari/tests.py +++ b/zaza/openstack/charm_tests/masakari/tests.py @@ -18,6 +18,7 @@ from datetime import datetime import logging +import unittest import tenacity import novaclient @@ -165,6 +166,7 @@ class MasakariTest(test_utils.OpenStackBaseTest): def test_instance_restart_on_fail(self): """Test singlee guest crash and recovery.""" + raise unittest.SkipTest("Bug #1866638") vm_name = 'zaza-test-instance-failover' vm = self.ensure_guest(vm_name) _, unit_name = self.get_guests_compute_info(vm_name) diff --git a/zaza/openstack/configure/masakari.py b/zaza/openstack/configure/masakari.py index 9d48e03..1f6621d 100644 --- a/zaza/openstack/configure/masakari.py +++ b/zaza/openstack/configure/masakari.py @@ -39,7 +39,7 @@ def roundrobin_assign_hosts_to_segments(nova_client, masakari_client): segment_ids = segment_ids * len(hypervisors) for hypervisor in hypervisors: target_segment = segment_ids.pop() - hostname = hypervisor.hypervisor_hostname.split('.')[0] + hostname = hypervisor.hypervisor_hostname logging.info('Adding {} to segment {}'.format(hostname, target_segment)) masakari_client.create_host( diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index 450a17d..a83ba5b 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -114,8 +114,9 @@ def get_unit_name_from_host_name(host_name, application, model_name=None): :param model_name: Name of model to query. :type model_name: str """ - # Assume that a juju managed hostname always ends in the machine number. - machine_number = host_name.split('-')[-1] + # Assume that a juju managed hostname always ends in the machine number and + # remove the domain name if it present. + machine_number = host_name.split('-')[-1].split('.')[0] unit_names = [ u.entity_id for u in model.get_units(application_name=application, From cbdb3086a1363cacafdce93968f4af2b98885b14 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 11 Mar 2020 09:56:30 +0100 Subject: [PATCH 328/898] keystone: expect security check no admin-token to pass Also add test to validate that the domain named ``default`` literally has an ID of ``default``. --- zaza/openstack/charm_tests/keystone/tests.py | 23 ++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index e057428..9f1d31d 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -328,6 +328,21 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): {'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip)}) _validate_token_data(openrc) + def test_backward_compatible_uuid_for_default_domain(self): + """Check domain named ``default`` literally has ``default`` as ID. + + Some third party software chooses to hard code this value for some + inexplicable reason. + """ + with self.v3_keystone_preferred(): + ks_session = openstack_utils.get_keystone_session( + openstack_utils.get_overcloud_auth()) + ks_client = openstack_utils.get_keystone_session_client( + ks_session) + domain = ks_client.domains.get('default') + logging.info(pprint.pformat(domain)) + assert domain.id == 'default' + class SecurityTests(BaseKeystoneTest): """Keystone security tests tests.""" @@ -343,13 +358,13 @@ class SecurityTests(BaseKeystoneTest): # this initial work to get validation in. There will be bugs targeted # to each one and resolved independently where possible. expected_failures = [ - 'disable-admin-token', ] expected_passes = [ 'check-max-request-body-size', - 'uses-sha256-for-hashing-tokens', - 'uses-fernet-token-after-default', + 'disable-admin-token', 'insecure-debug-is-false', + 'uses-fernet-token-after-default', + 'uses-sha256-for-hashing-tokens', 'validate-file-ownership', 'validate-file-permissions', ] @@ -363,7 +378,7 @@ class SecurityTests(BaseKeystoneTest): action_params={}), expected_passes, expected_failures, - expected_to_pass=False) + expected_to_pass=True) class LdapTests(BaseKeystoneTest): From e28b9e0c48aac07517a435b62c7cdd3153aadb96 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 11 Mar 2020 13:50:04 +0100 Subject: [PATCH 329/898] policyd: Use different policy for test of Keystone A side effect of migrating to bootstrapping Keystone as opposed to using the admin_token is that the charm credentials is now subject to the Keystone policy. At present the ``list_services`` policy is used as a test of the Policy Override feature, however revoking access to said call will make the charm go into an error state as it attempts to use it as part of managing the Keystone CRUD. Change the test to use the ``list_credentials`` policy for test instead. --- zaza/openstack/charm_tests/policyd/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index f332765..703f0af 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -258,7 +258,7 @@ class BasePolicydSpecialization(PolicydTest, class KeystonePolicydTest(BasePolicydSpecialization): - _rule = {'rule.yaml': "{'identity:list_services': '!'}"} + _rule = {'rule.yaml': "{'identity:list_credentials': '!'}"} def get_client_and_attempt_operation(self, keystone_session): ... etc. @@ -485,7 +485,7 @@ class BasePolicydSpecialization(PolicydTest, class KeystoneTests(BasePolicydSpecialization): """Test the policyd override using the keystone client.""" - _rule = {'rule.yaml': "{'identity:list_services': '!'}"} + _rule = {'rule.yaml': "{'identity:list_credentials': '!'}"} @classmethod def setUpClass(cls, application_name=None): @@ -506,7 +506,7 @@ class KeystoneTests(BasePolicydSpecialization): keystone_client = openstack_utils.get_keystone_session_client( self.get_keystone_session_demo_admin_user(ip)) try: - keystone_client.services.list() + keystone_client.credentials.list() except keystoneauth1.exceptions.http.Forbidden: raise PolicydOperationFailedException() From 9a3a716f6fb83eeb87ca171ffa65ebfbed720bdf Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 12 Mar 2020 06:42:14 +0000 Subject: [PATCH 330/898] Remove rogue single quote --- zaza/openstack/configure/masakari.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/configure/masakari.py b/zaza/openstack/configure/masakari.py index 1f6621d..9b8e9bf 100644 --- a/zaza/openstack/configure/masakari.py +++ b/zaza/openstack/configure/masakari.py @@ -148,7 +148,7 @@ def _svc_set_systemd_restart_mode(unit_name, service_name, mode, model_name): mode)) cmds = [ ("sed -i -e 's/^Restart=.*/Restart={}/g' " - "/lib/systemd/system/{}.service'").format(mode, service_name), + "/lib/systemd/system/{}.service").format(mode, service_name), 'systemctl daemon-reload'] logging.info('Running {} on {}'.format(cmds, unit_name)) zaza.model.run_on_unit( From 01578b8c55ad0ae427219b8ee786d73580f1bb22 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 13 Mar 2020 07:19:09 +0100 Subject: [PATCH 331/898] Add missing pika (RabbitMQ client) --- requirements.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index e43c4e5..c0cdbf8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ psutil>=1.1.1,<2.0.0 python-openstackclient>=3.14.0 aodhclient gnocchiclient>=7.0.5,<8.0.0 +pika>=1.1.0,<2.0.0 python-designateclient python-ceilometerclient python-cinderclient diff --git a/setup.py b/setup.py index 3090bd5..5475946 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ install_require = [ 'oslo.config', 'aodhclient', 'gnocchiclient>=7.0.5,<8.0.0', + 'pika>=1.1.0,<2.0.0', 'python-designateclient>=1.5,<3.0.0', 'python-heatclient', 'python-glanceclient', From 00ad21f42f9a6e60928f0195a61aa40bc45fb4a8 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 13 Mar 2020 19:00:27 +0100 Subject: [PATCH 332/898] swift: Add S3 API test --- requirements.txt | 1 + setup.py | 1 + zaza/openstack/charm_tests/swift/tests.py | 66 +++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/requirements.txt b/requirements.txt index e43c4e5..286d7da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ aiounittest async_generator +boto3 juju juju_wait PyYAML<=4.2,>=3.0 diff --git a/setup.py b/setup.py index 3090bd5..2c63c1e 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,7 @@ from setuptools.command.test import test as TestCommand version = "0.0.1.dev1" install_require = [ 'async_generator', + 'boto3', 'cryptography', 'hvac<0.7.0', 'jinja2', diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index f5e2370..b322241 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -17,6 +17,7 @@ """Encapsulate swift testing.""" import logging +import pprint import tenacity import zaza.model @@ -26,6 +27,8 @@ import zaza.openstack.configure.guest import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.utilities.swift as swift_utils +import boto3 + class SwiftImageCreateTest(test_utils.OpenStackBaseTest): """Test swift proxy via glance.""" @@ -226,3 +229,66 @@ class SwiftGlobalReplicationTests(test_utils.OpenStackBaseTest): self.assertEqual( len(obj_replicas.all_zones), 3) + + +class S3APITest(test_utils.OpenStackBaseTest): + """Test object storage S3 API.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(S3APITest, cls).setUpClass() + + session = openstack_utils.get_overcloud_keystone_session() + ks_client = openstack_utils.get_keystone_session_client(session) + + # Get token data so we can glean our user_id and project_id + token_data = ks_client.tokens.get_token_data(session.get_token()) + project_id = token_data['token']['project']['id'] + user_id = token_data['token']['user']['id'] + + # Store URL to service providing S3 compatible API + for entry in token_data['token']['catalog']: + if entry['type'] == 's3': + for endpoint in entry['endpoints']: + if endpoint['interface'] == 'public': + cls.s3_region = endpoint['region'] + cls.s3_endpoint = endpoint['url'] + + # Create AWS compatible application credentials in Keystone + cls.ec2_creds = ks_client.ec2.create(user_id, project_id) + + def test_s3_list_buckets(self): + """Use S3 API to list buckets.""" + # We use a mix of the high- and low-level API with common arguments + kwargs = { + 'region_name': self.s3_region, + 'aws_access_key_id': self.ec2_creds.access, + 'aws_secret_access_key': self.ec2_creds.secret, + 'endpoint_url': self.s3_endpoint, + } + s3_client = boto3.client('s3', **kwargs) + s3 = boto3.resource('s3', **kwargs) + + # Create bucket + bucket_name = 'zaza-s3' + bucket = s3.Bucket(bucket_name) + bucket.create() + + # Validate its presence + bucket_list = s3_client.list_buckets() + logging.info(pprint.pformat(bucket_list)) + for bkt in bucket_list['Buckets']: + if bkt['Name'] == bucket_name: + break + else: + AssertionError('Bucket "{}" not found'.format(bucket_name)) + + # Delete bucket + bucket.delete() + + # Validate its absence + bucket_list = s3_client.list_buckets() + logging.info(pprint.pformat(bucket_list)) + for bkt in bucket_list['Buckets']: + assert bkt['Name'] != bucket_name From c7360ea4b12b2b7d9e9c46bcaeac344de774198a Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 20 Mar 2020 14:40:12 +0100 Subject: [PATCH 333/898] Remove client version from Manila Test --- zaza/openstack/charm_tests/manila/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index 8a1fdee..5cd7e26 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -31,7 +31,7 @@ class ManilaTests(test_utils.OpenStackBaseTest): """Run class setup for running tests.""" super(ManilaTests, cls).setUpClass() cls.manila_client = manilaclient.Client( - session=cls.keystone_session, client_version='2') + session=cls.keystone_session) def test_manila_api(self): """Test that the Manila API is working.""" From 1ece576981a4751132c9b0ac9192eb70359290dd Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 23 Mar 2020 10:28:28 +0100 Subject: [PATCH 334/898] radosgw: Pass local CA cert to client when relevant --- zaza/openstack/charm_tests/ceph/tests.py | 15 ++++++++++----- zaza/openstack/charm_tests/swift/tests.py | 1 + zaza/openstack/charm_tests/test_utils.py | 1 + zaza/openstack/utilities/openstack.py | 18 ++++++++++++++++-- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index f48668a..a607a5f 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -605,7 +605,8 @@ class CephRGWTest(test_utils.OpenStackBaseTest): region_name = 'RegionOne' swift_client = zaza_openstack.get_swift_session_client( keystone_session, - region_name + region_name, + cacert=self.cacert, ) _container = 'demo-container' _test_data = 'Test data from Zaza' @@ -629,7 +630,8 @@ class CephRGWTest(test_utils.OpenStackBaseTest): keystone_session = zaza_openstack.get_overcloud_keystone_session() source_client = zaza_openstack.get_swift_session_client( keystone_session, - region_name='east-1' + region_name='east-1', + cacert=self.cacert, ) _container = 'demo-container' _test_data = 'Test data from Zaza' @@ -643,7 +645,8 @@ class CephRGWTest(test_utils.OpenStackBaseTest): target_client = zaza_openstack.get_swift_session_client( keystone_session, - region_name='east-1' + region_name='east-1', + cacert=self.cacert, ) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), @@ -675,11 +678,13 @@ class CephRGWTest(test_utils.OpenStackBaseTest): keystone_session = zaza_openstack.get_overcloud_keystone_session() source_client = zaza_openstack.get_swift_session_client( keystone_session, - region_name='east-1' + region_name='east-1', + cacert=self.cacert, ) target_client = zaza_openstack.get_swift_session_client( keystone_session, - region_name='west-1' + region_name='west-1', + cacert=self.cacert, ) zaza_model.run_action_on_leader( 'slave-ceph-radosgw', diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index b322241..32441df 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -266,6 +266,7 @@ class S3APITest(test_utils.OpenStackBaseTest): 'aws_access_key_id': self.ec2_creds.access, 'aws_secret_access_key': self.ec2_creds.secret, 'endpoint_url': self.s3_endpoint, + 'verify': self.cacert, } s3_client = boto3.client('s3', **kwargs) s3 = boto3.resource('s3', **kwargs) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index e33bd69..f2eebb3 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -131,6 +131,7 @@ class OpenStackBaseTest(unittest.TestCase): cls.application_name, model_name=cls.model_name) logging.debug('Leader unit is {}'.format(cls.lead_unit)) + cls.cacert = openstack_utils.get_cacert() def config_current(self, application_name=None, keys=None): """Get Current Config of an application normalized into key-values. diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 661d107..df10e6e 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -158,6 +158,16 @@ KEYSTONE_REMOTE_CACERT = ( KEYSTONE_LOCAL_CACERT = ("/tmp/{}".format(KEYSTONE_CACERT)) +def get_cacert(): + """Return path to CA Certificate bundle for verification during test. + + :returns: Path to CA Certificate bundle or None. + :rtype: Optional[str] + """ + if os.path.exists(KEYSTONE_LOCAL_CACERT): + return KEYSTONE_LOCAL_CACERT + + # Openstack Client helpers def get_ks_creds(cloud_creds, scope='PROJECT'): """Return the credentials for authenticating against keystone. @@ -244,18 +254,22 @@ def get_neutron_session_client(session): def get_swift_session_client(session, - region_name='RegionOne'): + region_name='RegionOne', + cacert=None): """Return swiftclient authenticated by keystone session. :param session: Keystone session object :type session: keystoneauth1.session.Session object :param region_name: Optional region name to use :type region_name: str + :param cacert: Path to CA Certificate + :type cacert: Optional[str] :returns: Authenticated swiftclient :rtype: swiftclient.Client object """ return swiftclient.Connection(session=session, - os_options={'region_name': region_name}) + os_options={'region_name': region_name}, + cacert=cacert) def get_octavia_session_client(session, service_type='load-balancer', From c6f0f79ea561ab87ae0c91c6e55c7ace26b8769a Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 23 Mar 2020 12:06:04 +0100 Subject: [PATCH 335/898] radosgw: Add retry for object storage test When testing with TLS there is a chance the deployment will appear done and idle prior to ceph-radosgw and Keystone have updated the service catalog. Retry the test in this circumstance. --- zaza/openstack/charm_tests/ceph/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index a607a5f..ef3b59b 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -592,6 +592,13 @@ class CephRGWTest(test_utils.OpenStackBaseTest): target_status='running' ) + # When testing with TLS there is a chance the deployment will appear done + # and idle prior to ceph-radosgw and Keystone have updated the service + # catalog. Retry the test in this circumstance. + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=10, max=300), + reraise=True, stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type( + ConnectionRefusedError)) def test_object_storage(self): """Verify object storage API. From c22753e4ef4ea7f87ab4b4d6ba7f61c8c574adb3 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 23 Mar 2020 13:40:09 +0100 Subject: [PATCH 336/898] radosgw: Adjust exceptions selected for retry A set of the exceptions we want to catch all descend from ``IOError``. Let's see if this covers them all. --- zaza/openstack/charm_tests/ceph/tests.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index ef3b59b..2e16617 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -597,8 +597,7 @@ class CephRGWTest(test_utils.OpenStackBaseTest): # catalog. Retry the test in this circumstance. @tenacity.retry(wait=tenacity.wait_exponential(multiplier=10, max=300), reraise=True, stop=tenacity.stop_after_attempt(10), - retry=tenacity.retry_if_exception_type( - ConnectionRefusedError)) + retry=tenacity.retry_if_exception_type(IOError)) def test_object_storage(self): """Verify object storage API. From 7076053627ec1c621c9f1cc5596a5b15fe885510 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 23 Mar 2020 17:49:22 +0000 Subject: [PATCH 337/898] Update rabbitmq tests for version 3.8.2 on focal Ubuntu focal ships with rabbitmq server 3.8.2 which has changed the text output format for the cli commands that the tests rely on. Fortunately, 3.8.2 also adds a --formatter=json option. This patch takes advantage of that. --- .../charm_tests/rabbitmq_server/utils.py | 118 +++++++++++++++++- 1 file changed, 114 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index dba2d92..9826add 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -120,6 +120,26 @@ def delete_user(units, username="testuser1"): output = zaza.model.run_on_unit(units[0].entity_id, cmd_user_del) +def is_rabbitmq_version_ge_382(unit): + """Test is the rabbitmq version on the :param:`unit` is 3.8.2+. + + Returns True if the rabbitmq_server version installed on the :param:`unit` + is >= 3.8.2 + + :param unit: the unit to test + :type unit: :class:`juju.model.ModelEntity` + :returns: True if the server is 3.8.2 or later + :rtype: Boolean + """ + cmd = 'rabbitmqctl version' + output = zaza.model.run_on_unit(unit.entity_id, cmd)['Stdout'].strip() + logging.debug('{} rabbitmq version:{}'.format(unit.entity_id, output)) + try: + return tuple(map(int, output.split('.')[:3])) >= (3, 8, 2) + except Exception: + return False + + def get_cluster_status(unit): """Get RabbitMQ cluster status output. @@ -138,10 +158,34 @@ def get_cluster_status(unit): def get_cluster_running_nodes(unit): """Get a list of RabbitMQ cluster's running nodes. + Return a list of the running rabbitmq cluster nodes from the specified + unit. + + NOTE: this calls one of two functions depending on whether the installed + version on the unit is 3.8.2 and newer, or older. If newer then the + --formatter=json option is used to simplify parsing of the cluster data. + + :param unit: the unit to fetch running nodes list from + :type unit: :class:`juju.model.ModelEntity` + :returns: List containing node names of running nodes + :rtype: List[str] + """ + if is_rabbitmq_version_ge_382(unit): + return _get_cluster_running_nodes_38(unit) + else: + return _get_cluster_running_nodes_pre_38(unit) + + +def _get_cluster_running_nodes_pre_38(unit): + """Get a list of RabbitMQ cluster's running nodes (pre 3.8.2). + Parse rabbitmqctl cluster_status output string, return list of running rabbitmq cluster nodes. + :param unit: unit pointer + :type unit: :class:`juju.model.ModelEntity` :returns: List containing node names of running nodes + :rtype: List[str] """ # NOTE(beisner): rabbitmqctl cluster_status output is not # json-parsable, do string chop foo, then json.loads that. @@ -156,6 +200,23 @@ def get_cluster_running_nodes(unit): return [] +def _get_cluster_running_nodes_38(unit): + """Get a list of RabbitMQ cluster's running nodes (3.8.2+). + + Return a list of the running rabbitmq cluster nodes from the specified + unit. + + :param unit: the unit to fetch running nodes list from + :type unit: :class:`juju.model.ModelEntity` + :returns: List containing node names of running nodes + :rtype: List[str] + """ + cmd = 'rabbitmqctl cluster_status --formatter=json' + output = zaza.model.run_on_unit(unit.entity_id, cmd)['Stdout'].strip() + decoded = json.loads(output) + return decoded['running_nodes'] + + def validate_cluster_running_nodes(units): """Check all rmq unit hostnames are represented in cluster_status. @@ -454,10 +515,53 @@ def get_amqp_message_by_unit(unit, queue="test", def check_unit_cluster_nodes(unit, unit_node_names): - """Check if unit exists in list of Rmq cluster node names.""" - unit_name = unit.entity_id + """Check if unit exists in list of Rmq cluster node names. + + NOTE: this calls one of two functions depending on whether the installed + version on the unit is 3.8.2 and newer, or older. If newer then the + --formatter=json option is used to simplify parsing of the cluster data. + + :param unit: the unit to fetch running nodes list from + :type unit: :class:`juju.model.ModelEntity` + :param unit_node_names: The unit node names to check against + :type unit_node_names: List[str] + :returns: List containing node names of running nodes + :rtype: List[str] + """ + if is_rabbitmq_version_ge_382(unit): + return _check_unit_cluster_nodes_38(unit, unit_node_names) + else: + return _check_unit_cluster_nodes_pre_38(unit, unit_node_names) + + +def _check_unit_cluster_nodes_38(unit, unit_node_names): + """Check if unit exists in list of Rmq cluster node names (3.8.2+). + + :param unit: the unit to fetch running nodes list from + :type unit: :class:`juju.model.ModelEntity` + :param unit_node_names: The unit node names to check against + :type unit_node_names: List[str] + :returns: List containing node names of running nodes + :rtype: List[str] + """ + cmd = 'rabbitmqctl cluster_status --formatter=json' + output = zaza.model.run_on_unit(unit.entity_id, cmd)['Stdout'].strip() + decoded = json.loads(output) + return _post_check_unit_cluster_nodes( + unit, decoded['disk_nodes'], unit_node_names) + + +def _check_unit_cluster_nodes_pre_38(unit, unit_node_names): + """Check if unit exists in list of Rmq cluster node names (pre 3.8.2). + + :param unit: the unit to fetch running nodes list from + :type unit: :class:`juju.model.ModelEntity` + :param unit_node_names: The unit node names to check against + :type unit_node_names: List[str] + :returns: List containing node names of running nodes + :rtype: List[str] + """ nodes = [] - errors = [] str_stat = get_cluster_status(unit) # make the interesting part of rabbitmqctl cluster_status output # json-parseable. @@ -466,11 +570,17 @@ def check_unit_cluster_nodes(unit, unit_node_names): pos_end = str_stat.find(']}]},', pos_start) + 1 str_nodes = str_stat[pos_start:pos_end].replace("'", '"') nodes = json.loads(str_nodes) + return _post_check_unit_cluster_nodes(unit, nodes, unit_node_names) + + +def _post_check_unit_cluster_nodes(unit, nodes, unit_node_names): + """Finish of the check_unit_cluster_nodes function (internal).""" + unit_name = unit.entity_id + errors = [] for node in nodes: if node not in unit_node_names: errors.append('Cluster registration check failed on {}: ' '{} should not be registered with RabbitMQ ' 'after unit removal.\n' ''.format(unit_name, node)) - return errors From 8e55f588c8b4efec3889cbf6a3a6bf590ca98ef5 Mon Sep 17 00:00:00 2001 From: Felipe Reyes Date: Sat, 14 Mar 2020 22:04:48 -0300 Subject: [PATCH 338/898] mysql: add check for seeded file in pxc Verify the seeded file is present with the expected content since it will be used by *-relation-changed hook to determine if it can process requests of related units. Related-Bug: #1868326 --- zaza/openstack/charm_tests/mysql/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index a800712..f674e3f 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -29,6 +29,9 @@ import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.utilities.generic as generic_utils +PXC_SEEDED_FILE = "/var/lib/percona-xtradb-cluster/seeded" + + class MySQLBaseTest(test_utils.OpenStackBaseTest): """Base for mysql charm tests.""" @@ -202,6 +205,10 @@ class PerconaClusterCharmTests(MySQLCommonTests, PerconaClusterBaseTest): " (wanted=%s, cluster_size=%s)" % (self.units, cluster_size)) assert cluster_size >= self.units, msg + logging.info("Ensuring PXC seeded file is present") + zaza.model.block_until_file_has_contents(self.application, + PXC_SEEDED_FILE, "done") + def test_130_change_root_password(self): """Change root password. From 93fd631b1ae9e4246faad3c9acbf5af49e329766 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 3 Mar 2020 09:33:10 +0100 Subject: [PATCH 339/898] Split up and re-organize upgrade helpers --- requirements.txt | 1 + .../utilities/test_zaza_utilities_generic.py | 182 ------- .../test_zaza_utilities_series_upgrade.py | 239 +++++++++ .../charm_tests/series_upgrade/tests.py | 6 +- zaza/openstack/utilities/charm_upgrade.py | 15 + zaza/openstack/utilities/generic.py | 334 ------------ zaza/openstack/utilities/openstack_upgrade.py | 18 +- zaza/openstack/utilities/series_upgrade.py | 479 ++++++++++++++++++ zaza/openstack/utilities/upgrade_utils.py | 30 ++ 9 files changed, 771 insertions(+), 533 deletions(-) create mode 100644 unit_tests/utilities/test_zaza_utilities_series_upgrade.py create mode 100644 zaza/openstack/utilities/charm_upgrade.py create mode 100644 zaza/openstack/utilities/series_upgrade.py create mode 100644 zaza/openstack/utilities/upgrade_utils.py diff --git a/requirements.txt b/requirements.txt index cbf31df..68f8f71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ aiounittest +asyncio async_generator boto3 juju diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index d18b646..0f3ff9c 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -173,45 +173,6 @@ class TestGenericUtils(ut_utils.BaseTestCase): _yaml_dict) self._open.assert_called_once_with(_filename, "r") - def test_dist_upgrade(self): - _unit = "app/2" - generic_utils.dist_upgrade(_unit) - dist_upgrade_cmd = ( - """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ - """-o "Dpkg::Options::=--force-confdef" """ - """-o "Dpkg::Options::=--force-confold" dist-upgrade""") - self.model.run_on_unit.assert_has_calls([ - mock.call(_unit, 'sudo apt update'), - mock.call(_unit, dist_upgrade_cmd)]) - - def test_do_release_upgrade(self): - _unit = "app/2" - generic_utils.do_release_upgrade(_unit) - self.subprocess.check_call.assert_called_once_with( - ['juju', 'ssh', _unit, 'sudo', 'DEBIAN_FRONTEND=noninteractive', - 'do-release-upgrade', '-f', 'DistUpgradeViewNonInteractive']) - - def test_wrap_do_release_upgrade(self): - self.patch_object(generic_utils, "do_release_upgrade") - self.patch_object(generic_utils, "run_via_ssh") - self.patch_object(generic_utils.model, "scp_to_unit") - _unit = "app/2" - _from_series = "xenial" - _to_series = "bionic" - _workaround_script = "scriptname" - _files = ["filename", _workaround_script] - _scp_calls = [] - _run_calls = [ - mock.call(_unit, _workaround_script)] - for filename in _files: - _scp_calls.append(mock.call(_unit, filename, filename)) - generic_utils.wrap_do_release_upgrade( - _unit, to_series=_to_series, from_series=_from_series, - workaround_script=_workaround_script, files=_files) - self.scp_to_unit.assert_has_calls(_scp_calls) - self.run_via_ssh.assert_has_calls(_run_calls) - self.do_release_upgrade.assert_called_once_with(_unit) - def test_reboot(self): _unit = "app/2" generic_utils.reboot(_unit) @@ -237,149 +198,6 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.set_application_config.assert_called_once_with( _application, {_origin: _pocket}) - def test_series_upgrade(self): - self.patch_object(generic_utils.model, "block_until_all_units_idle") - self.patch_object(generic_utils.model, "block_until_unit_wl_status") - self.patch_object(generic_utils.model, "prepare_series_upgrade") - self.patch_object(generic_utils.model, "complete_series_upgrade") - self.patch_object(generic_utils.model, "set_series") - self.patch_object(generic_utils, "set_origin") - self.patch_object(generic_utils, "wrap_do_release_upgrade") - self.patch_object(generic_utils, "reboot") - _unit = "app/2" - _application = "app" - _machine_num = "4" - _from_series = "xenial" - _to_series = "bionic" - _origin = "source" - _files = ["filename", "scriptname"] - _workaround_script = "scriptname" - generic_utils.series_upgrade( - _unit, _machine_num, origin=_origin, - to_series=_to_series, from_series=_from_series, - workaround_script=_workaround_script, files=_files) - self.block_until_all_units_idle.called_with() - self.prepare_series_upgrade.assert_called_once_with( - _machine_num, to_series=_to_series) - self.wrap_do_release_upgrade.assert_called_once_with( - _unit, to_series=_to_series, from_series=_from_series, - workaround_script=_workaround_script, files=_files) - self.complete_series_upgrade.assert_called_once_with(_machine_num) - self.set_series.assert_called_once_with(_application, _to_series) - self.set_origin.assert_called_once_with(_application, _origin) - self.reboot.assert_called_once_with(_unit) - - def test_series_upgrade_application_pause_peers_and_subordinates(self): - self.patch_object(generic_utils.model, "run_action") - self.patch_object(generic_utils, "series_upgrade") - _application = "app" - _from_series = "xenial" - _to_series = "bionic" - _origin = "source" - _files = ["filename", "scriptname"] - _workaround_script = "scriptname" - _completed_machines = [] - # Peers and Subordinates - _run_action_calls = [ - mock.call("{}-hacluster/1".format(_application), - "pause", action_params={}), - mock.call("{}/1".format(_application), "pause", action_params={}), - mock.call("{}-hacluster/2".format(_application), - "pause", action_params={}), - mock.call("{}/2".format(_application), "pause", action_params={}), - ] - _series_upgrade_calls = [] - for machine_num in ("0", "1", "2"): - _series_upgrade_calls.append( - mock.call("{}/{}".format(_application, machine_num), - machine_num, origin=_origin, - from_series=_from_series, to_series=_to_series, - workaround_script=_workaround_script, files=_files, - post_upgrade_functions=None), - ) - - # Pause primary peers and subordinates - generic_utils.series_upgrade_application( - _application, origin=_origin, - to_series=_to_series, from_series=_from_series, - pause_non_leader_primary=True, - pause_non_leader_subordinate=True, - completed_machines=_completed_machines, - workaround_script=_workaround_script, files=_files), - self.run_action.assert_has_calls(_run_action_calls) - self.series_upgrade.assert_has_calls(_series_upgrade_calls) - - def test_series_upgrade_application_pause_subordinates(self): - self.patch_object(generic_utils.model, "run_action") - self.patch_object(generic_utils, "series_upgrade") - _application = "app" - _from_series = "xenial" - _to_series = "bionic" - _origin = "source" - _files = ["filename", "scriptname"] - _workaround_script = "scriptname" - _completed_machines = [] - # Subordinates only - _run_action_calls = [ - mock.call("{}-hacluster/1".format(_application), - "pause", action_params={}), - mock.call("{}-hacluster/2".format(_application), - "pause", action_params={}), - ] - _series_upgrade_calls = [] - - for machine_num in ("0", "1", "2"): - _series_upgrade_calls.append( - mock.call("{}/{}".format(_application, machine_num), - machine_num, origin=_origin, - from_series=_from_series, to_series=_to_series, - workaround_script=_workaround_script, files=_files, - post_upgrade_functions=None), - ) - - # Pause subordinates - generic_utils.series_upgrade_application( - _application, origin=_origin, - to_series=_to_series, from_series=_from_series, - pause_non_leader_primary=False, - pause_non_leader_subordinate=True, - completed_machines=_completed_machines, - workaround_script=_workaround_script, files=_files), - self.run_action.assert_has_calls(_run_action_calls) - self.series_upgrade.assert_has_calls(_series_upgrade_calls) - - def test_series_upgrade_application_no_pause(self): - self.patch_object(generic_utils.model, "run_action") - self.patch_object(generic_utils, "series_upgrade") - _application = "app" - _from_series = "xenial" - _to_series = "bionic" - _origin = "source" - _series_upgrade_calls = [] - _files = ["filename", "scriptname"] - _workaround_script = "scriptname" - _completed_machines = [] - - for machine_num in ("0", "1", "2"): - _series_upgrade_calls.append( - mock.call("{}/{}".format(_application, machine_num), - machine_num, origin=_origin, - from_series=_from_series, to_series=_to_series, - workaround_script=_workaround_script, files=_files, - post_upgrade_functions=None), - ) - - # No Pausiing - generic_utils.series_upgrade_application( - _application, origin=_origin, - to_series=_to_series, from_series=_from_series, - pause_non_leader_primary=False, - pause_non_leader_subordinate=False, - completed_machines=_completed_machines, - workaround_script=_workaround_script, files=_files) - self.run_action.assert_not_called() - self.series_upgrade.assert_has_calls(_series_upgrade_calls) - def test_set_dpkg_non_interactive_on_unit(self): self.patch_object(generic_utils, "model") _unit_name = "app/1" diff --git a/unit_tests/utilities/test_zaza_utilities_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py new file mode 100644 index 0000000..b6f21e9 --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py @@ -0,0 +1,239 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unit_tests.utils as ut_utils +import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.series_upgrade as series_upgrade_utils + +FAKE_STATUS = { + 'can-upgrade-to': '', + 'charm': 'local:trusty/app-136', + 'subordinate-to': [], + 'units': {'app/0': {'leader': True, + 'machine': '0', + 'subordinates': { + 'app-hacluster/0': { + 'charm': 'local:trusty/hacluster-0', + 'leader': True}}}, + 'app/1': {'machine': '1', + 'subordinates': { + 'app-hacluster/1': { + 'charm': 'local:trusty/hacluster-0'}}}, + 'app/2': {'machine': '2', + 'subordinates': { + 'app-hacluster/2': { + 'charm': 'local:trusty/hacluster-0'}}}}} + + +class TestSeriesUpgrade(ut_utils.BaseTestCase): + def setUp(self): + super(TestSeriesUpgrade, self).setUp() + # Patch all subprocess calls + self.patch( + 'zaza.openstack.utilities.generic.subprocess', + new_callable=mock.MagicMock(), + name='subprocess' + ) + self.patch_object(generic_utils, "run_via_ssh") + # Juju Status Object and data + self.juju_status = mock.MagicMock() + self.juju_status.applications.__getitem__.return_value = FAKE_STATUS + self.patch_object(series_upgrade_utils, "model") + self.model.get_status.return_value = self.juju_status + + def test_series_upgrade(self): + self.patch_object( + series_upgrade_utils.model, "block_until_all_units_idle") + self.patch_object( + series_upgrade_utils.model, "block_until_unit_wl_status") + self.patch_object(series_upgrade_utils.model, "prepare_series_upgrade") + self.patch_object( + series_upgrade_utils.model, "complete_series_upgrade") + self.patch_object(series_upgrade_utils.model, "set_series") + self.patch_object(generic_utils, "set_origin") + self.patch_object(series_upgrade_utils, "wrap_do_release_upgrade") + self.patch_object(generic_utils, "reboot") + _unit = "app/2" + _application = "app" + _machine_num = "4" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + series_upgrade_utils.series_upgrade( + _unit, _machine_num, origin=_origin, + to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) + self.block_until_all_units_idle.called_with() + self.prepare_series_upgrade.assert_called_once_with( + _machine_num, to_series=_to_series) + self.wrap_do_release_upgrade.assert_called_once_with( + _unit, to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) + self.complete_series_upgrade.assert_called_once_with(_machine_num) + self.set_series.assert_called_once_with(_application, _to_series) + self.set_origin.assert_called_once_with(_application, _origin) + self.reboot.assert_called_once_with(_unit) + + def test_series_upgrade_application_pause_peers_and_subordinates(self): + self.patch_object(series_upgrade_utils.model, "run_action") + self.patch_object(series_upgrade_utils, "series_upgrade") + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + _completed_machines = [] + # Peers and Subordinates + _run_action_calls = [ + mock.call("{}-hacluster/1".format(_application), + "pause", action_params={}), + mock.call("{}/1".format(_application), "pause", action_params={}), + mock.call("{}-hacluster/2".format(_application), + "pause", action_params={}), + mock.call("{}/2".format(_application), "pause", action_params={}), + ] + _series_upgrade_calls = [] + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files, + post_upgrade_functions=None), + ) + + # Pause primary peers and subordinates + series_upgrade_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + completed_machines=_completed_machines, + workaround_script=_workaround_script, files=_files), + self.run_action.assert_has_calls(_run_action_calls) + self.series_upgrade.assert_has_calls(_series_upgrade_calls) + + def test_series_upgrade_application_pause_subordinates(self): + self.patch_object(series_upgrade_utils.model, "run_action") + self.patch_object(series_upgrade_utils, "series_upgrade") + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + _completed_machines = [] + # Subordinates only + _run_action_calls = [ + mock.call("{}-hacluster/1".format(_application), + "pause", action_params={}), + mock.call("{}-hacluster/2".format(_application), + "pause", action_params={}), + ] + _series_upgrade_calls = [] + + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files, + post_upgrade_functions=None), + ) + + # Pause subordinates + series_upgrade_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=False, + pause_non_leader_subordinate=True, + completed_machines=_completed_machines, + workaround_script=_workaround_script, files=_files), + self.run_action.assert_has_calls(_run_action_calls) + self.series_upgrade.assert_has_calls(_series_upgrade_calls) + + def test_series_upgrade_application_no_pause(self): + self.patch_object(series_upgrade_utils.model, "run_action") + self.patch_object(series_upgrade_utils, "series_upgrade") + _application = "app" + _from_series = "xenial" + _to_series = "bionic" + _origin = "source" + _series_upgrade_calls = [] + _files = ["filename", "scriptname"] + _workaround_script = "scriptname" + _completed_machines = [] + + for machine_num in ("0", "1", "2"): + _series_upgrade_calls.append( + mock.call("{}/{}".format(_application, machine_num), + machine_num, origin=_origin, + from_series=_from_series, to_series=_to_series, + workaround_script=_workaround_script, files=_files, + post_upgrade_functions=None), + ) + + # No Pausiing + series_upgrade_utils.series_upgrade_application( + _application, origin=_origin, + to_series=_to_series, from_series=_from_series, + pause_non_leader_primary=False, + pause_non_leader_subordinate=False, + completed_machines=_completed_machines, + workaround_script=_workaround_script, files=_files) + self.run_action.assert_not_called() + self.series_upgrade.assert_has_calls(_series_upgrade_calls) + + def test_dist_upgrade(self): + _unit = "app/2" + series_upgrade_utils.dist_upgrade(_unit) + dist_upgrade_cmd = ( + """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + self.model.run_on_unit.assert_has_calls([ + mock.call(_unit, 'sudo apt update'), + mock.call(_unit, dist_upgrade_cmd)]) + + def test_do_release_upgrade(self): + _unit = "app/2" + series_upgrade_utils.do_release_upgrade(_unit) + self.run_via_ssh.assert_called_once_with( + _unit, + 'DEBIAN_FRONTEND=noninteractive do-release-upgrade ' + '-f DistUpgradeViewNonInteractive') + + def test_wrap_do_release_upgrade(self): + self.patch_object(series_upgrade_utils, "do_release_upgrade") + self.patch_object(series_upgrade_utils.model, "scp_to_unit") + _unit = "app/2" + _from_series = "xenial" + _to_series = "bionic" + _workaround_script = "scriptname" + _files = ["filename", _workaround_script] + _scp_calls = [] + _run_calls = [ + mock.call(_unit, _workaround_script)] + for filename in _files: + _scp_calls.append(mock.call(_unit, filename, filename)) + series_upgrade_utils.wrap_do_release_upgrade( + _unit, to_series=_to_series, from_series=_from_series, + workaround_script=_workaround_script, files=_files) + self.scp_to_unit.assert_has_calls(_scp_calls) + self.run_via_ssh.assert_has_calls(_run_calls) + self.do_release_upgrade.assert_called_once_with(_unit) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index eb7add6..f94e93d 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -23,7 +23,7 @@ import unittest from zaza import model from zaza.openstack.utilities import ( cli as cli_utils, - generic as generic_utils, + series_upgrade as series_upgrade_utils, ) from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest @@ -95,7 +95,7 @@ class SeriesUpgradeTest(unittest.TestCase): if "mongodb" in applications[application]["charm"]: # Mongodb needs to run series upgrade # on its secondaries first. - generic_utils.series_upgrade_non_leaders_first( + series_upgrade_utils.series_upgrade_non_leaders_first( application, from_series=self.from_series, to_series=self.to_series, @@ -105,7 +105,7 @@ class SeriesUpgradeTest(unittest.TestCase): # The rest are likley APIs use defaults - generic_utils.series_upgrade_application( + series_upgrade_utils.series_upgrade_application( application, pause_non_leader_primary=pause_non_leader_primary, pause_non_leader_subordinate=pause_non_leader_subordinate, diff --git a/zaza/openstack/utilities/charm_upgrade.py b/zaza/openstack/utilities/charm_upgrade.py new file mode 100644 index 0000000..017b642 --- /dev/null +++ b/zaza/openstack/utilities/charm_upgrade.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of functions to support charm upgrade testing.""" diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index c0566b0..065d8ce 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -25,11 +25,6 @@ from zaza import model from zaza.openstack.utilities import juju as juju_utils from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE -from zaza.charm_lifecycle import utils as cl_utils - -SUBORDINATE_PAUSE_RESUME_BLACKLIST = [ - "cinder-ceph", -] def dict_to_yaml(dict_data): @@ -192,255 +187,6 @@ def get_yaml_config(config_file): return yaml.safe_load(open(config_file, 'r').read()) -def run_post_upgrade_functions(post_upgrade_functions): - """Execute list supplied functions. - - :param post_upgrade_functions: List of functions - :type post_upgrade_functions: [function, function, ...] - """ - if post_upgrade_functions: - for func in post_upgrade_functions: - logging.info("Running {}".format(func)) - cl_utils.get_class(func)() - - -def series_upgrade_non_leaders_first(application, from_series="trusty", - to_series="xenial", - completed_machines=[], - post_upgrade_functions=None): - """Series upgrade non leaders first. - - Wrap all the functionality to handle series upgrade for charms - which must have non leaders upgraded first. - - :param application: Name of application to upgrade series - :type application: str - :param from_series: The series from which to upgrade - :type from_series: str - :param to_series: The series to which to upgrade - :type to_series: str - :param completed_machines: List of completed machines which do no longer - require series upgrade. - :type completed_machines: list - :returns: None - :rtype: None - """ - status = model.get_status().applications[application] - leader = None - non_leaders = [] - for unit in status["units"]: - if status["units"][unit].get("leader"): - leader = unit - else: - non_leaders.append(unit) - - # Series upgrade the non-leaders first - for unit in non_leaders: - machine = status["units"][unit]["machine"] - if machine not in completed_machines: - logging.info("Series upgrade non-leader unit: {}" - .format(unit)) - series_upgrade(unit, machine, - from_series=from_series, to_series=to_series, - origin=None, - post_upgrade_functions=post_upgrade_functions) - run_post_upgrade_functions(post_upgrade_functions) - completed_machines.append(machine) - else: - logging.info("Skipping unit: {}. Machine: {} already upgraded. " - .format(unit, machine, application)) - model.block_until_all_units_idle() - - # Series upgrade the leader - machine = status["units"][leader]["machine"] - logging.info("Series upgrade leader: {}".format(leader)) - if machine not in completed_machines: - series_upgrade(leader, machine, - from_series=from_series, to_series=to_series, - origin=None, - post_upgrade_functions=post_upgrade_functions) - completed_machines.append(machine) - else: - logging.info("Skipping unit: {}. Machine: {} already upgraded." - .format(unit, machine, application)) - model.block_until_all_units_idle() - - -def series_upgrade_application(application, pause_non_leader_primary=True, - pause_non_leader_subordinate=True, - from_series="trusty", to_series="xenial", - origin='openstack-origin', - completed_machines=[], - files=None, workaround_script=None, - post_upgrade_functions=None): - """Series upgrade application. - - Wrap all the functionality to handle series upgrade for a given - application. Including pausing non-leader units. - - :param application: Name of application to upgrade series - :type application: str - :param pause_non_leader_primary: Whether the non-leader applications should - be paused - :type pause_non_leader_primary: bool - :param pause_non_leader_subordinate: Whether the non-leader subordinate - hacluster applications should be - paused - :type pause_non_leader_subordinate: bool - :param from_series: The series from which to upgrade - :type from_series: str - :param to_series: The series to which to upgrade - :type to_series: str - :param origin: The configuration setting variable name for changing origin - source. (openstack-origin or source) - :type origin: str - :param completed_machines: List of completed machines which do no longer - require series upgrade. - :type completed_machines: list - :param files: Workaround files to scp to unit under upgrade - :type files: list - :param workaround_script: Workaround script to run during series upgrade - :type workaround_script: str - :returns: None - :rtype: None - """ - status = model.get_status().applications[application] - - # For some applications (percona-cluster) the leader unit must upgrade - # first. For API applications the non-leader haclusters must be paused - # before upgrade. Finally, for some applications this is arbitrary but - # generalized. - leader = None - non_leaders = [] - for unit in status["units"]: - if status["units"][unit].get("leader"): - leader = unit - else: - non_leaders.append(unit) - - # Pause the non-leaders - for unit in non_leaders: - if pause_non_leader_subordinate: - if status["units"][unit].get("subordinates"): - for subordinate in status["units"][unit]["subordinates"]: - _app = subordinate.split('/')[0] - if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: - logging.info("Skipping pausing {} - blacklisted" - .format(subordinate)) - else: - logging.info("Pausing {}".format(subordinate)) - model.run_action( - subordinate, "pause", action_params={}) - if pause_non_leader_primary: - logging.info("Pausing {}".format(unit)) - model.run_action(unit, "pause", action_params={}) - - machine = status["units"][leader]["machine"] - # Series upgrade the leader - logging.info("Series upgrade leader: {}".format(leader)) - if machine not in completed_machines: - series_upgrade(leader, machine, - from_series=from_series, to_series=to_series, - origin=origin, workaround_script=workaround_script, - files=files, - post_upgrade_functions=post_upgrade_functions) - completed_machines.append(machine) - else: - logging.info("Skipping unit: {}. Machine: {} already upgraded." - "But setting origin on the application {}" - .format(unit, machine, application)) - logging.info("Set origin on {}".format(application)) - set_origin(application, origin) - model.block_until_all_units_idle() - - # Series upgrade the non-leaders - for unit in non_leaders: - machine = status["units"][unit]["machine"] - if machine not in completed_machines: - logging.info("Series upgrade non-leader unit: {}" - .format(unit)) - series_upgrade(unit, machine, - from_series=from_series, to_series=to_series, - origin=origin, workaround_script=workaround_script, - files=files, - post_upgrade_functions=post_upgrade_functions) - completed_machines.append(machine) - else: - logging.info("Skipping unit: {}. Machine: {} already upgraded. " - "But setting origin on the application {}" - .format(unit, machine, application)) - logging.info("Set origin on {}".format(application)) - set_origin(application, origin) - model.block_until_all_units_idle() - - -def series_upgrade(unit_name, machine_num, - from_series="trusty", to_series="xenial", - origin='openstack-origin', - files=None, workaround_script=None, - post_upgrade_functions=None): - """Perform series upgrade on a unit. - - :param unit_name: Unit Name - :type unit_name: str - :param machine_num: Machine number - :type machine_num: str - :param from_series: The series from which to upgrade - :type from_series: str - :param to_series: The series to which to upgrade - :type to_series: str - :param origin: The configuration setting variable name for changing origin - source. (openstack-origin or source) - :type origin: str - :param files: Workaround files to scp to unit under upgrade - :type files: list - :param workaround_script: Workaround script to run during series upgrade - :type workaround_script: str - :returns: None - :rtype: None - """ - logging.info("Series upgrade {}".format(unit_name)) - application = unit_name.split('/')[0] - set_dpkg_non_interactive_on_unit(unit_name) - dist_upgrade(unit_name) - model.block_until_all_units_idle() - logging.info("Prepare series upgrade on {}".format(machine_num)) - model.prepare_series_upgrade(machine_num, to_series=to_series) - logging.info("Waiting for workload status 'blocked' on {}" - .format(unit_name)) - model.block_until_unit_wl_status(unit_name, "blocked") - logging.info("Waiting for model idleness") - model.block_until_all_units_idle() - wrap_do_release_upgrade(unit_name, from_series=from_series, - to_series=to_series, files=files, - workaround_script=workaround_script) - logging.info("Reboot {}".format(unit_name)) - reboot(unit_name) - logging.info("Waiting for workload status 'blocked' on {}" - .format(unit_name)) - model.block_until_unit_wl_status(unit_name, "blocked") - logging.info("Waiting for model idleness") - model.block_until_all_units_idle() - logging.info("Set origin on {}".format(application)) - # Allow for charms which have neither source nor openstack-origin - if origin: - set_origin(application, origin) - model.block_until_all_units_idle() - logging.info("Complete series upgrade on {}".format(machine_num)) - model.complete_series_upgrade(machine_num) - model.block_until_all_units_idle() - logging.info("Running run_post_upgrade_functions {}".format( - post_upgrade_functions)) - run_post_upgrade_functions(post_upgrade_functions) - logging.info("Waiting for workload status 'active' on {}" - .format(unit_name)) - model.block_until_unit_wl_status(unit_name, "active") - model.block_until_all_units_idle() - # This step may be performed by juju in the future - logging.info("Set series on {} to {}".format(application, to_series)) - model.set_series(application, to_series) - - def set_origin(application, origin='openstack-origin', pocket='distro'): """Set the configuration option for origin source. @@ -459,46 +205,6 @@ def set_origin(application, origin='openstack-origin', pocket='distro'): model.set_application_config(application, {origin: pocket}) -def wrap_do_release_upgrade(unit_name, from_series="trusty", - to_series="xenial", - files=None, workaround_script=None): - """Wrap do release upgrade. - - In a production environment this step would be run administratively. - For testing purposes we need this automated. - - :param unit_name: Unit Name - :type unit_name: str - :param from_series: The series from which to upgrade - :type from_series: str - :param to_series: The series to which to upgrade - :type to_series: str - :param files: Workaround files to scp to unit under upgrade - :type files: list - :param workaround_script: Workaround script to run during series upgrade - :type workaround_script: str - :returns: None - :rtype: None - """ - # Pre upgrade hacks - # There are a few necessary hacks to accomplish an automated upgrade - # to overcome some packaging bugs. - # Copy scripts - if files: - logging.info("SCP files") - for _file in files: - logging.info("SCP {}".format(_file)) - model.scp_to_unit(unit_name, _file, os.path.basename(_file)) - - # Run Script - if workaround_script: - logging.info("Running workaround script") - run_via_ssh(unit_name, workaround_script) - - # Actually do the do_release_upgrade - do_release_upgrade(unit_name) - - def run_via_ssh(unit_name, cmd): """Run command on unit via ssh. @@ -521,26 +227,6 @@ def run_via_ssh(unit_name, cmd): logging.warn(e) -def dist_upgrade(unit_name): - """Run dist-upgrade on unit after update package db. - - :param unit_name: Unit Name - :type unit_name: str - :returns: None - :rtype: None - """ - logging.info('Updating package db ' + unit_name) - update_cmd = 'sudo apt update' - model.run_on_unit(unit_name, update_cmd) - - logging.info('Updating existing packages ' + unit_name) - dist_upgrade_cmd = ( - """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ - """-o "Dpkg::Options::=--force-confdef" """ - """-o "Dpkg::Options::=--force-confold" dist-upgrade""") - model.run_on_unit(unit_name, dist_upgrade_cmd) - - def check_commands_on_units(commands, units): """Check that all commands in a list exit zero on all units in a list. @@ -566,26 +252,6 @@ def check_commands_on_units(commands, units): return None -def do_release_upgrade(unit_name): - """Run do-release-upgrade noninteractive. - - :param unit_name: Unit Name - :type unit_name: str - :returns: None - :rtype: None - """ - logging.info('Upgrading ' + unit_name) - # NOTE: It is necessary to run this via juju ssh rather than juju run due - # to timeout restrictions and error handling. - cmd = ['juju', 'ssh', unit_name, 'sudo', 'DEBIAN_FRONTEND=noninteractive', - 'do-release-upgrade', '-f', 'DistUpgradeViewNonInteractive'] - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError as e: - logging.warn("Failed do-release-upgrade for {}".format(unit_name)) - logging.warn(e) - - def reboot(unit_name): """Reboot unit. diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py index 3c4aa8f..5d5c729 100755 --- a/zaza/openstack/utilities/openstack_upgrade.py +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -22,20 +22,10 @@ import zaza.openstack.utilities.juju as juju_utils import zaza.model from zaza import sync_wrapper - -SERVICE_GROUPS = { - 'Core Identity': ['keystone'], - 'Storage': [ - 'ceph-mon', 'ceph-osd', 'ceph-fs', 'ceph-radosgw', 'swift-proxy', - 'swift-storage'], - 'Control Plane': [ - 'aodh', 'barbican', 'ceilometer', 'cinder', 'designate', - 'designate-bind', 'glance', 'gnocchi', 'heat', 'manila', - 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', - 'nova-cloud-controller', 'openstack-dashboard'], - 'Compute': ['nova-compute']} - -UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] +from zaza.openstack.utilities.upgrade_utils import ( + SERVICE_GROUPS, + UPGRADE_EXCLUDE_LIST, +) async def async_pause_units(units, model_name=None): diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py new file mode 100644 index 0000000..e495c64 --- /dev/null +++ b/zaza/openstack/utilities/series_upgrade.py @@ -0,0 +1,479 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of functions for testing series upgrade.""" + +import logging +import os + +from zaza import model +from zaza.charm_lifecycle import utils as cl_utils +import zaza.openstack.utilities.generic as os_utils + + +SUBORDINATE_PAUSE_RESUME_BLACKLIST = [ + "cinder-ceph", +] + + +def run_post_upgrade_functions(post_upgrade_functions): + """Execute list supplied functions. + + :param post_upgrade_functions: List of functions + :type post_upgrade_functions: [function, function, ...] + """ + if post_upgrade_functions: + for func in post_upgrade_functions: + logging.info("Running {}".format(func)) + cl_utils.get_class(func)() + + +def series_upgrade_non_leaders_first(application, from_series="trusty", + to_series="xenial", + completed_machines=[], + post_upgrade_functions=None): + """Series upgrade non leaders first. + + Wrap all the functionality to handle series upgrade for charms + which must have non leaders upgraded first. + + :param application: Name of application to upgrade series + :type application: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type completed_machines: list + :returns: None + :rtype: None + """ + status = model.get_status().applications[application] + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Series upgrade the non-leaders first + for unit in non_leaders: + machine = status["units"][unit]["machine"] + if machine not in completed_machines: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + series_upgrade(unit, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + run_post_upgrade_functions(post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + .format(unit, machine, application)) + model.block_until_all_units_idle() + + # Series upgrade the leader + machine = status["units"][leader]["machine"] + logging.info("Series upgrade leader: {}".format(leader)) + if machine not in completed_machines: + series_upgrade(leader, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded." + .format(unit, machine, application)) + model.block_until_all_units_idle() + + +def series_upgrade_application(application, pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + from_series="trusty", to_series="xenial", + origin='openstack-origin', + completed_machines=[], + files=None, workaround_script=None, + post_upgrade_functions=None): + """Series upgrade application. + + Wrap all the functionality to handle series upgrade for a given + application. Including pausing non-leader units. + + :param application: Name of application to upgrade series + :type application: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type completed_machines: list + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + status = model.get_status().applications[application] + + # For some applications (percona-cluster) the leader unit must upgrade + # first. For API applications the non-leader haclusters must be paused + # before upgrade. Finally, for some applications this is arbitrary but + # generalized. + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Pause the non-leaders + for unit in non_leaders: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + _app = subordinate.split('/')[0] + if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: + logging.info("Skipping pausing {} - blacklisted" + .format(subordinate)) + else: + logging.info("Pausing {}".format(subordinate)) + model.run_action( + subordinate, "pause", action_params={}) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + model.run_action(unit, "pause", action_params={}) + + machine = status["units"][leader]["machine"] + # Series upgrade the leader + logging.info("Series upgrade leader: {}".format(leader)) + if machine not in completed_machines: + series_upgrade(leader, machine, + from_series=from_series, to_series=to_series, + origin=origin, workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded." + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + os_utils.set_origin(application, origin) + model.block_until_all_units_idle() + + # Series upgrade the non-leaders + for unit in non_leaders: + machine = status["units"][unit]["machine"] + if machine not in completed_machines: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + series_upgrade(unit, machine, + from_series=from_series, to_series=to_series, + origin=origin, workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + os_utils.set_origin(application, origin) + model.block_until_all_units_idle() + + +def series_upgrade(unit_name, machine_num, + from_series="trusty", to_series="xenial", + origin='openstack-origin', + files=None, workaround_script=None, + post_upgrade_functions=None): + """Perform series upgrade on a unit. + + :param unit_name: Unit Name + :type unit_name: str + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + logging.info("Series upgrade {}".format(unit_name)) + application = unit_name.split('/')[0] + os_utils.set_dpkg_non_interactive_on_unit(unit_name) + dist_upgrade(unit_name) + model.block_until_all_units_idle() + logging.info("Prepare series upgrade on {}".format(machine_num)) + model.prepare_series_upgrade(machine_num, to_series=to_series) + logging.info("Waiting for workload status 'blocked' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "blocked") + logging.info("Waiting for model idleness") + model.block_until_all_units_idle() + wrap_do_release_upgrade(unit_name, from_series=from_series, + to_series=to_series, files=files, + workaround_script=workaround_script) + logging.info("Reboot {}".format(unit_name)) + os_utils.reboot(unit_name) + logging.info("Waiting for workload status 'blocked' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "blocked") + logging.info("Waiting for model idleness") + model.block_until_all_units_idle() + logging.info("Set origin on {}".format(application)) + # Allow for charms which have neither source nor openstack-origin + if origin: + os_utils.set_origin(application, origin) + model.block_until_all_units_idle() + logging.info("Complete series upgrade on {}".format(machine_num)) + model.complete_series_upgrade(machine_num) + model.block_until_all_units_idle() + logging.info("Running run_post_upgrade_functions {}".format( + post_upgrade_functions)) + run_post_upgrade_functions(post_upgrade_functions) + logging.info("Waiting for workload status 'active' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "active") + model.block_until_all_units_idle() + # This step may be performed by juju in the future + logging.info("Set series on {} to {}".format(application, to_series)) + model.set_series(application, to_series) + + +async def async_series_upgrade(unit_name, machine_num, + from_series="trusty", to_series="xenial", + origin='openstack-origin', + files=None, workaround_script=None, + post_upgrade_functions=None): + """Perform series upgrade on a unit. + + :param unit_name: Unit Name + :type unit_name: str + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + logging.info("Series upgrade {}".format(unit_name)) + application = unit_name.split('/')[0] + await os_utils.async_set_dpkg_non_interactive_on_unit(unit_name) + await async_dist_upgrade(unit_name) + await model.async_block_until_all_units_idle() + logging.info("Prepare series upgrade on {}".format(machine_num)) + await model.async_prepare_series_upgrade(machine_num, to_series=to_series) + logging.info("Waiting for workload status 'blocked' on {}" + .format(unit_name)) + await model.async_block_until_unit_wl_status(unit_name, "blocked") + logging.info("Waiting for model idleness") + await model.async_block_until_all_units_idle() + await async_wrap_do_release_upgrade(unit_name, from_series=from_series, + to_series=to_series, files=files, + workaround_script=workaround_script) + logging.info("Reboot {}".format(unit_name)) + os_utils.reboot(unit_name) + logging.info("Waiting for workload status 'blocked' on {}" + .format(unit_name)) + await model.async_block_until_unit_wl_status(unit_name, "blocked") + logging.info("Waiting for model idleness") + await model.async_block_until_all_units_idle() + logging.info("Set origin on {}".format(application)) + # Allow for charms which have neither source nor openstack-origin + if origin: + await os_utils.async_set_origin(application, origin) + await model.async_block_until_all_units_idle() + logging.info("Complete series upgrade on {}".format(machine_num)) + await model.async_complete_series_upgrade(machine_num) + await model.async_block_until_all_units_idle() + logging.info("Running run_post_upgrade_functions {}".format( + post_upgrade_functions)) + run_post_upgrade_functions(post_upgrade_functions) + logging.info("Waiting for workload status 'active' on {}" + .format(unit_name)) + await model.async_block_until_unit_wl_status(unit_name, "active") + await model.async_block_until_all_units_idle() + # This step may be performed by juju in the future + logging.info("Set series on {} to {}".format(application, to_series)) + await model.async_set_series(application, to_series) + +def wrap_do_release_upgrade(unit_name, from_series="trusty", + to_series="xenial", + files=None, workaround_script=None): + """Wrap do release upgrade. + + In a production environment this step would be run administratively. + For testing purposes we need this automated. + + :param unit_name: Unit Name + :type unit_name: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + # Pre upgrade hacks + # There are a few necessary hacks to accomplish an automated upgrade + # to overcome some packaging bugs. + # Copy scripts + if files: + logging.info("SCP files") + for _file in files: + logging.info("SCP {}".format(_file)) + model.scp_to_unit(unit_name, _file, os.path.basename(_file)) + + # Run Script + if workaround_script: + logging.info("Running workaround script") + os_utils.run_via_ssh(unit_name, workaround_script) + + # Actually do the do_release_upgrade + do_release_upgrade(unit_name) + + +async def async_wrap_do_release_upgrade(unit_name, from_series="trusty", + to_series="xenial", + files=None, workaround_script=None): + """Wrap do release upgrade. + + In a production environment this step would be run administratively. + For testing purposes we need this automated. + + :param unit_name: Unit Name + :type unit_name: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + # Pre upgrade hacks + # There are a few necessary hacks to accomplish an automated upgrade + # to overcome some packaging bugs. + # Copy scripts + if files: + logging.info("SCP files") + for _file in files: + logging.info("SCP {}".format(_file)) + await model.async_scp_to_unit(unit_name, _file, os.path.basename(_file)) + + # Run Script + if workaround_script: + logging.info("Running workaround script") + os_utils.run_via_ssh(unit_name, workaround_script) + + # Actually do the do_release_upgrade + do_release_upgrade(unit_name) + + +def dist_upgrade(unit_name): + """Run dist-upgrade on unit after update package db. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Updating package db ' + unit_name) + update_cmd = 'sudo apt update' + model.run_on_unit(unit_name, update_cmd) + + logging.info('Updating existing packages ' + unit_name) + dist_upgrade_cmd = ( + """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + model.run_on_unit(unit_name, dist_upgrade_cmd) + + +async def async_dist_upgrade(unit_name): + """Run dist-upgrade on unit after update package db. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Updating package db ' + unit_name) + update_cmd = 'sudo apt update' + await model.async_run_on_unit(unit_name, update_cmd) + + logging.info('Updating existing packages ' + unit_name) + dist_upgrade_cmd = ( + """sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + await model.async_run_on_unit(unit_name, dist_upgrade_cmd) + + +def do_release_upgrade(unit_name): + """Run do-release-upgrade noninteractive. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Upgrading ' + unit_name) + # NOTE: It is necessary to run this via juju ssh rather than juju run due + # to timeout restrictions and error handling. + os_utils.run_via_ssh( + unit_name, + 'DEBIAN_FRONTEND=noninteractive ' + 'do-release-upgrade -f DistUpgradeViewNonInteractive') diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py new file mode 100644 index 0000000..445d529 --- /dev/null +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -0,0 +1,30 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of functions to support upgrade testing.""" + + +SERVICE_GROUPS = { + 'Core Identity': ['keystone'], + 'Storage': [ + 'ceph-mon', 'ceph-osd', 'ceph-fs', 'ceph-radosgw', 'swift-proxy', + 'swift-storage'], + 'Control Plane': [ + 'aodh', 'barbican', 'ceilometer', 'cinder', 'designate', + 'designate-bind', 'glance', 'gnocchi', 'heat', 'manila', + 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', + 'nova-cloud-controller', 'openstack-dashboard'], + 'Compute': ['nova-compute']} + +UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] From 8450216f7287afae480763fe7738a17484f4e2c8 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 17 Mar 2020 17:07:56 +0100 Subject: [PATCH 340/898] Create parallel (async) series upgrade --- requirements.txt | 1 - .../test_zaza_utilities_openstack_upgrade.py | 32 -- .../test_zaza_utilities_upgrade_utils.py | 112 +++++++ .../charm_tests/series_upgrade/tests.py | 190 ++++++++++- zaza/openstack/utilities/generic.py | 81 +++++ zaza/openstack/utilities/openstack_upgrade.py | 85 +---- zaza/openstack/utilities/series_upgrade.py | 297 +++++++++++++++++- zaza/openstack/utilities/upgrade_utils.py | 109 ++++++- 8 files changed, 754 insertions(+), 153 deletions(-) create mode 100644 unit_tests/utilities/test_zaza_utilities_upgrade_utils.py diff --git a/requirements.txt b/requirements.txt index 68f8f71..cbf31df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ aiounittest -asyncio async_generator boto3 juju diff --git a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py index f711bb2..bed43b3 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py @@ -172,38 +172,6 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): {'source': 'new-src'}, model_name=None) - def test__extract_charm_name_from_url(self): - self.assertEqual( - openstack_upgrade._extract_charm_name_from_url( - 'local:bionic/heat-12'), - 'heat') - self.assertEqual( - openstack_upgrade._extract_charm_name_from_url( - 'cs:bionic/heat-12'), - 'heat') - self.assertEqual( - openstack_upgrade._extract_charm_name_from_url('cs:heat'), - 'heat') - - def test_get_upgrade_candidates(self): - expect = copy.deepcopy(self.juju_status.applications) - del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST - del expect['ntp'] # Filter as it has no source option - del expect['neutron-openvswitch'] # Filter as it is a subordinates - self.assertEqual( - openstack_upgrade.get_upgrade_candidates(), - expect) - - def test_get_upgrade_groups(self): - self.assertEqual( - openstack_upgrade.get_upgrade_groups(), - { - 'Compute': ['nova-compute'], - 'Control Plane': ['cinder'], - 'Core Identity': [], - 'Storage': [], - 'sweep_up': []}) - def test_is_action_upgradable(self): self.assertTrue( openstack_upgrade.is_action_upgradable('cinder')) diff --git a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py new file mode 100644 index 0000000..e1d5ff8 --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py @@ -0,0 +1,112 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import mock + +import unit_tests.utils as ut_utils +import zaza.openstack.utilities.upgrade_utils as openstack_upgrade + + +class TestUpgradeUtils(ut_utils.BaseTestCase): + def setUp(self): + super(TestUpgradeUtils, self).setUp() + self.patch_object( + openstack_upgrade.zaza.model, + "get_units") + self.juju_status = mock.MagicMock() + self.patch_object( + openstack_upgrade.zaza.model, + "get_status", + return_value=self.juju_status) + self.patch_object( + openstack_upgrade.zaza.model, + "get_application_config") + + def _get_application_config(app, model_name=None): + app_config = { + 'ceph-mon': {'verbose': True, 'source': 'old-src'}, + 'neutron-openvswitch': {'verbose': True}, + 'ntp': {'verbose': True}, + 'percona-cluster': {'verbose': True, 'source': 'old-src'}, + 'cinder': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + 'neutron-api': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + 'nova-compute': { + 'verbose': True, + 'openstack-origin': 'old-src', + 'action-managed-upgrade': False}, + } + return app_config[app] + self.get_application_config.side_effect = _get_application_config + self.juju_status.applications = { + 'mydb': { # Filter as it is on UPGRADE_EXCLUDE_LIST + 'charm': 'cs:percona-cluster'}, + 'neutron-openvswitch': { # Filter as it is a subordinates + 'charm': 'cs:neutron-openvswitch', + 'subordinate-to': 'nova-compute'}, + 'ntp': { # Filter as it has no source option + 'charm': 'cs:ntp'}, + 'nova-compute': { + 'charm': 'cs:nova-compute', + 'units': { + 'nova-compute/0': { + 'subordinates': { + 'neutron-openvswitch/2': { + 'charm': 'cs:neutron-openvswitch-22'}}}}}, + 'cinder': { + 'charm': 'cs:cinder-23', + 'units': { + 'cinder/1': { + 'subordinates': { + 'cinder-hacluster/0': { + 'charm': 'cs:hacluster-42'}, + 'cinder-ceph/3': { + 'charm': 'cs:cinder-ceph-2'}}}}}} + + def test_get_upgrade_candidates(self): + expect = copy.deepcopy(self.juju_status.applications) + del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST + del expect['ntp'] # Filter as it has no source option + del expect['neutron-openvswitch'] # Filter as it is a subordinates + self.assertEqual( + openstack_upgrade.get_upgrade_candidates(), + expect) + + def test_get_upgrade_groups(self): + self.assertEqual( + openstack_upgrade.get_upgrade_groups(), + { + 'Core Identity': [], + 'Control Plane': ['cinder'], + 'Data Plane': ['nova-compute'], + 'sweep_up': []}) + + def test_extract_charm_name_from_url(self): + self.assertEqual( + openstack_upgrade.extract_charm_name_from_url( + 'local:bionic/heat-12'), + 'heat') + self.assertEqual( + openstack_upgrade.extract_charm_name_from_url( + 'cs:bionic/heat-12'), + 'heat') + self.assertEqual( + openstack_upgrade.extract_charm_name_from_url('cs:heat'), + 'heat') diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index f94e93d..6f6eb4e 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -16,6 +16,7 @@ """Define class for Series Upgrade.""" +import asyncio import logging import os import unittest @@ -24,6 +25,7 @@ from zaza import model from zaza.openstack.utilities import ( cli as cli_utils, series_upgrade as series_upgrade_utils, + upgrade_utils as upgrade_utils, ) from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest @@ -54,45 +56,45 @@ class SeriesUpgradeTest(unittest.TestCase): pause_non_leader_primary = True post_upgrade_functions = [] # Skip subordinates - if applications[application]["subordinate-to"]: + if app_details["subordinate-to"]: continue - if "easyrsa" in applications[application]["charm"]: + if "easyrsa" in app_details["charm"]: logging.warn("Skipping series upgrade of easyrsa Bug #1850121") continue - if "etcd" in applications[application]["charm"]: + if "etcd" in app_details["charm"]: logging.warn("Skipping series upgrade of easyrsa Bug #1850124") continue - if "percona-cluster" in applications[application]["charm"]: + if "percona-cluster" in app_details["charm"]: origin = "source" pause_non_leader_primary = True pause_non_leader_subordinate = True - if "rabbitmq-server" in applications[application]["charm"]: + if "rabbitmq-server" in app_details["charm"]: origin = "source" pause_non_leader_primary = True pause_non_leader_subordinate = False - if "nova-compute" in applications[application]["charm"]: + if "nova-compute" in app_details["charm"]: pause_non_leader_primary = False pause_non_leader_subordinate = False - if "ceph" in applications[application]["charm"]: + if "ceph" in app_details["charm"]: origin = "source" pause_non_leader_primary = False pause_non_leader_subordinate = False - if "designate-bind" in applications[application]["charm"]: + if "designate-bind" in app_details["charm"]: origin = None - if "tempest" in applications[application]["charm"]: + if "tempest" in app_details["charm"]: origin = None - if "memcached" in applications[application]["charm"]: + if "memcached" in app_details["charm"]: origin = None pause_non_leader_primary = False pause_non_leader_subordinate = False - if "vault" in applications[application]["charm"]: + if "vault" in app_details["charm"]: origin = None pause_non_leader_primary = False pause_non_leader_subordinate = True post_upgrade_functions = [ ('zaza.openstack.charm_tests.vault.setup.' 'mojo_unseal_by_unit')] - if "mongodb" in applications[application]["charm"]: + if "mongodb" in app_details["charm"]: # Mongodb needs to run series upgrade # on its secondaries first. series_upgrade_utils.series_upgrade_non_leaders_first( @@ -117,7 +119,7 @@ class SeriesUpgradeTest(unittest.TestCase): files=self.files, post_upgrade_functions=post_upgrade_functions) - if "rabbitmq-server" in applications[application]["charm"]: + if "rabbitmq-server" in app_details["charm"]: logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( @@ -126,7 +128,7 @@ class SeriesUpgradeTest(unittest.TestCase): action_params={}) model.block_until_all_units_idle() - if "percona-cluster" in applications[application]["charm"]: + if "percona-cluster" in app_details["charm"]: logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( @@ -215,5 +217,165 @@ class XenialBionicSeriesUpgrade(SeriesUpgradeTest): cls.to_series = "bionic" +class ParallelSeriesUpgradeTest(unittest.TestCase): + """Class to encapsulate Sereis Upgrade Tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + cli_utils.setup_logging() + cls.from_series = None + cls.to_series = None + cls.workaround_script = None + cls.files = [] + + def test_200_run_series_upgrade(self): + """Run series upgrade.""" + # Set Feature Flag + os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" + upgrade_groups = upgrade_utils.get_upgrade_groups() + applications = model.get_status().applications + upgrade_groups['support'] = [ + app for app in upgrade_utils.UPGRADE_EXCLUDE_LIST + if app in applications.keys()] + upgrade_groups['deferred'] = [] + completed_machines = [] + deferred_applications = [] + for group_name, group in upgrade_groups.items(): + logging.warn("About to upgrade {} ({})".format(group_name, group)) + upgrade_group = [] + for application, app_details in applications.items(): + # Defaults + origin = "openstack-origin" + pause_non_leader_subordinate = True + pause_non_leader_primary = True + post_upgrade_functions = [] + name = upgrade_utils.extract_charm_name_from_url( + app_details['charm']) + if name not in group and application not in group: + if group_name is not "deferred" and \ + name not in upgrade_groups['deferred']: + upgrade_groups['deferred'].append(name) + continue + if group_name is not "deferred" and \ + name in upgrade_groups['deferred']: + upgrade_groups['deferred'].remove(name) + # Skip subordinates + if app_details["subordinate-to"]: + continue + if "easyrsa" in app_details["charm"]: + logging.warn( + "Skipping series upgrade of easyrsa Bug #1850121") + continue + if "etcd" in app_details["charm"]: + logging.warn( + "Skipping series upgrade of easyrsa Bug #1850124") + continue + logging.warn("About to upgrade {}".format(application)) + if "percona-cluster" in app_details["charm"]: + origin = "source" + pause_non_leader_primary = True + pause_non_leader_subordinate = True + if "rabbitmq-server" in app_details["charm"]: + origin = "source" + pause_non_leader_primary = True + pause_non_leader_subordinate = False + if "nova-compute" in app_details["charm"]: + pause_non_leader_primary = False + pause_non_leader_subordinate = False + if "ceph" in app_details["charm"]: + origin = "source" + pause_non_leader_primary = False + pause_non_leader_subordinate = False + if "designate-bind" in app_details["charm"]: + origin = None + if "tempest" in app_details["charm"]: + origin = None + if "memcached" in app_details["charm"]: + origin = None + pause_non_leader_primary = False + pause_non_leader_subordinate = False + if "vault" in app_details["charm"]: + origin = None + pause_non_leader_primary = False + pause_non_leader_subordinate = True + post_upgrade_functions = [ + ('zaza.openstack.charm_tests.vault.setup.' + 'mojo_unseal_by_unit')] + if "mongodb" in app_details["charm"]: + # Mongodb needs to run series upgrade + # on its secondaries first. + upgrade_group.append(series_upgrade_utils.async_series_upgrade_non_leaders_first( + application, + from_series=self.from_series, + to_series=self.to_series, + completed_machines=completed_machines, + post_upgrade_functions=post_upgrade_functions)) + continue + + # The rest are likley APIs use defaults + + upgrade_group.append(series_upgrade_utils.async_series_upgrade_application( + application, + pause_non_leader_primary=pause_non_leader_primary, + pause_non_leader_subordinate=pause_non_leader_subordinate, + from_series=self.from_series, + to_series=self.to_series, + origin=origin, + completed_machines=completed_machines, + workaround_script=self.workaround_script, + files=self.files, + post_upgrade_functions=post_upgrade_functions)) + + asyncio.get_event_loop().run_until_complete( + asyncio.gather(*upgrade_group, return_exceptions=True)) + if "rabbitmq-server" in group: + logging.info( + "Running complete-cluster-series-upgrade action on leader") + model.run_action_on_leader( + 'rabbitmq-server', + 'complete-cluster-series-upgrade', + action_params={}) + model.block_until_all_units_idle() + + if "percona-cluster" in group: + logging.info( + "Running complete-cluster-series-upgrade action on leader") + model.run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) + model.block_until_all_units_idle() + + + +class ParallelTrustyXenialSeriesUpgrade(ParallelSeriesUpgradeTest): + """Trusty to Xenial Series Upgrade. + + Makes no assumptions about what is in the deployment. + """ + + @classmethod + def setUpClass(cls): + """Run setup for Trusty to Xenial Series Upgrades.""" + super(ParallelTrustyXenialSeriesUpgrade, cls).setUpClass() + cls.from_series = "trusty" + cls.to_series = "xenial" + + +class ParallelXenialBionicSeriesUpgrade(ParallelSeriesUpgradeTest): + """Xenial to Bionic Series Upgrade. + + Makes no assumptions about what is in the deployment. + """ + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(ParallelXenialBionicSeriesUpgrade, cls).setUpClass() + cls.from_series = "xenial" + cls.to_series = "bionic" + if __name__ == "__main__": unittest.main() + diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 065d8ce..8e85f60 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -14,6 +14,7 @@ """Collection of functions that did not fit anywhere else.""" +import asyncio import logging import os import socket @@ -205,6 +206,25 @@ def set_origin(application, origin='openstack-origin', pocket='distro'): model.set_application_config(application, {origin: pocket}) +async def async_set_origin(application, origin='openstack-origin', + pocket='distro'): + """Set the configuration option for origin source. + + :param application: Name of application to upgrade series + :type application: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param pocket: Origin source cloud pocket. + i.e. 'distro' or 'cloud:xenial-newton' + :type pocket: str + :returns: None + :rtype: None + """ + logging.info("Set origin on {} to {}".format(application, origin)) + await model.async_set_application_config(application, {origin: pocket}) + + def run_via_ssh(unit_name, cmd): """Run command on unit via ssh. @@ -227,6 +247,28 @@ def run_via_ssh(unit_name, cmd): logging.warn(e) +async def async_run_via_ssh(unit_name, cmd): + """Run command on unit via ssh. + + For executing commands on units when the juju agent is down. + + :param unit_name: Unit Name + :param cmd: Command to execute on remote unit + :type cmd: str + :returns: None + :rtype: None + """ + if "sudo" not in cmd: + # cmd.insert(0, "sudo") + cmd = "sudo {}".format(cmd) + cmd = ['juju', 'ssh', unit_name, cmd] + try: + await check_call(cmd) + except subprocess.CalledProcessError as e: + logging.warn("Failed command {} on {}".format(cmd, unit_name)) + logging.warn(e) + + def check_commands_on_units(commands, units): """Check that all commands in a list exit zero on all units in a list. @@ -270,6 +312,29 @@ def reboot(unit_name): pass +async def async_reboot(unit_name): + """Reboot unit. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + # NOTE: When used with series upgrade the agent will be down. + # Even juju run will not work + await async_run_via_ssh(unit_name, "sudo reboot && exit") + + +async def check_call(cmd): + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + stdout, stderr = await proc.communicate() + if proc.returncode != 0: + raise subprocess.CalledProcessError(proc.returncode, cmd) + + def set_dpkg_non_interactive_on_unit( unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"): """Set dpkg options on unit. @@ -286,6 +351,22 @@ def set_dpkg_non_interactive_on_unit( model.run_on_unit(unit_name, cmd) +async def async_set_dpkg_non_interactive_on_unit( + unit_name, apt_conf_d="/etc/apt/apt.conf.d/50unattended-upgrades"): + """Set dpkg options on unit. + + :param unit_name: Unit Name + :type unit_name: str + :param apt_conf_d: Apt.conf file to update + :type apt_conf_d: str + """ + DPKG_NON_INTERACTIVE = 'DPkg::options { "--force-confdef"; };' + # Check if the option exists. If not, add it to the apt.conf.d file + cmd = ("grep '{option}' {file_name} || echo '{option}' >> {file_name}" + .format(option=DPKG_NON_INTERACTIVE, file_name=apt_conf_d)) + await model.async_run_on_unit(unit_name, cmd) + + def get_process_id_list(unit_name, process_name, expect_success=True): """Get a list of process ID(s). diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py index 5d5c729..87a8159 100755 --- a/zaza/openstack/utilities/openstack_upgrade.py +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -16,7 +16,6 @@ This module contains a number of functions for upgrading OpenStack. """ -import re import logging import zaza.openstack.utilities.juju as juju_utils @@ -25,6 +24,8 @@ from zaza import sync_wrapper from zaza.openstack.utilities.upgrade_utils import ( SERVICE_GROUPS, UPGRADE_EXCLUDE_LIST, + get_upgrade_candidates, + get_upgrade_groups, ) @@ -179,88 +180,6 @@ def set_upgrade_application_config(applications, new_source, model_name=model_name) -def _extract_charm_name_from_url(charm_url): - """Extract the charm name from the charm url. - - E.g. Extract 'heat' from local:bionic/heat-12 - - :param charm_url: Name of model to query. - :type charm_url: str - :returns: Charm name - :rtype: str - """ - charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) - return charm_name.split(':')[-1] - - -def get_upgrade_candidates(model_name=None): - """Extract list of apps from model that can be upgraded. - - :param model_name: Name of model to query. - :type model_name: str - :returns: List of application that can have their payload upgraded. - :rtype: [] - """ - status = zaza.model.get_status(model_name=model_name) - candidates = {} - for app, app_config in status.applications.items(): - # Filter out subordinates - if app_config.get("subordinate-to"): - logging.warning( - "Excluding {} from upgrade, it is a subordinate".format(app)) - continue - - # Filter out charms on the naughty list - charm_name = _extract_charm_name_from_url(app_config['charm']) - if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: - logging.warning( - "Excluding {} from upgrade, on the exclude list".format(app)) - continue - - # Filter out charms that have no source option - charm_options = zaza.model.get_application_config( - app, model_name=model_name).keys() - src_options = ['openstack-origin', 'source'] - if not [x for x in src_options if x in charm_options]: - logging.warning( - "Excluding {} from upgrade, no src option".format(app)) - continue - - candidates[app] = app_config - return candidates - - -def get_upgrade_groups(model_name=None): - """Place apps in the model into their upgrade groups. - - Place apps in the model into their upgrade groups. If an app is deployed - but is not in SERVICE_GROUPS then it is placed in a sweep_up group. - - :param model_name: Name of model to query. - :type model_name: str - :returns: Dict of group lists keyed on group name. - :rtype: {} - """ - apps_in_model = get_upgrade_candidates(model_name=model_name) - - groups = {} - for phase_name, charms in SERVICE_GROUPS.items(): - group = [] - for app, app_config in apps_in_model.items(): - charm_name = _extract_charm_name_from_url(app_config['charm']) - if charm_name in charms: - group.append(app) - groups[phase_name] = group - - sweep_up = [] - for app in apps_in_model: - if not (app in [a for group in groups.values() for a in group]): - sweep_up.append(app) - - groups['sweep_up'] = sweep_up - return groups - - def is_action_upgradable(app, model_name=None): """Can application be upgraded using action managed upgrade method. diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index e495c64..bdd8491 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -101,6 +101,68 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", model.block_until_all_units_idle() +async def async_series_upgrade_non_leaders_first(application, from_series="trusty", + to_series="xenial", + completed_machines=[], + post_upgrade_functions=None): + """Series upgrade non leaders first. + + Wrap all the functionality to handle series upgrade for charms + which must have non leaders upgraded first. + + :param application: Name of application to upgrade series + :type application: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type completed_machines: list + :returns: None + :rtype: None + """ + status = (await model.async_get_status()).applications[application] + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Series upgrade the non-leaders first + for unit in non_leaders: + machine = status["units"][unit]["machine"] + if machine not in completed_machines: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + await async_series_upgrade(unit, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + await async_run_post_upgrade_functions(post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + .format(unit, machine, application)) + await model.async_block_until_all_units_idle() + + # Series upgrade the leader + machine = status["units"][leader]["machine"] + logging.info("Series upgrade leader: {}".format(leader)) + if machine not in completed_machines: + await async_series_upgrade(leader, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded." + .format(unit, machine, application)) + await model.async_block_until_all_units_idle() + + def series_upgrade_application(application, pause_non_leader_primary=True, pause_non_leader_subordinate=True, from_series="trusty", to_series="xenial", @@ -209,6 +271,144 @@ def series_upgrade_application(application, pause_non_leader_primary=True, model.block_until_all_units_idle() +async def async_series_upgrade_application(application, + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + from_series="trusty", + to_series="xenial", + origin='openstack-origin', + completed_machines=None, + files=None, workaround_script=None, + post_upgrade_functions=None): + """Series upgrade application. + + Wrap all the functionality to handle series upgrade for a given + application. Including pausing non-leader units. + + :param application: Name of application to upgrade series + :type application: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param completed_machines: List of completed machines which do no longer + require series upgrade. + :type completed_machines: list + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :returns: None + :rtype: None + """ + if completed_machines is None: + completed_machines = [] + status = (await model.async_get_status()).applications[application] + + # For some applications (percona-cluster) the leader unit must upgrade + # first. For API applications the non-leader haclusters must be paused + # before upgrade. Finally, for some applications this is arbitrary but + # generalized. + leader = None + non_leaders = [] + for unit in status["units"]: + if status["units"][unit].get("leader"): + leader = unit + else: + non_leaders.append(unit) + + # Pause the non-leaders + for unit in non_leaders: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + _app = subordinate.split('/')[0] + if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: + logging.info("Skipping pausing {} - blacklisted" + .format(subordinate)) + else: + logging.info("Pausing {}".format(subordinate)) + await model.async_run_action( + subordinate, "pause", action_params={}) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + await model.async_run_action(unit, "pause", action_params={}) + + machine = status["units"][leader]["machine"] + # Series upgrade the leader + logging.info("Series upgrade leader: {}".format(leader)) + if machine not in completed_machines: + await async_series_upgrade(leader, machine, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded." + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + await os_utils.async_set_origin(application, origin) + await wait_for_unit_idle(unit) + + # Series upgrade the non-leaders + for unit in non_leaders: + machine = status["units"][unit]["machine"] + if machine not in completed_machines: + logging.info("Series upgrade non-leader unit: {}" + .format(unit)) + await async_series_upgrade(unit, machine, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) + else: + logging.info("Skipping unit: {}. Machine: {} already upgraded. " + "But setting origin on the application {}" + .format(unit, machine, application)) + logging.info("Set origin on {}".format(application)) + await os_utils.async_set_origin(application, origin) + await wait_for_unit_idle(unit) + + +async def wait_for_unit_idle(unit_name): + app = unit_name.split('/')[0] + try: + await model.async_block_until(_unit_idle(app, unit_name), + timeout=600) + except concurrent.futures._base.TimeoutError: + raise ModelTimeout("Zaza has timed out waiting on the unit to " + "reach idle state.") + +def _unit_idle(app, unit_name): + async def f(): + x = await get_agent_status(app, unit_name) + return x == "idle" + return f + #return await get_agent_status(app, unit_name) is "idle" + +async def get_agent_status(app, unit_name): + return (await model.async_get_status()). \ + applications[app]['units'][unit_name] \ + ['agent-status']['status'] + + def series_upgrade(unit_name, machine_num, from_series="trusty", to_series="xenial", origin='openstack-origin', @@ -305,42 +505,93 @@ async def async_series_upgrade(unit_name, machine_num, application = unit_name.split('/')[0] await os_utils.async_set_dpkg_non_interactive_on_unit(unit_name) await async_dist_upgrade(unit_name) - await model.async_block_until_all_units_idle() + await wait_for_unit_idle(unit_name) logging.info("Prepare series upgrade on {}".format(machine_num)) - await model.async_prepare_series_upgrade(machine_num, to_series=to_series) + await async_prepare_series_upgrade(machine_num, to_series=to_series) logging.info("Waiting for workload status 'blocked' on {}" .format(unit_name)) await model.async_block_until_unit_wl_status(unit_name, "blocked") - logging.info("Waiting for model idleness") - await model.async_block_until_all_units_idle() + logging.info("Waiting for unit {} idleness".format(unit_name)) + await wait_for_unit_idle(unit_name) await async_wrap_do_release_upgrade(unit_name, from_series=from_series, to_series=to_series, files=files, workaround_script=workaround_script) logging.info("Reboot {}".format(unit_name)) - os_utils.reboot(unit_name) + await os_utils.async_reboot(unit_name) logging.info("Waiting for workload status 'blocked' on {}" .format(unit_name)) await model.async_block_until_unit_wl_status(unit_name, "blocked") - logging.info("Waiting for model idleness") - await model.async_block_until_all_units_idle() + logging.info("Waiting for unit {} idleness".format(unit_name)) + await wait_for_unit_idle(unit_name) logging.info("Set origin on {}".format(application)) # Allow for charms which have neither source nor openstack-origin if origin: await os_utils.async_set_origin(application, origin) - await model.async_block_until_all_units_idle() + await wait_for_unit_idle(unit_name) logging.info("Complete series upgrade on {}".format(machine_num)) - await model.async_complete_series_upgrade(machine_num) - await model.async_block_until_all_units_idle() + await async_complete_series_upgrade(machine_num) + await wait_for_unit_idle(unit_name) logging.info("Running run_post_upgrade_functions {}".format( post_upgrade_functions)) run_post_upgrade_functions(post_upgrade_functions) logging.info("Waiting for workload status 'active' on {}" .format(unit_name)) await model.async_block_until_unit_wl_status(unit_name, "active") - await model.async_block_until_all_units_idle() + await wait_for_unit_idle(unit_name) # This step may be performed by juju in the future logging.info("Set series on {} to {}".format(application, to_series)) - await model.async_set_series(application, to_series) + await async_set_series(application, to_series) + + +async def async_prepare_series_upgrade(machine_num, to_series="xenial"): + """Execute juju series-upgrade prepare on machine. + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + :param machine_num: Machine number + :type machine_num: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + juju_model = await model.async_get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + machine_num, "prepare", to_series, "--yes"] + logging.info("About to call '{}'".format(cmd)) + await os_utils.check_call(cmd) + + +async def async_complete_series_upgrade(machine_num): + """Execute juju series-upgrade complete on machine. + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + :param machine_num: Machine number + :type machine_num: str + :returns: None + :rtype: None + """ + juju_model = await model.async_get_juju_model() + cmd = ["juju", "upgrade-series", "-m", juju_model, + machine_num, "complete"] + await os_utils.check_call(cmd) + + +async def async_set_series(application, to_series): + """Execute juju set-series complete on application. + NOTE: This is a new feature in juju and not yet in libjuju. + :param application: Name of application to upgrade series + :type application: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + juju_model = await model.async_get_juju_model() + cmd = ["juju", "set-series", "-m", juju_model, + application, to_series] + await os_utils.check_call(cmd) def wrap_do_release_upgrade(unit_name, from_series="trusty", to_series="xenial", @@ -416,10 +667,10 @@ async def async_wrap_do_release_upgrade(unit_name, from_series="trusty", # Run Script if workaround_script: logging.info("Running workaround script") - os_utils.run_via_ssh(unit_name, workaround_script) + await os_utils.async_run_via_ssh(unit_name, workaround_script) # Actually do the do_release_upgrade - do_release_upgrade(unit_name) + await async_do_release_upgrade(unit_name) def dist_upgrade(unit_name): @@ -477,3 +728,21 @@ def do_release_upgrade(unit_name): unit_name, 'DEBIAN_FRONTEND=noninteractive ' 'do-release-upgrade -f DistUpgradeViewNonInteractive') + + +async def async_do_release_upgrade(unit_name): + """Run do-release-upgrade noninteractive. + + :param unit_name: Unit Name + :type unit_name: str + :returns: None + :rtype: None + """ + logging.info('Upgrading ' + unit_name) + # NOTE: It is necessary to run this via juju ssh rather than juju run due + # to timeout restrictions and error handling. + await os_utils.async_run_via_ssh( + unit_name, + 'DEBIAN_FRONTEND=noninteractive ' + 'do-release-upgrade -f DistUpgradeViewNonInteractive') + diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 445d529..8851b6f 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -13,18 +13,109 @@ # limitations under the License. """Collection of functions to support upgrade testing.""" +import re +import logging +import collections +import zaza.model -SERVICE_GROUPS = { - 'Core Identity': ['keystone'], - 'Storage': [ - 'ceph-mon', 'ceph-osd', 'ceph-fs', 'ceph-radosgw', 'swift-proxy', - 'swift-storage'], - 'Control Plane': [ - 'aodh', 'barbican', 'ceilometer', 'cinder', 'designate', +SERVICE_GROUPS = collections.OrderedDict([ + ('Core Identity', ['keystone']), + ('Control Plane', [ + 'aodh', 'barbican', 'ceilometer', 'ceph-mon', 'ceph-fs', + 'ceph-radosgw', 'cinder', 'designate', 'designate-bind', 'glance', 'gnocchi', 'heat', 'manila', 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', - 'nova-cloud-controller', 'openstack-dashboard'], - 'Compute': ['nova-compute']} + 'nova-cloud-controller', 'openstack-dashboard']), + ('Data Plane', [ + 'nova-compute', 'ceph-osd', 'swift-proxy', 'swift-storage']) +]) UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] + +# Series upgrade ordering should be: [ +# UPGRADE_EXCLUDE_LIST, +# SERVICE_GROUPS['Core Identity'], +# SERVICE_GROUPS['Control Plane'] + SERVICE_GROUPS['Storage'], +# SERVICE_GROUPS['Data Plane'], +# ] + +def get_upgrade_candidates(model_name=None): + """Extract list of apps from model that can be upgraded. + + :param model_name: Name of model to query. + :type model_name: str + :returns: List of application that can have their payload upgraded. + :rtype: [] + """ + status = zaza.model.get_status(model_name=model_name) + candidates = {} + for app, app_config in status.applications.items(): + # Filter out subordinates + if app_config.get("subordinate-to"): + logging.warning( + "Excluding {} from upgrade, it is a subordinate".format(app)) + continue + + # Filter out charms on the naughty list + charm_name = extract_charm_name_from_url(app_config['charm']) + if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: + logging.warning( + "Excluding {} from upgrade, on the exclude list".format(app)) + continue + + # Filter out charms that have no source option + charm_options = zaza.model.get_application_config( + app, model_name=model_name).keys() + src_options = ['openstack-origin', 'source'] + if not [x for x in src_options if x in charm_options]: + logging.warning( + "Excluding {} from upgrade, no src option".format(app)) + continue + + candidates[app] = app_config + return candidates + + +def get_upgrade_groups(model_name=None): + """Place apps in the model into their upgrade groups. + + Place apps in the model into their upgrade groups. If an app is deployed + but is not in SERVICE_GROUPS then it is placed in a sweep_up group. + + :param model_name: Name of model to query. + :type model_name: str + :returns: Dict of group lists keyed on group name. + :rtype: collections.OrderedDict + """ + apps_in_model = get_upgrade_candidates(model_name=model_name) + + groups = collections.OrderedDict() + for phase_name, charms in SERVICE_GROUPS.items(): + group = [] + for app, app_config in apps_in_model.items(): + charm_name = extract_charm_name_from_url(app_config['charm']) + if charm_name in charms: + group.append(app) + groups[phase_name] = group + + sweep_up = [] + for app in apps_in_model: + if not (app in [a for group in groups.values() for a in group]): + sweep_up.append(app) + + groups['sweep_up'] = sweep_up + return groups + +def extract_charm_name_from_url(charm_url): + """Extract the charm name from the charm url. + + E.g. Extract 'heat' from local:bionic/heat-12 + + :param charm_url: Name of model to query. + :type charm_url: str + :returns: Charm name + :rtype: str + """ + charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) + return charm_name.split(':')[-1] From 0b92a15da7427005b38977322689434ad19b9aee Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 18 Mar 2020 10:49:29 +0100 Subject: [PATCH 341/898] raise exceptions in the upgrade process --- zaza/openstack/charm_tests/series_upgrade/tests.py | 2 +- zaza/openstack/utilities/generic.py | 4 +++- zaza/openstack/utilities/series_upgrade.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 6f6eb4e..1ade8d0 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -328,7 +328,7 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): post_upgrade_functions=post_upgrade_functions)) asyncio.get_event_loop().run_until_complete( - asyncio.gather(*upgrade_group, return_exceptions=True)) + asyncio.gather(*upgrade_group)) if "rabbitmq-server" in group: logging.info( "Running complete-cluster-series-upgrade action on leader") diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 8e85f60..596feea 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -247,7 +247,7 @@ def run_via_ssh(unit_name, cmd): logging.warn(e) -async def async_run_via_ssh(unit_name, cmd): +async def async_run_via_ssh(unit_name, cmd, raise_exceptions=False): """Run command on unit via ssh. For executing commands on units when the juju agent is down. @@ -267,6 +267,8 @@ async def async_run_via_ssh(unit_name, cmd): except subprocess.CalledProcessError as e: logging.warn("Failed command {} on {}".format(cmd, unit_name)) logging.warn(e) + if raise_exceptions: + raise e def check_commands_on_units(commands, units): diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index bdd8491..c1f2cd5 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -744,5 +744,6 @@ async def async_do_release_upgrade(unit_name): await os_utils.async_run_via_ssh( unit_name, 'DEBIAN_FRONTEND=noninteractive ' - 'do-release-upgrade -f DistUpgradeViewNonInteractive') + 'do-release-upgrade -f DistUpgradeViewNonInteractive', + raise_exceptions=True) From 0a94b97cf412fc5874fb01fa1794936cd2cf7dcd Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 18 Mar 2020 12:55:26 +0100 Subject: [PATCH 342/898] update for style bits --- .../test_zaza_utilities_openstack_upgrade.py | 1 - .../charm_tests/series_upgrade/tests.py | 49 +++++----- zaza/openstack/utilities/generic.py | 7 ++ zaza/openstack/utilities/openstack_upgrade.py | 3 - zaza/openstack/utilities/series_upgrade.py | 92 ++++++++++++------- zaza/openstack/utilities/upgrade_utils.py | 7 +- 6 files changed, 93 insertions(+), 66 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py index bed43b3..4d619c5 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy import mock import unit_tests.utils as ut_utils diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 1ade8d0..9d10c1a 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -49,7 +49,7 @@ class SeriesUpgradeTest(unittest.TestCase): applications = model.get_status().applications completed_machines = [] - for application in applications: + for application, app_details in applications: # Defaults origin = "openstack-origin" pause_non_leader_subordinate = True @@ -240,7 +240,6 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): if app in applications.keys()] upgrade_groups['deferred'] = [] completed_machines = [] - deferred_applications = [] for group_name, group in upgrade_groups.items(): logging.warn("About to upgrade {} ({})".format(group_name, group)) upgrade_group = [] @@ -253,11 +252,11 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): name = upgrade_utils.extract_charm_name_from_url( app_details['charm']) if name not in group and application not in group: - if group_name is not "deferred" and \ + if group_name != "deferred" and \ name not in upgrade_groups['deferred']: upgrade_groups['deferred'].append(name) continue - if group_name is not "deferred" and \ + if group_name != "deferred" and \ name in upgrade_groups['deferred']: upgrade_groups['deferred'].remove(name) # Skip subordinates @@ -305,27 +304,30 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): if "mongodb" in app_details["charm"]: # Mongodb needs to run series upgrade # on its secondaries first. - upgrade_group.append(series_upgrade_utils.async_series_upgrade_non_leaders_first( - application, - from_series=self.from_series, - to_series=self.to_series, - completed_machines=completed_machines, - post_upgrade_functions=post_upgrade_functions)) + upgrade_group.append( + series_upgrade_utils + .async_series_upgrade_non_leaders_first( + application, + from_series=self.from_series, + to_series=self.to_series, + completed_machines=completed_machines, + post_upgrade_functions=post_upgrade_functions)) continue # The rest are likley APIs use defaults - - upgrade_group.append(series_upgrade_utils.async_series_upgrade_application( - application, - pause_non_leader_primary=pause_non_leader_primary, - pause_non_leader_subordinate=pause_non_leader_subordinate, - from_series=self.from_series, - to_series=self.to_series, - origin=origin, - completed_machines=completed_machines, - workaround_script=self.workaround_script, - files=self.files, - post_upgrade_functions=post_upgrade_functions)) + pause = pause_non_leader_subordinate + upgrade_group.append( + series_upgrade_utils.async_series_upgrade_application( + application, + pause_non_leader_primary=pause_non_leader_primary, + pause_non_leader_subordinate=pause, + from_series=self.from_series, + to_series=self.to_series, + origin=origin, + completed_machines=completed_machines, + workaround_script=self.workaround_script, + files=self.files, + post_upgrade_functions=post_upgrade_functions)) asyncio.get_event_loop().run_until_complete( asyncio.gather(*upgrade_group)) @@ -348,7 +350,6 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): model.block_until_all_units_idle() - class ParallelTrustyXenialSeriesUpgrade(ParallelSeriesUpgradeTest): """Trusty to Xenial Series Upgrade. @@ -376,6 +377,6 @@ class ParallelXenialBionicSeriesUpgrade(ParallelSeriesUpgradeTest): cls.from_series = "xenial" cls.to_series = "bionic" + if __name__ == "__main__": unittest.main() - diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 596feea..a63abc5 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -328,6 +328,13 @@ async def async_reboot(unit_name): async def check_call(cmd): + """Asynchronous function to check a subprocess call. + + :param cmd: Command to execute + :type cmd: List[str] + :returns: None + :rtype: None + """ proc = await asyncio.create_subprocess_exec( *cmd, stdout=asyncio.subprocess.PIPE, diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py index 87a8159..9ad4d50 100755 --- a/zaza/openstack/utilities/openstack_upgrade.py +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -22,9 +22,6 @@ import zaza.openstack.utilities.juju as juju_utils import zaza.model from zaza import sync_wrapper from zaza.openstack.utilities.upgrade_utils import ( - SERVICE_GROUPS, - UPGRADE_EXCLUDE_LIST, - get_upgrade_candidates, get_upgrade_groups, ) diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index c1f2cd5..e32a06f 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -14,6 +14,7 @@ """Collection of functions for testing series upgrade.""" +import concurrent import logging import os @@ -101,7 +102,8 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", model.block_until_all_units_idle() -async def async_series_upgrade_non_leaders_first(application, from_series="trusty", +async def async_series_upgrade_non_leaders_first(application, + from_series="trusty", to_series="xenial", completed_machines=[], post_upgrade_functions=None): @@ -137,11 +139,12 @@ async def async_series_upgrade_non_leaders_first(application, from_series="trust if machine not in completed_machines: logging.info("Series upgrade non-leader unit: {}" .format(unit)) - await async_series_upgrade(unit, machine, - from_series=from_series, to_series=to_series, - origin=None, - post_upgrade_functions=post_upgrade_functions) - await async_run_post_upgrade_functions(post_upgrade_functions) + await async_series_upgrade( + unit, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) + run_post_upgrade_functions(post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded. " @@ -152,10 +155,11 @@ async def async_series_upgrade_non_leaders_first(application, from_series="trust machine = status["units"][leader]["machine"] logging.info("Series upgrade leader: {}".format(leader)) if machine not in completed_machines: - await async_series_upgrade(leader, machine, - from_series=from_series, to_series=to_series, - origin=None, - post_upgrade_functions=post_upgrade_functions) + await async_series_upgrade( + leader, machine, + from_series=from_series, to_series=to_series, + origin=None, + post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded." @@ -348,13 +352,14 @@ async def async_series_upgrade_application(application, # Series upgrade the leader logging.info("Series upgrade leader: {}".format(leader)) if machine not in completed_machines: - await async_series_upgrade(leader, machine, - from_series=from_series, - to_series=to_series, - origin=origin, - workaround_script=workaround_script, - files=files, - post_upgrade_functions=post_upgrade_functions) + await async_series_upgrade( + leader, machine, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded." @@ -370,13 +375,14 @@ async def async_series_upgrade_application(application, if machine not in completed_machines: logging.info("Series upgrade non-leader unit: {}" .format(unit)) - await async_series_upgrade(unit, machine, - from_series=from_series, - to_series=to_series, - origin=origin, - workaround_script=workaround_script, - files=files, - post_upgrade_functions=post_upgrade_functions) + await async_series_upgrade( + unit, machine, + from_series=from_series, + to_series=to_series, + origin=origin, + workaround_script=workaround_script, + files=files, + post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded. " @@ -387,26 +393,44 @@ async def async_series_upgrade_application(application, await wait_for_unit_idle(unit) +# TODO: Move these functions into zaza.model async def wait_for_unit_idle(unit_name): + """Wait until the unit's agent is idle. + + :param unit_name: The unit name of the application, ex: mysql/0 + :type unit_name: str + :returns: None + :rtype: None + """ app = unit_name.split('/')[0] try: - await model.async_block_until(_unit_idle(app, unit_name), + await model.async_block_until( + _unit_idle(app, unit_name), timeout=600) except concurrent.futures._base.TimeoutError: - raise ModelTimeout("Zaza has timed out waiting on the unit to " - "reach idle state.") + raise model.ModelTimeout("Zaza has timed out waiting on the unit to " + "reach idle state.") + def _unit_idle(app, unit_name): async def f(): x = await get_agent_status(app, unit_name) return x == "idle" return f - #return await get_agent_status(app, unit_name) is "idle" + async def get_agent_status(app, unit_name): + """Get the current status of the specified unit. + + :param app: The name of the Juju application, ex: mysql + :type app: str + :param unit_name: The unit name of the application, ex: mysql/0 + :type unit_name: str + :returns: The agent status, either active / idle, returned by Juju + :rtype: str + """ return (await model.async_get_status()). \ - applications[app]['units'][unit_name] \ - ['agent-status']['status'] + applications[app]['units'][unit_name]['agent-status']['status'] def series_upgrade(unit_name, machine_num, @@ -545,6 +569,7 @@ async def async_series_upgrade(unit_name, machine_num, async def async_prepare_series_upgrade(machine_num, to_series="xenial"): """Execute juju series-upgrade prepare on machine. + NOTE: This is a new feature in juju behind a feature flag and not yet in libjuju. export JUJU_DEV_FEATURE_FLAGS=upgrade-series @@ -564,6 +589,7 @@ async def async_prepare_series_upgrade(machine_num, to_series="xenial"): async def async_complete_series_upgrade(machine_num): """Execute juju series-upgrade complete on machine. + NOTE: This is a new feature in juju behind a feature flag and not yet in libjuju. export JUJU_DEV_FEATURE_FLAGS=upgrade-series @@ -580,6 +606,7 @@ async def async_complete_series_upgrade(machine_num): async def async_set_series(application, to_series): """Execute juju set-series complete on application. + NOTE: This is a new feature in juju and not yet in libjuju. :param application: Name of application to upgrade series :type application: str @@ -593,6 +620,7 @@ async def async_set_series(application, to_series): application, to_series] await os_utils.check_call(cmd) + def wrap_do_release_upgrade(unit_name, from_series="trusty", to_series="xenial", files=None, workaround_script=None): @@ -662,7 +690,8 @@ async def async_wrap_do_release_upgrade(unit_name, from_series="trusty", logging.info("SCP files") for _file in files: logging.info("SCP {}".format(_file)) - await model.async_scp_to_unit(unit_name, _file, os.path.basename(_file)) + await model.async_scp_to_unit( + unit_name, _file, os.path.basename(_file)) # Run Script if workaround_script: @@ -746,4 +775,3 @@ async def async_do_release_upgrade(unit_name): 'DEBIAN_FRONTEND=noninteractive ' 'do-release-upgrade -f DistUpgradeViewNonInteractive', raise_exceptions=True) - diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 8851b6f..4e2fbf5 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -33,12 +33,6 @@ SERVICE_GROUPS = collections.OrderedDict([ UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] -# Series upgrade ordering should be: [ -# UPGRADE_EXCLUDE_LIST, -# SERVICE_GROUPS['Core Identity'], -# SERVICE_GROUPS['Control Plane'] + SERVICE_GROUPS['Storage'], -# SERVICE_GROUPS['Data Plane'], -# ] def get_upgrade_candidates(model_name=None): """Extract list of apps from model that can be upgraded. @@ -107,6 +101,7 @@ def get_upgrade_groups(model_name=None): groups['sweep_up'] = sweep_up return groups + def extract_charm_name_from_url(charm_url): """Extract the charm name from the charm url. From d4ee838d4b0cefd8c1017290a06d7f13ccc2e0ef Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 19 Mar 2020 12:16:18 +0100 Subject: [PATCH 343/898] some refactoring and updates from review --- .../charm_tests/series_upgrade/tests.py | 16 +--- zaza/openstack/utilities/series_upgrade.py | 41 ++++++++- zaza/openstack/utilities/upgrade_utils.py | 84 ++++++++++++++----- 3 files changed, 104 insertions(+), 37 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 9d10c1a..941e000 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -233,12 +233,8 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): """Run series upgrade.""" # Set Feature Flag os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" - upgrade_groups = upgrade_utils.get_upgrade_groups() + upgrade_groups = upgrade_utils.get_series_upgrade_groups() applications = model.get_status().applications - upgrade_groups['support'] = [ - app for app in upgrade_utils.UPGRADE_EXCLUDE_LIST - if app in applications.keys()] - upgrade_groups['deferred'] = [] completed_machines = [] for group_name, group in upgrade_groups.items(): logging.warn("About to upgrade {} ({})".format(group_name, group)) @@ -249,16 +245,6 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): pause_non_leader_subordinate = True pause_non_leader_primary = True post_upgrade_functions = [] - name = upgrade_utils.extract_charm_name_from_url( - app_details['charm']) - if name not in group and application not in group: - if group_name != "deferred" and \ - name not in upgrade_groups['deferred']: - upgrade_groups['deferred'].append(name) - continue - if group_name != "deferred" and \ - name in upgrade_groups['deferred']: - upgrade_groups['deferred'].remove(name) # Skip subordinates if app_details["subordinate-to"]: continue diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index e32a06f..52e9c49 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -14,6 +14,8 @@ """Collection of functions for testing series upgrade.""" +import collections +import copy import concurrent import logging import os @@ -40,6 +42,34 @@ def run_post_upgrade_functions(post_upgrade_functions): cl_utils.get_class(func)() +def get_charm_settings(): + default = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'upgrade_function': async_series_upgrade_application, + 'post_upgrade_functions': []} + + _app_settings = collections.defaultdict(lambda: default) + exceptions = { + 'rabbitmq-server': { + 'origin': 'source', + 'pause_non_leader_subordinate': False}, + 'percona-cluster': {'origin': 'source'}, + 'nova-compute' : { + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False}, + 'mongodb': { + 'upgrade_function': async_series_upgrade_non_leaders_first, + 'origin': None, + + }} + for key, value in exceptions.items(): + _app_settings[key] = copy.deepcopy(default) + _app_settings[key].update(value) + return _app_settings + + def series_upgrade_non_leaders_first(application, from_series="trusty", to_series="xenial", completed_machines=[], @@ -283,7 +313,8 @@ async def async_series_upgrade_application(application, origin='openstack-origin', completed_machines=None, files=None, workaround_script=None, - post_upgrade_functions=None): + post_upgrade_functions=None, + post_application_upgrade_functions=None): """Series upgrade application. Wrap all the functionality to handle series upgrade for a given @@ -312,6 +343,13 @@ async def async_series_upgrade_application(application, :type files: list :param workaround_script: Workaround script to run during series upgrade :type workaround_script: str + :param post_upgrade_functions: A list of functions to call after upgrading + each unit of an application + :type post_upgrade_functions: List[fn] + :param post_application_upgrade_functions: A list of functions to call + once after updating all units + of an application + :type post_application_upgrade_functions: List[fn] :returns: None :rtype: None """ @@ -391,6 +429,7 @@ async def async_series_upgrade_application(application, logging.info("Set origin on {}".format(application)) await os_utils.async_set_origin(application, origin) await wait_for_unit_idle(unit) + run_post_upgrade_functions(post_application_upgrade_functions) # TODO: Move these functions into zaza.model diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 4e2fbf5..1c19569 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -34,41 +34,54 @@ SERVICE_GROUPS = collections.OrderedDict([ UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] -def get_upgrade_candidates(model_name=None): +def get_upgrade_candidates(model_name=None, filters=None): """Extract list of apps from model that can be upgraded. :param model_name: Name of model to query. :type model_name: str + :param filters: List of filter functions to apply + :type filters: List[fn] :returns: List of application that can have their payload upgraded. :rtype: [] """ + if filters is None: + filters = [] status = zaza.model.get_status(model_name=model_name) candidates = {} for app, app_config in status.applications.items(): - # Filter out subordinates - if app_config.get("subordinate-to"): - logging.warning( - "Excluding {} from upgrade, it is a subordinate".format(app)) - continue + for f in filters: + if f(app, model_name=model_name): + continue + candidates[app] = app_config + return candidates - # Filter out charms on the naughty list - charm_name = extract_charm_name_from_url(app_config['charm']) - if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: - logging.warning( - "Excluding {} from upgrade, on the exclude list".format(app)) - continue - # Filter out charms that have no source option +def _filter_subordinates(app, model_name=None): + if app_config.get("subordinate-to"): + logging.warning( + "Excluding {} from upgrade, it is a subordinate".format(app)) + return True + return False + + +def _filter_openstack_upgrade_list(app, model_name=None): + charm_name = extract_charm_name_from_url(app_config['charm']) + if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: + logging.warning( + "Excluding {} from upgrade, on the exclude list".format(app)) + return True + return False + + +def _filter_non_openstack_services(app, model_name=None): charm_options = zaza.model.get_application_config( app, model_name=model_name).keys() src_options = ['openstack-origin', 'source'] if not [x for x in src_options if x in charm_options]: logging.warning( "Excluding {} from upgrade, no src option".format(app)) - continue - - candidates[app] = app_config - return candidates + return True + return False def get_upgrade_groups(model_name=None): @@ -82,26 +95,55 @@ def get_upgrade_groups(model_name=None): :returns: Dict of group lists keyed on group name. :rtype: collections.OrderedDict """ - apps_in_model = get_upgrade_candidates(model_name=model_name) + apps_in_model = get_upgrade_candidates( + model_name=model_name, + filters=[ + _filter_subordinates, + _filter_openstack_upgrade_list, + _filter_non_openstack_services, + ]) + return _build_service_groups(apps_in_model) + +def get_series_upgrade_groups(model_name=None): + """Place apps in the model into their upgrade groups. + + Place apps in the model into their upgrade groups. If an app is deployed + but is not in SERVICE_GROUPS then it is placed in a sweep_up group. + + :param model_name: Name of model to query. + :type model_name: str + :returns: Dict of group lists keyed on group name. + :rtype: collections.OrderedDict + """ + apps_in_model = get_upgrade_candidates( + model_name=model_name, + filters=[ + _filter_subordinates, + ]) + + return _build_service_groups(apps_in_model) + + +def _build_service_groups(applications): groups = collections.OrderedDict() for phase_name, charms in SERVICE_GROUPS.items(): group = [] - for app, app_config in apps_in_model.items(): + for app, app_config in applications.items(): charm_name = extract_charm_name_from_url(app_config['charm']) if charm_name in charms: group.append(app) groups[phase_name] = group sweep_up = [] - for app in apps_in_model: + for app in applications: if not (app in [a for group in groups.values() for a in group]): sweep_up.append(app) - groups['sweep_up'] = sweep_up return groups + def extract_charm_name_from_url(charm_url): """Extract the charm name from the charm url. From f73b4c1d6d8feaf42d32f0d5b04641322590d9c6 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 19 Mar 2020 14:15:45 +0100 Subject: [PATCH 344/898] fix up unit tests and assumptions --- .../test_zaza_utilities_upgrade_utils.py | 13 +++++++++--- zaza/openstack/utilities/upgrade_utils.py | 20 ++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py index e1d5ff8..3c27074 100644 --- a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py +++ b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py @@ -81,10 +81,8 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): 'charm': 'cs:cinder-ceph-2'}}}}}} def test_get_upgrade_candidates(self): + self.maxDiff = None expect = copy.deepcopy(self.juju_status.applications) - del expect['mydb'] # Filter as it is on UPGRADE_EXCLUDE_LIST - del expect['ntp'] # Filter as it has no source option - del expect['neutron-openvswitch'] # Filter as it is a subordinates self.assertEqual( openstack_upgrade.get_upgrade_candidates(), expect) @@ -98,6 +96,15 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): 'Data Plane': ['nova-compute'], 'sweep_up': []}) + def test_get_series_upgrade_groups(self): + self.assertEqual( + openstack_upgrade.get_series_upgrade_groups(), + { + 'Core Identity': [], + 'Control Plane': ['cinder'], + 'Data Plane': ['nova-compute'], + 'sweep_up': ['mydb', 'ntp']}) + def test_extract_charm_name_from_url(self): self.assertEqual( openstack_upgrade.extract_charm_name_from_url( diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 1c19569..e345eeb 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -49,14 +49,18 @@ def get_upgrade_candidates(model_name=None, filters=None): status = zaza.model.get_status(model_name=model_name) candidates = {} for app, app_config in status.applications.items(): - for f in filters: - if f(app, model_name=model_name): - continue - candidates[app] = app_config + if _include_app(app, app_config, filters, model_name=model_name): + candidates[app] = app_config return candidates -def _filter_subordinates(app, model_name=None): +def _include_app(app, app_config, filters, model_name=None): + for filt in filters: + if filt(app, app_config, model_name=model_name): + return False + return True + +def _filter_subordinates(app, app_config, model_name=None): if app_config.get("subordinate-to"): logging.warning( "Excluding {} from upgrade, it is a subordinate".format(app)) @@ -64,16 +68,18 @@ def _filter_subordinates(app, model_name=None): return False -def _filter_openstack_upgrade_list(app, model_name=None): +def _filter_openstack_upgrade_list(app, app_config, model_name=None): charm_name = extract_charm_name_from_url(app_config['charm']) + print("About to check if {} or {} in {}".format(app, charm_name, UPGRADE_EXCLUDE_LIST)) if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: + print("Excluding {} from upgrade, on the exclude list".format(app)) logging.warning( "Excluding {} from upgrade, on the exclude list".format(app)) return True return False -def _filter_non_openstack_services(app, model_name=None): +def _filter_non_openstack_services(app, app_config, model_name=None): charm_options = zaza.model.get_application_config( app, model_name=model_name).keys() src_options = ['openstack-origin', 'source'] From 540e0ea4fcfb73f9f68a308218bd7c25983c709c Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 19 Mar 2020 17:29:08 +0100 Subject: [PATCH 345/898] tidy up style things, add docstring --- .../test_zaza_utilities_upgrade_utils.py | 39 +++++++++++------- zaza/openstack/utilities/generic.py | 2 + zaza/openstack/utilities/series_upgrade.py | 41 +++++++++++-------- zaza/openstack/utilities/upgrade_utils.py | 28 ++++++------- 4 files changed, 65 insertions(+), 45 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py index 3c27074..1f21b49 100644 --- a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py +++ b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections import copy import mock +import pprint import unit_tests.utils as ut_utils import zaza.openstack.utilities.upgrade_utils as openstack_upgrade @@ -81,29 +83,36 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): 'charm': 'cs:cinder-ceph-2'}}}}}} def test_get_upgrade_candidates(self): - self.maxDiff = None - expect = copy.deepcopy(self.juju_status.applications) + expected = copy.deepcopy(self.juju_status.applications) self.assertEqual( openstack_upgrade.get_upgrade_candidates(), - expect) + expected) def test_get_upgrade_groups(self): + expected = collections.OrderedDict([ + ('Core Identity', []), + ('Control Plane', ['cinder']), + ('Data Plane', ['nova-compute']), + ('sweep_up', [])]) + actual = openstack_upgrade.get_upgrade_groups() + pprint.pprint(expected) + pprint.pprint(actual) self.assertEqual( - openstack_upgrade.get_upgrade_groups(), - { - 'Core Identity': [], - 'Control Plane': ['cinder'], - 'Data Plane': ['nova-compute'], - 'sweep_up': []}) + actual, + expected) def test_get_series_upgrade_groups(self): + expected = collections.OrderedDict([ + ('Core Identity', []), + ('Control Plane', ['cinder']), + ('Data Plane', ['nova-compute']), + ('sweep_up', ['mydb', 'ntp'])]) + actual = openstack_upgrade.get_series_upgrade_groups() + pprint.pprint(expected) + pprint.pprint(actual) self.assertEqual( - openstack_upgrade.get_series_upgrade_groups(), - { - 'Core Identity': [], - 'Control Plane': ['cinder'], - 'Data Plane': ['nova-compute'], - 'sweep_up': ['mydb', 'ntp']}) + actual, + expected) def test_extract_charm_name_from_url(self): self.assertEqual( diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index a63abc5..3a22404 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -341,6 +341,8 @@ async def check_call(cmd): stderr=asyncio.subprocess.PIPE) stdout, stderr = await proc.communicate() if proc.returncode != 0: + logging.warn("STDOUT: {}".format(stdout)) + logging.warn("STDERR: {}".format(stderr)) raise subprocess.CalledProcessError(proc.returncode, cmd) diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 52e9c49..f217b17 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -43,6 +43,12 @@ def run_post_upgrade_functions(post_upgrade_functions): def get_charm_settings(): + """Application series upgrade settings. + + :returns: A Dict of the settings to manage a series upgrade + :rtype: Dict + + """ default = { 'origin': 'openstack-origin', 'pause_non_leader_subordinate': True, @@ -56,7 +62,7 @@ def get_charm_settings(): 'origin': 'source', 'pause_non_leader_subordinate': False}, 'percona-cluster': {'origin': 'source'}, - 'nova-compute' : { + 'nova-compute': { 'pause_non_leader_primary': False, 'pause_non_leader_subordinate': False}, 'mongodb': { @@ -305,16 +311,17 @@ def series_upgrade_application(application, pause_non_leader_primary=True, model.block_until_all_units_idle() -async def async_series_upgrade_application(application, - pause_non_leader_primary=True, - pause_non_leader_subordinate=True, - from_series="trusty", - to_series="xenial", - origin='openstack-origin', - completed_machines=None, - files=None, workaround_script=None, - post_upgrade_functions=None, - post_application_upgrade_functions=None): +async def async_series_upgrade_application( + application, + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + from_series="trusty", + to_series="xenial", + origin='openstack-origin', + completed_machines=None, + files=None, workaround_script=None, + post_upgrade_functions=None, + post_application_upgrade_functions=None): """Series upgrade application. Wrap all the functionality to handle series upgrade for a given @@ -433,11 +440,13 @@ async def async_series_upgrade_application(application, # TODO: Move these functions into zaza.model -async def wait_for_unit_idle(unit_name): +async def wait_for_unit_idle(unit_name, timeout=600): """Wait until the unit's agent is idle. :param unit_name: The unit name of the application, ex: mysql/0 :type unit_name: str + :param timeout: How long to wait before timing out + :type timeout: int :returns: None :rtype: None """ @@ -445,10 +454,10 @@ async def wait_for_unit_idle(unit_name): try: await model.async_block_until( _unit_idle(app, unit_name), - timeout=600) + timeout=timeout) except concurrent.futures._base.TimeoutError: - raise model.ModelTimeout("Zaza has timed out waiting on the unit to " - "reach idle state.") + raise model.ModelTimeout("Zaza has timed out waiting on {} to " + "reach idle state.".format(unit_name)) def _unit_idle(app, unit_name): @@ -593,7 +602,7 @@ async def async_series_upgrade(unit_name, machine_num, await wait_for_unit_idle(unit_name) logging.info("Complete series upgrade on {}".format(machine_num)) await async_complete_series_upgrade(machine_num) - await wait_for_unit_idle(unit_name) + await wait_for_unit_idle(unit_name, timeout=1200) logging.info("Running run_post_upgrade_functions {}".format( post_upgrade_functions)) run_post_upgrade_functions(post_upgrade_functions) diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index e345eeb..c611398 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -60,6 +60,7 @@ def _include_app(app, app_config, filters, model_name=None): return False return True + def _filter_subordinates(app, app_config, model_name=None): if app_config.get("subordinate-to"): logging.warning( @@ -70,7 +71,6 @@ def _filter_subordinates(app, app_config, model_name=None): def _filter_openstack_upgrade_list(app, app_config, model_name=None): charm_name = extract_charm_name_from_url(app_config['charm']) - print("About to check if {} or {} in {}".format(app, charm_name, UPGRADE_EXCLUDE_LIST)) if app in UPGRADE_EXCLUDE_LIST or charm_name in UPGRADE_EXCLUDE_LIST: print("Excluding {} from upgrade, on the exclude list".format(app)) logging.warning( @@ -80,14 +80,14 @@ def _filter_openstack_upgrade_list(app, app_config, model_name=None): def _filter_non_openstack_services(app, app_config, model_name=None): - charm_options = zaza.model.get_application_config( - app, model_name=model_name).keys() - src_options = ['openstack-origin', 'source'] - if not [x for x in src_options if x in charm_options]: - logging.warning( - "Excluding {} from upgrade, no src option".format(app)) - return True - return False + charm_options = zaza.model.get_application_config( + app, model_name=model_name).keys() + src_options = ['openstack-origin', 'source'] + if not [x for x in src_options if x in charm_options]: + logging.warning( + "Excluding {} from upgrade, no src option".format(app)) + return True + return False def get_upgrade_groups(model_name=None): @@ -107,10 +107,11 @@ def get_upgrade_groups(model_name=None): _filter_subordinates, _filter_openstack_upgrade_list, _filter_non_openstack_services, - ]) + ]) return _build_service_groups(apps_in_model) + def get_series_upgrade_groups(model_name=None): """Place apps in the model into their upgrade groups. @@ -124,9 +125,7 @@ def get_series_upgrade_groups(model_name=None): """ apps_in_model = get_upgrade_candidates( model_name=model_name, - filters=[ - _filter_subordinates, - ]) + filters=[_filter_subordinates]) return _build_service_groups(apps_in_model) @@ -146,10 +145,11 @@ def _build_service_groups(applications): if not (app in [a for group in groups.values() for a in group]): sweep_up.append(app) groups['sweep_up'] = sweep_up + for name, group in groups.items(): + group.sort() return groups - def extract_charm_name_from_url(charm_url): """Extract the charm name from the charm url. From a1e5649d127fd97f33683134916420a811dec7e9 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 24 Mar 2020 16:58:37 +0100 Subject: [PATCH 346/898] Refactor if-statement into upgrade config generator --- .../charm_tests/series_upgrade/tests.py | 165 +++++------------- zaza/openstack/utilities/series_upgrade.py | 35 +--- zaza/openstack/utilities/upgrade_utils.py | 101 ++++++++++- 3 files changed, 133 insertions(+), 168 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 941e000..0470c26 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -24,14 +24,29 @@ import unittest from zaza import model from zaza.openstack.utilities import ( cli as cli_utils, - series_upgrade as series_upgrade_utils, upgrade_utils as upgrade_utils, ) from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest +def _filter_easyrsa(app, app_config, model_name=None): + logging.warn("Skipping series upgrade of easyrsa Bug #1850121") + charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) + if "easyrsa" in charm_name: + return True + return False + + +def _filter_etcd(app, app_config, model_name=None): + logging.warn("Skipping series upgrade of easyrsa Bug #1850124") + charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) + if "etcd" in charm_name: + return True + return False + + class SeriesUpgradeTest(unittest.TestCase): - """Class to encapsulate Sereis Upgrade Tests.""" + """Class to encapsulate Series Upgrade Tests.""" @classmethod def setUpClass(cls): @@ -50,75 +65,31 @@ class SeriesUpgradeTest(unittest.TestCase): applications = model.get_status().applications completed_machines = [] for application, app_details in applications: - # Defaults - origin = "openstack-origin" - pause_non_leader_subordinate = True - pause_non_leader_primary = True - post_upgrade_functions = [] # Skip subordinates if app_details["subordinate-to"]: continue if "easyrsa" in app_details["charm"]: - logging.warn("Skipping series upgrade of easyrsa Bug #1850121") + logging.warn( + "Skipping series upgrade of easyrsa Bug #1850121") continue if "etcd" in app_details["charm"]: - logging.warn("Skipping series upgrade of easyrsa Bug #1850124") + logging.warn( + "Skipping series upgrade of easyrsa Bug #1850124") continue - if "percona-cluster" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = True - pause_non_leader_subordinate = True - if "rabbitmq-server" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = True - pause_non_leader_subordinate = False - if "nova-compute" in app_details["charm"]: - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "ceph" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "designate-bind" in app_details["charm"]: - origin = None - if "tempest" in app_details["charm"]: - origin = None - if "memcached" in app_details["charm"]: - origin = None - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "vault" in app_details["charm"]: - origin = None - pause_non_leader_primary = False - pause_non_leader_subordinate = True - post_upgrade_functions = [ - ('zaza.openstack.charm_tests.vault.setup.' - 'mojo_unseal_by_unit')] - if "mongodb" in app_details["charm"]: - # Mongodb needs to run series upgrade - # on its secondaries first. - series_upgrade_utils.series_upgrade_non_leaders_first( - application, - from_series=self.from_series, - to_series=self.to_series, - completed_machines=completed_machines, - post_upgrade_functions=post_upgrade_functions) - continue - - # The rest are likley APIs use defaults - - series_upgrade_utils.series_upgrade_application( + upgrade_config = upgrade_utils.app_config( + app_details['charm'], + async=False) + upgrade_function = upgrade_config.pop('upgrade_function') + logging.warn("About to upgrade {}".format(application)) + upgrade_function( application, - pause_non_leader_primary=pause_non_leader_primary, - pause_non_leader_subordinate=pause_non_leader_subordinate, + **upgrade_config, from_series=self.from_series, to_series=self.to_series, - origin=origin, completed_machines=completed_machines, workaround_script=self.workaround_script, files=self.files, - post_upgrade_functions=post_upgrade_functions) - + ) if "rabbitmq-server" in app_details["charm"]: logging.info( "Running complete-cluster-series-upgrade action on leader") @@ -218,7 +189,7 @@ class XenialBionicSeriesUpgrade(SeriesUpgradeTest): class ParallelSeriesUpgradeTest(unittest.TestCase): - """Class to encapsulate Sereis Upgrade Tests.""" + """Class to encapsulate Series Upgrade Tests.""" @classmethod def setUpClass(cls): @@ -233,88 +204,30 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): """Run series upgrade.""" # Set Feature Flag os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" - upgrade_groups = upgrade_utils.get_series_upgrade_groups() + upgrade_groups = upgrade_utils.get_series_upgrade_groups( + extra_filters=[_filter_etcd, _filter_easyrsa]) applications = model.get_status().applications completed_machines = [] for group_name, group in upgrade_groups.items(): logging.warn("About to upgrade {} ({})".format(group_name, group)) upgrade_group = [] for application, app_details in applications.items(): - # Defaults - origin = "openstack-origin" - pause_non_leader_subordinate = True - pause_non_leader_primary = True - post_upgrade_functions = [] - # Skip subordinates - if app_details["subordinate-to"]: - continue - if "easyrsa" in app_details["charm"]: - logging.warn( - "Skipping series upgrade of easyrsa Bug #1850121") - continue - if "etcd" in app_details["charm"]: - logging.warn( - "Skipping series upgrade of easyrsa Bug #1850124") + if application not in group: continue + upgrade_config = upgrade_utils.app_config(app_details['charm']) + upgrade_function = upgrade_config.pop('upgrade_function') logging.warn("About to upgrade {}".format(application)) - if "percona-cluster" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = True - pause_non_leader_subordinate = True - if "rabbitmq-server" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = True - pause_non_leader_subordinate = False - if "nova-compute" in app_details["charm"]: - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "ceph" in app_details["charm"]: - origin = "source" - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "designate-bind" in app_details["charm"]: - origin = None - if "tempest" in app_details["charm"]: - origin = None - if "memcached" in app_details["charm"]: - origin = None - pause_non_leader_primary = False - pause_non_leader_subordinate = False - if "vault" in app_details["charm"]: - origin = None - pause_non_leader_primary = False - pause_non_leader_subordinate = True - post_upgrade_functions = [ - ('zaza.openstack.charm_tests.vault.setup.' - 'mojo_unseal_by_unit')] - if "mongodb" in app_details["charm"]: - # Mongodb needs to run series upgrade - # on its secondaries first. - upgrade_group.append( - series_upgrade_utils - .async_series_upgrade_non_leaders_first( - application, - from_series=self.from_series, - to_series=self.to_series, - completed_machines=completed_machines, - post_upgrade_functions=post_upgrade_functions)) - continue - - # The rest are likley APIs use defaults - pause = pause_non_leader_subordinate + logging.info("\tConfig: {}".format(upgrade_config)) upgrade_group.append( - series_upgrade_utils.async_series_upgrade_application( + upgrade_function( application, - pause_non_leader_primary=pause_non_leader_primary, - pause_non_leader_subordinate=pause, + **upgrade_config, from_series=self.from_series, to_series=self.to_series, - origin=origin, completed_machines=completed_machines, workaround_script=self.workaround_script, files=self.files, - post_upgrade_functions=post_upgrade_functions)) - + )) asyncio.get_event_loop().run_until_complete( asyncio.gather(*upgrade_group)) if "rabbitmq-server" in group: diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index f217b17..6b5fd2b 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -42,40 +42,6 @@ def run_post_upgrade_functions(post_upgrade_functions): cl_utils.get_class(func)() -def get_charm_settings(): - """Application series upgrade settings. - - :returns: A Dict of the settings to manage a series upgrade - :rtype: Dict - - """ - default = { - 'origin': 'openstack-origin', - 'pause_non_leader_subordinate': True, - 'pause_non_leader_primary': True, - 'upgrade_function': async_series_upgrade_application, - 'post_upgrade_functions': []} - - _app_settings = collections.defaultdict(lambda: default) - exceptions = { - 'rabbitmq-server': { - 'origin': 'source', - 'pause_non_leader_subordinate': False}, - 'percona-cluster': {'origin': 'source'}, - 'nova-compute': { - 'pause_non_leader_primary': False, - 'pause_non_leader_subordinate': False}, - 'mongodb': { - 'upgrade_function': async_series_upgrade_non_leaders_first, - 'origin': None, - - }} - for key, value in exceptions.items(): - _app_settings[key] = copy.deepcopy(default) - _app_settings[key].update(value) - return _app_settings - - def series_upgrade_non_leaders_first(application, from_series="trusty", to_series="xenial", completed_machines=[], @@ -370,6 +336,7 @@ async def async_series_upgrade_application( # generalized. leader = None non_leaders = [] + logging.info("Configuring leader / non leaders for {}".format(application)) for unit in status["units"]: if status["units"][unit].get("leader"): leader = unit diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index c611398..b1a7c50 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -90,7 +90,7 @@ def _filter_non_openstack_services(app, app_config, model_name=None): return False -def get_upgrade_groups(model_name=None): +def get_upgrade_groups(model_name=None, extra_filters=None): """Place apps in the model into their upgrade groups. Place apps in the model into their upgrade groups. If an app is deployed @@ -101,18 +101,28 @@ def get_upgrade_groups(model_name=None): :returns: Dict of group lists keyed on group name. :rtype: collections.OrderedDict """ + filters = [ + _filter_subordinates, + _filter_openstack_upgrade_list, + _filter_non_openstack_services, + ] + if extra_filters: + if isinstance(extra_filters, list): + filters.extend(extra_filters) + elif callable(extra_filters): + filters.append(extra_filters) + else: + raise RuntimeError( + "extra_filters should be a list of " + "callables") apps_in_model = get_upgrade_candidates( model_name=model_name, - filters=[ - _filter_subordinates, - _filter_openstack_upgrade_list, - _filter_non_openstack_services, - ]) + filters=filters,) return _build_service_groups(apps_in_model) -def get_series_upgrade_groups(model_name=None): +def get_series_upgrade_groups(model_name=None, extra_filters=None): """Place apps in the model into their upgrade groups. Place apps in the model into their upgrade groups. If an app is deployed @@ -123,9 +133,19 @@ def get_series_upgrade_groups(model_name=None): :returns: Dict of group lists keyed on group name. :rtype: collections.OrderedDict """ + filters = [_filter_subordinates] + if extra_filters: + if isinstance(extra_filters, list): + filters.extend(extra_filters) + elif callable(extra_filters): + filters.append(extra_filters) + else: + raise RuntimeError( + "extra_filters should be a list of " + "callables") apps_in_model = get_upgrade_candidates( model_name=model_name, - filters=[_filter_subordinates]) + filters=filters) return _build_service_groups(apps_in_model) @@ -162,3 +182,68 @@ def extract_charm_name_from_url(charm_url): """ charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) return charm_name.split(':')[-1] + + +def app_config(charm_name, async=True): + """Returns a dict with the upgrade config for an application. + + :param charm_name: Name of the charm about to upgrade + :type charm_name: str + :param async: Whether the upgreade functions should be async + :type async: bool + :returns: A dicitonary of the upgrade config for the application + :rtype: Dict + """ + if async: + default_upgrade = async_series_upgrade_application + secondary_first_upgrade = async_series_upgrade_non_leaders_first + else: + default_upgrade = series_upgrade_application + secondary_first_upgrade = series_upgrade_non_leaders_first + default = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'upgrade_function': default_upgrade, + 'post_upgrade_functions': []} + _app_settings = collections.defaultdict(lambda: default) + ceph = { + 'origin': "source", + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + } + exceptions = { + 'rabbitmq-server': { + 'origin': 'source', + 'pause_non_leader_subordinate': False, }, + 'percona-cluster': {'origin': 'source', }, + 'nova-compute' : { + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, }, + 'ceph': ceph, + 'ceph-mon': ceph, + 'ceph-osd': ceph, + 'designate-bind': {'origin': None, }, + 'tempest': {'origin': None, }, + 'memcached': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + }, + 'vault': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': True, + 'post_upgrade_functions': [ + ('zaza.openstack.charm_tests.vault.setup.' + 'mojo_unseal_by_unit')] + }, + 'mongodb': { + 'upgrade_function': secondary_first_upgrade, + } + + } + for key, value in exceptions.items(): + _app_settings[key] = copy.deepcopy(default) + _app_settings[key].update(value) + return _app_settings[charm_name] From 71b9d5a6fba1f06f5869726d4853f38c53b285e9 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 25 Mar 2020 08:59:02 +0100 Subject: [PATCH 347/898] lint fixes --- .../charm_tests/series_upgrade/tests.py | 16 +++-- zaza/openstack/utilities/series_upgrade.py | 65 +++++++++++++++++++ zaza/openstack/utilities/upgrade_utils.py | 65 ------------------- 3 files changed, 74 insertions(+), 72 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 0470c26..dc69b05 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -24,23 +24,24 @@ import unittest from zaza import model from zaza.openstack.utilities import ( cli as cli_utils, + series_upgrade as series_upgrade_utils, upgrade_utils as upgrade_utils, ) from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest def _filter_easyrsa(app, app_config, model_name=None): - logging.warn("Skipping series upgrade of easyrsa Bug #1850121") charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) if "easyrsa" in charm_name: + logging.warn("Skipping series upgrade of easyrsa Bug #1850121") return True return False def _filter_etcd(app, app_config, model_name=None): - logging.warn("Skipping series upgrade of easyrsa Bug #1850124") charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) if "etcd" in charm_name: + logging.warn("Skipping series upgrade of easyrsa Bug #1850124") return True return False @@ -76,9 +77,10 @@ class SeriesUpgradeTest(unittest.TestCase): logging.warn( "Skipping series upgrade of easyrsa Bug #1850124") continue - upgrade_config = upgrade_utils.app_config( - app_details['charm'], - async=False) + charm_name = upgrade_utils.extract_charm_name_from_url(app_details['charm']) + upgrade_config = series_upgrade_utils.app_config( + charm_name, + is_async=False) upgrade_function = upgrade_config.pop('upgrade_function') logging.warn("About to upgrade {}".format(application)) upgrade_function( @@ -214,10 +216,10 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): for application, app_details in applications.items(): if application not in group: continue - upgrade_config = upgrade_utils.app_config(app_details['charm']) + charm_name = upgrade_utils.extract_charm_name_from_url(app_details['charm']) + upgrade_config = series_upgrade_utils.app_config(charm_name) upgrade_function = upgrade_config.pop('upgrade_function') logging.warn("About to upgrade {}".format(application)) - logging.info("\tConfig: {}".format(upgrade_config)) upgrade_group.append( upgrade_function( application, diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 6b5fd2b..c530612 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -30,6 +30,71 @@ SUBORDINATE_PAUSE_RESUME_BLACKLIST = [ ] +def app_config(charm_name, is_async=True): + """Return a dict with the upgrade config for an application. + + :param charm_name: Name of the charm about to upgrade + :type charm_name: str + :param async: Whether the upgreade functions should be async + :type async: bool + :returns: A dicitonary of the upgrade config for the application + :rtype: Dict + """ + if is_async: + default_upgrade = async_series_upgrade_application + secondary_first_upgrade = async_series_upgrade_non_leaders_first + else: + default_upgrade = series_upgrade_application + secondary_first_upgrade = series_upgrade_non_leaders_first + default = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'upgrade_function': default_upgrade, + 'post_upgrade_functions': []} + _app_settings = collections.defaultdict(lambda: default) + ceph = { + 'origin': "source", + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + } + exceptions = { + 'rabbitmq-server': { + 'origin': 'source', + 'pause_non_leader_subordinate': False, }, + 'percona-cluster': {'origin': 'source', }, + 'nova-compute': { + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, }, + 'ceph': ceph, + 'ceph-mon': ceph, + 'ceph-osd': ceph, + 'designate-bind': {'origin': None, }, + 'tempest': {'origin': None, }, + 'memcached': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + }, + 'vault': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': True, + 'post_upgrade_functions': [ + ('zaza.openstack.charm_tests.vault.setup.' + 'mojo_unseal_by_unit')] + }, + 'mongodb': { + 'upgrade_function': secondary_first_upgrade, + } + + } + for key, value in exceptions.items(): + _app_settings[key] = copy.deepcopy(default) + _app_settings[key].update(value) + return _app_settings[charm_name] + + def run_post_upgrade_functions(post_upgrade_functions): """Execute list supplied functions. diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index b1a7c50..062e9b1 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -182,68 +182,3 @@ def extract_charm_name_from_url(charm_url): """ charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) return charm_name.split(':')[-1] - - -def app_config(charm_name, async=True): - """Returns a dict with the upgrade config for an application. - - :param charm_name: Name of the charm about to upgrade - :type charm_name: str - :param async: Whether the upgreade functions should be async - :type async: bool - :returns: A dicitonary of the upgrade config for the application - :rtype: Dict - """ - if async: - default_upgrade = async_series_upgrade_application - secondary_first_upgrade = async_series_upgrade_non_leaders_first - else: - default_upgrade = series_upgrade_application - secondary_first_upgrade = series_upgrade_non_leaders_first - default = { - 'origin': 'openstack-origin', - 'pause_non_leader_subordinate': True, - 'pause_non_leader_primary': True, - 'upgrade_function': default_upgrade, - 'post_upgrade_functions': []} - _app_settings = collections.defaultdict(lambda: default) - ceph = { - 'origin': "source", - 'pause_non_leader_primary': False, - 'pause_non_leader_subordinate': False, - } - exceptions = { - 'rabbitmq-server': { - 'origin': 'source', - 'pause_non_leader_subordinate': False, }, - 'percona-cluster': {'origin': 'source', }, - 'nova-compute' : { - 'pause_non_leader_primary': False, - 'pause_non_leader_subordinate': False, }, - 'ceph': ceph, - 'ceph-mon': ceph, - 'ceph-osd': ceph, - 'designate-bind': {'origin': None, }, - 'tempest': {'origin': None, }, - 'memcached': { - 'origin': None, - 'pause_non_leader_primary': False, - 'pause_non_leader_subordinate': False, - }, - 'vault': { - 'origin': None, - 'pause_non_leader_primary': False, - 'pause_non_leader_subordinate': True, - 'post_upgrade_functions': [ - ('zaza.openstack.charm_tests.vault.setup.' - 'mojo_unseal_by_unit')] - }, - 'mongodb': { - 'upgrade_function': secondary_first_upgrade, - } - - } - for key, value in exceptions.items(): - _app_settings[key] = copy.deepcopy(default) - _app_settings[key].update(value) - return _app_settings[charm_name] From 6bf6dcd8e8f82ced47bb67aa16dabc1cc557cc91 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 25 Mar 2020 15:36:22 +0100 Subject: [PATCH 348/898] fix lint --- zaza/openstack/charm_tests/series_upgrade/tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index dc69b05..239681f 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -77,7 +77,8 @@ class SeriesUpgradeTest(unittest.TestCase): logging.warn( "Skipping series upgrade of easyrsa Bug #1850124") continue - charm_name = upgrade_utils.extract_charm_name_from_url(app_details['charm']) + charm_name = upgrade_utils.extract_charm_name_from_url( + app_details['charm']) upgrade_config = series_upgrade_utils.app_config( charm_name, is_async=False) @@ -216,7 +217,8 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): for application, app_details in applications.items(): if application not in group: continue - charm_name = upgrade_utils.extract_charm_name_from_url(app_details['charm']) + charm_name = upgrade_utils.extract_charm_name_from_url( + app_details['charm']) upgrade_config = series_upgrade_utils.app_config(charm_name) upgrade_function = upgrade_config.pop('upgrade_function') logging.warn("About to upgrade {}".format(application)) From d7f7134e584bc06ecaeb99fb08b759be437a6842 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 26 Mar 2020 14:34:14 +0100 Subject: [PATCH 349/898] tidy up some waiting --- zaza/openstack/utilities/series_upgrade.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index c530612..866e019 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -625,13 +625,11 @@ async def async_series_upgrade(unit_name, machine_num, logging.info("Waiting for workload status 'blocked' on {}" .format(unit_name)) await model.async_block_until_unit_wl_status(unit_name, "blocked") - logging.info("Waiting for unit {} idleness".format(unit_name)) - await wait_for_unit_idle(unit_name) - logging.info("Set origin on {}".format(application)) # Allow for charms which have neither source nor openstack-origin if origin: + logging.info("Set origin on {}".format(application)) await os_utils.async_set_origin(application, origin) - await wait_for_unit_idle(unit_name) + await wait_for_unit_idle(unit_name) logging.info("Complete series upgrade on {}".format(machine_num)) await async_complete_series_upgrade(machine_num) await wait_for_unit_idle(unit_name, timeout=1200) From c55aa81079fd9216516300216f246ee8146b9676 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 26 Mar 2020 15:47:51 +0100 Subject: [PATCH 350/898] Pin OpenStack requirements to stable/train Ussuri drops Python 3.5 rules from upper-constraints --- setup.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 8b855d6..0184f72 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ from setuptools.command.test import test as TestCommand version = "0.0.1.dev1" install_require = [ + 'futurist<2.0.0', 'async_generator', 'boto3', 'cryptography', @@ -35,21 +36,21 @@ install_require = [ 'lxml', 'PyYAML', 'tenacity', - 'oslo.config', - 'aodhclient', + 'oslo.config<6.12.0', + 'aodhclient<1.4.0', 'gnocchiclient>=7.0.5,<8.0.0', 'pika>=1.1.0,<2.0.0', 'python-designateclient>=1.5,<3.0.0', - 'python-heatclient', - 'python-glanceclient', - 'python-keystoneclient', - 'python-manilaclient', - 'python-novaclient', - 'python-neutronclient', - 'python-octaviaclient', + 'python-heatclient<2.0.0', + 'python-glanceclient<3.0.0', + 'python-keystoneclient<3.22.0', + 'python-manilaclient<2.0.0', + 'python-novaclient<16.0.0', + 'python-neutronclient<7.0.0', + 'python-octaviaclient<1.11.0', 'python-ceilometerclient', - 'python-cinderclient', - 'python-swiftclient', + 'python-cinderclient<6.0.0', + 'python-swiftclient<3.9.0', 'zaza@git+https://github.com/openstack-charmers/zaza.git#egg=zaza', ] From bf3c3628a10b159674cf7cb8258bd51e89fae1b9 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 27 Mar 2020 13:45:48 +0100 Subject: [PATCH 351/898] Revert "Remove client version from Manila Test" --- zaza/openstack/charm_tests/manila/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index 5cd7e26..8a1fdee 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -31,7 +31,7 @@ class ManilaTests(test_utils.OpenStackBaseTest): """Run class setup for running tests.""" super(ManilaTests, cls).setUpClass() cls.manila_client = manilaclient.Client( - session=cls.keystone_session) + session=cls.keystone_session, client_version='2') def test_manila_api(self): """Test that the Manila API is working.""" From 3f8a09605efa76db258180b9ca85c9b2864439c4 Mon Sep 17 00:00:00 2001 From: Camille Rodriguez <42066843+camille-rodriguez@users.noreply.github.com> Date: Mon, 30 Mar 2020 16:36:50 -0500 Subject: [PATCH 352/898] Add functional tests for charm-keystone-kerberos (#206) * kerberos authentication tests * kerberos authentication setup and test * kerberos authentication setup and test * kerberos authentication setup and test * pep8 corrections * pep8 corrections * remove hardcoded code * add logging * fix keytab location * add logging * add logging and custom exception for kerberos test server errors * syntax list * adding empty keytab in setup * pep8 * Kerberos custom exception * date * fix creation of empty file * logging * wait until workload is active * wait until workload is active * Edit tests for optimal configuration and logging * remove empty lines * add logging to tests * add logging to tests, import subprocess * tests.py need setup.py get_unit_full_hostname function * token retrieval test * add dns to test host as well as keystone * test keystone token * user needs to be the one created in the kerberos system (i.e admin) * openstack token issue to confirm authentication * openstack token issue to confirm authentication * pep8 * cmd has to be formatted differently for subprocess call * fix error * piping password to the kinit prompt * add a verification for the cached kerberos token file * add a verification for the cached kerberos token file * must wait for subprocess to go through to check result * change config to run the kerberos client tests on a ubuntu vm * change config to run the kerberos client tests on a ubuntu vm * change config to run the kerberos client tests on a ubuntu vm * install krb5-client on ubuntu test host * install krb5-client on ubuntu test host * retrieve openstack token * stderr is a dict * adding python-pip and keystoneauth1[kerberos] * adding python-pip and keystoneauth1[kerberos] * adding python-pip and keystoneauth1[kerberos] * add other mandatory packages to test host * pep8 * unused exception, removing * revert change * rename run_all_tests for run_all_configuration_tests * install the python3 packages --- .../charm_tests/kerberos/__init__.py | 21 ++ zaza/openstack/charm_tests/kerberos/setup.py | 228 ++++++++++++++++++ zaza/openstack/charm_tests/kerberos/tests.py | 74 ++++++ 3 files changed, 323 insertions(+) create mode 100644 zaza/openstack/charm_tests/kerberos/__init__.py create mode 100644 zaza/openstack/charm_tests/kerberos/setup.py create mode 100644 zaza/openstack/charm_tests/kerberos/tests.py diff --git a/zaza/openstack/charm_tests/kerberos/__init__.py b/zaza/openstack/charm_tests/kerberos/__init__.py new file mode 100644 index 0000000..e257ccd --- /dev/null +++ b/zaza/openstack/charm_tests/kerberos/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing keystone-kerberos.""" + + +class KerberosConfigurationError(Exception): + """Custom exception for Kerberos test server.""" + + pass diff --git a/zaza/openstack/charm_tests/kerberos/setup.py b/zaza/openstack/charm_tests/kerberos/setup.py new file mode 100644 index 0000000..f58a961 --- /dev/null +++ b/zaza/openstack/charm_tests/kerberos/setup.py @@ -0,0 +1,228 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup for keystone-kerberos tests.""" + +import logging +import tempfile +import zaza.model +from zaza.openstack.utilities import openstack as openstack_utils +from zaza.openstack.charm_tests.kerberos import KerberosConfigurationError + + +def get_unit_full_hostname(unit_name): + """Retrieve the full hostname of a unit.""" + for unit in zaza.model.get_units(unit_name): + result = zaza.model.run_on_unit(unit.entity_id, 'hostname -f') + hostname = result['Stdout'].rstrip() + return hostname + + +def add_empty_resource_file_to_keystone_kerberos(): + """Add an empty resource to keystone kerberos to complete the setup.""" + logging.info('Attaching an empty keystone keytab to the keystone-kerberos' + ' unit') + tmp_file = '/tmp/empty.keytab' + with open(tmp_file, 'w'): + pass + + zaza.model.attach_resource('keystone-kerberos', + 'keystone_keytab', + tmp_file) + logging.info('Waiting for keystone-kerberos unit to be active and idle') + unit_name = zaza.model.get_units('keystone-kerberos')[0].name + zaza.model.block_until_unit_wl_status(unit_name, "active") + zaza.model.block_until_all_units_idle() + + +def add_dns_entry(kerberos_hostname="kerberos.testubuntu.com"): + """Add a dns entry in /etc/hosts for the kerberos test server. + + :param kerberos_hostname: FQDN of Kerberos server + :type kerberos_hostname: string + """ + logging.info('Retrieving kerberos IP and hostname') + kerberos_ip = zaza.model.get_app_ips("kerberos-server")[0] + cmd = "sudo sed -i '/localhost/i\\{}\t{}' /etc/hosts"\ + .format(kerberos_ip, kerberos_hostname) + + app_names = ['keystone', 'ubuntu-test-host'] + for app_name in app_names: + logging.info('Adding dns entry to the {} unit'.format(app_name)) + zaza_unit = zaza.model.get_units(app_name)[0] + zaza.model.run_on_unit(zaza_unit.entity_id, cmd) + + +def configure_keystone_service_in_kerberos(): + """Configure the keystone service in Kerberos. + + A principal needs to be added to the kerberos server to get a keytab for + this service. The keytab is used for the authentication of the keystone + service. + """ + logging.info('Configure keystone service in Kerberos') + unit = zaza.model.get_units('kerberos-server')[0] + keystone_hostname = get_unit_full_hostname('keystone') + commands = ['sudo kadmin.local -q "addprinc -randkey -clearpolicy ' + 'HTTP/{}"'.format(keystone_hostname), + 'sudo kadmin.local -q "ktadd ' + '-k /home/ubuntu/keystone.keytab ' + 'HTTP/{}"'.format(keystone_hostname), + 'sudo chmod 777 /home/ubuntu/keystone.keytab'] + + try: + for command in commands: + logging.info( + 'Sending command to the kerberos-server: {}'.format(command)) + result = zaza.model.run_on_unit(unit.name, command) + if result['Stderr']: + raise KerberosConfigurationError + elif result['Stdout']: + logging.info('Stdout: {}'.format(result['Stdout'])) + except KerberosConfigurationError: + logging.error('An error occured : {}'.format(result['Stderr'])) + + +def retrieve_and_attach_keytab(): + """Retrieve and attach the keytab to the keystone-kerberos unit.""" + kerberos_server = zaza.model.get_units('kerberos-server')[0] + + dump_file = "keystone.keytab" + remote_file = "/home/ubuntu/keystone.keytab" + with tempfile.TemporaryDirectory() as tmpdirname: + tmp_file = "{}/{}".format(tmpdirname, dump_file) + logging.info('Retrieving keystone.keytab from the kerberos server.') + zaza.model.scp_from_unit( + kerberos_server.name, + remote_file, + tmp_file) + logging.info('Attaching the keystone_keytab resource to ' + 'keystone-kerberos') + zaza.model.attach_resource('keystone-kerberos', + 'keystone_keytab', + tmp_file) + + +def openstack_setup_kerberos(): + """Create a test domain, project, and user for kerberos tests.""" + kerberos_domain = 'k8s' + kerberos_project = 'k8s' + kerberos_user = 'admin' + kerberos_password = 'password123' + role = 'admin' + + logging.info('Retrieving a keystone session and client.') + keystone_session = openstack_utils.get_overcloud_keystone_session() + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + logging.info('Creating domain, project and user for Kerberos tests.') + domain = keystone_client.domains.create(kerberos_domain, + description='Kerberos Domain', + enabled=True) + project = keystone_client.projects.create(kerberos_project, + domain, + description='Test project', + enabled=True) + demo_user = keystone_client.users.create(kerberos_user, + domain=domain, + project=project, + password=kerberos_password, + email='demo@demo.com', + description='Demo User', + enabled=True) + admin_role = keystone_client.roles.find(name=role) + keystone_client.roles.grant( + admin_role, + user=demo_user, + project_domain=domain, + project=project + ) + keystone_client.roles.grant( + admin_role, + user=demo_user, + domain=domain + ) + + +def setup_kerberos_configuration_for_test_host(): + """Retrieve the keytab and krb5.conf to setup the ubuntu test host.""" + kerberos_server = zaza.model.get_units('kerberos-server')[0] + ubuntu_test_host = zaza.model.get_units('ubuntu-test-host')[0] + + dump_file = "krb5.keytab" + remote_file = "/etc/krb5.keytab" + host_keytab_path = '/home/ubuntu/krb5.keytab' + with tempfile.TemporaryDirectory() as tmpdirname: + tmp_file = "{}/{}".format(tmpdirname, dump_file) + logging.info("Retrieving {} from {}.".format(remote_file, + kerberos_server.name)) + zaza.model.scp_from_unit( + kerberos_server.name, + remote_file, + tmp_file) + + logging.info("SCP {} to {} on {}.".format(tmp_file, + host_keytab_path, + ubuntu_test_host.name)) + zaza.model.scp_to_unit( + ubuntu_test_host.name, + tmp_file, + host_keytab_path) + + dump_file = "krb5.conf" + remote_file = "/etc/krb5.conf" + temp_krb5_path = "/home/ubuntu/krb5.conf" + with tempfile.TemporaryDirectory() as tmpdirname: + tmp_file = "{}/{}".format(tmpdirname, dump_file) + logging.info("Retrieving {} from {}".format(remote_file, + kerberos_server.name)) + zaza.model.scp_from_unit( + kerberos_server.name, + remote_file, + tmp_file) + + logging.info("SCP {} to {} on {}.".format(tmp_file, + temp_krb5_path, + ubuntu_test_host)) + zaza.model.scp_to_unit( + ubuntu_test_host.name, + tmp_file, + temp_krb5_path) + logging.info('Moving {} to {} on {}.'.format(temp_krb5_path, + remote_file, ubuntu_test_host.name)) + zaza.model.run_on_unit(ubuntu_test_host.name, ('sudo mv {} {}'. + format(temp_krb5_path, remote_file))) + + +def install_apt_packages_on_ubuntu_test_host(): + """Install apt packages on a zaza unit.""" + ubuntu_test_host = zaza.model.get_units('ubuntu-test-host')[0] + packages = ['krb5-user', 'python3-openstackclient', + 'python3-requests-kerberos'] + for package in packages: + logging.info('Installing {}'.format(package)) + result = zaza.model.run_on_unit(ubuntu_test_host.name, + "apt install {} -y".format(package)) + assert result['Code'] == '0', result['Stderr'] + + +def run_all_configuration_steps(): + """Execute all the necessary functions for the tests setup.""" + add_empty_resource_file_to_keystone_kerberos() + add_dns_entry() + configure_keystone_service_in_kerberos() + retrieve_and_attach_keytab() + openstack_setup_kerberos() + setup_kerberos_configuration_for_test_host() + install_apt_packages_on_ubuntu_test_host() diff --git a/zaza/openstack/charm_tests/kerberos/tests.py b/zaza/openstack/charm_tests/kerberos/tests.py new file mode 100644 index 0000000..e122352 --- /dev/null +++ b/zaza/openstack/charm_tests/kerberos/tests.py @@ -0,0 +1,74 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Keystone Kerberos Tests.""" + +import logging + +import zaza.model +from zaza.openstack.charm_tests.kerberos.setup import get_unit_full_hostname +from zaza.openstack.charm_tests.keystone import BaseKeystoneTest +from zaza.openstack.utilities import openstack as openstack_utils + + +class CharmKeystoneKerberosTest(BaseKeystoneTest): + """Charm Keystone Kerberos Test.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Keystone Kerberos charm tests.""" + super(CharmKeystoneKerberosTest, cls).setUpClass() + + def test_keystone_kerberos_authentication(self): + """Validate auth to Openstack through the kerberos method.""" + logging.info('Retrieving a kerberos token with kinit for admin user') + + ubuntu_test_host = zaza.model.get_units('ubuntu-test-host')[0] + result = zaza.model.run_on_unit(ubuntu_test_host.name, + "echo password123 | kinit admin") + assert result['Code'] == '0', result['Stderr'] + + logging.info('Changing token mod for user access') + result = zaza.model.run_on_unit( + ubuntu_test_host.name, + "sudo install -m 777 /tmp/krb5cc_0 /tmp/krb5cc_1000" + ) + assert result['Code'] == '0', result['Stderr'] + + logging.info('Fetching user/project info in Openstack') + domain_name = 'k8s' + project_name = 'k8s' + keystone_session = openstack_utils.get_overcloud_keystone_session() + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + domain_id = keystone_client.domains.find(name=domain_name).id + project_id = keystone_client.projects.find(name=project_name).id + keystone_hostname = get_unit_full_hostname('keystone') + + logging.info('Retrieving an Openstack token to validate auth') + cmd = 'openstack token issue -f value -c id ' \ + '--os-auth-url http://{}:5000/krb/v3 ' \ + '--os-project-id {} ' \ + '--os-project-name {} ' \ + '--os-project-domain-id {} ' \ + '--os-region-name RegionOne ' \ + '--os-interface public ' \ + '--os-identity-api-version 3 ' \ + '--os-auth-type v3kerberos'.format(keystone_hostname, + project_id, + project_name, + domain_id) + + result = zaza.model.run_on_unit(ubuntu_test_host.name, cmd) + assert result['Code'] == '0', result['Stderr'] From 9cb7876e19c35da53893bd893bad67d8984d3951 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 31 Mar 2020 16:22:03 +0000 Subject: [PATCH 353/898] Add BaseCharmTest class without OpenStack setup. The ceph-iscsi charm is a non-openstack api charm that also needs config changed and pause/resume tests. --- zaza/openstack/charm_tests/test_utils.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index f2eebb3..ac8eb27 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -89,8 +89,8 @@ def audit_assertions(action, assert value == "PASS", "Unexpected failure: {}".format(key) -class OpenStackBaseTest(unittest.TestCase): - """Generic helpers for testing OpenStack API charms.""" +class BaseCharmTest(unittest.TestCase): + """Generic helpers for testing charms.""" run_resource_cleanup = False @@ -120,8 +120,6 @@ class OpenStackBaseTest(unittest.TestCase): cls.model_name = cls.model_aliases[model_alias] else: cls.model_name = model.get_juju_model() - cls.keystone_session = openstack_utils.get_overcloud_keystone_session( - model_name=cls.model_name) cls.test_config = lifecycle_utils.get_charm_config(fatal=False) if application_name: cls.application_name = application_name @@ -131,7 +129,6 @@ class OpenStackBaseTest(unittest.TestCase): cls.application_name, model_name=cls.model_name) logging.debug('Leader unit is {}'.format(cls.lead_unit)) - cls.cacert = openstack_utils.get_cacert() def config_current(self, application_name=None, keys=None): """Get Current Config of an application normalized into key-values. @@ -384,3 +381,15 @@ class OpenStackBaseTest(unittest.TestCase): 'running', model_name=self.model_name, pgrep_full=pgrep_full) + + +class OpenStackBaseTest(BaseCharmTest): + """Generic helpers for testing OpenStack API charms.""" + + @classmethod + def setUpClass(cls, application_name=None, model_alias=None): + """Run setup for test class to create common resources.""" + super(OpenStackBaseTest, cls).setUpClass() + cls.keystone_session = openstack_utils.get_overcloud_keystone_session( + model_name=cls.model_name) + cls.cacert = openstack_utils.get_cacert() From 42b0f5fcb1733e45dee4288156c7a37a9f0a95f1 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 1 Apr 2020 08:21:09 +0200 Subject: [PATCH 354/898] Secondary first upgrades should also support the origin argument Closes #210 --- .../test_zaza_utilities_series_upgrade.py | 36 +++++++++++++++++++ zaza/openstack/utilities/series_upgrade.py | 18 +++++++--- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py index b6f21e9..340fc59 100644 --- a/unit_tests/utilities/test_zaza_utilities_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py @@ -237,3 +237,39 @@ class TestSeriesUpgrade(ut_utils.BaseTestCase): self.scp_to_unit.assert_has_calls(_scp_calls) self.run_via_ssh.assert_has_calls(_run_calls) self.do_release_upgrade.assert_called_once_with(_unit) + + def test_app_config_openstack_charm(self): + upgrade = series_upgrade_utils.async_series_upgrade_application + expected = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'upgrade_function': upgrade, + 'post_upgrade_functions': [], + } + config = series_upgrade_utils.app_config('keystone') + self.assertEqual(expected, config) + + def test_app_config_mongo(self): + upgrade = series_upgrade_utils.async_series_upgrade_non_leaders_first + expected = { + 'origin': None, + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'upgrade_function': upgrade, + 'post_upgrade_functions': [], + } + config = series_upgrade_utils.app_config('mongodb') + self.assertEqual(expected, config) + + def test_app_config_ceph(self): + upgrade = series_upgrade_utils.async_series_upgrade_application + expected = { + 'origin': 'source', + 'pause_non_leader_subordinate': False, + 'pause_non_leader_primary': False, + 'upgrade_function': upgrade, + 'post_upgrade_functions': [], + } + config = series_upgrade_utils.app_config('ceph-mon') + self.assertEqual(expected, config) diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 866e019..7f3efa3 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -85,9 +85,9 @@ def app_config(charm_name, is_async=True): 'mojo_unseal_by_unit')] }, 'mongodb': { + 'origin': None, 'upgrade_function': secondary_first_upgrade, } - } for key, value in exceptions.items(): _app_settings[key] = copy.deepcopy(default) @@ -109,6 +109,7 @@ def run_post_upgrade_functions(post_upgrade_functions): def series_upgrade_non_leaders_first(application, from_series="trusty", to_series="xenial", + origin='openstack-origin', completed_machines=[], post_upgrade_functions=None): """Series upgrade non leaders first. @@ -122,6 +123,9 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", :type from_series: str :param to_series: The series to which to upgrade :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str :param completed_machines: List of completed machines which do no longer require series upgrade. :type completed_machines: list @@ -145,7 +149,7 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", .format(unit)) series_upgrade(unit, machine, from_series=from_series, to_series=to_series, - origin=None, + origin=origin, post_upgrade_functions=post_upgrade_functions) run_post_upgrade_functions(post_upgrade_functions) completed_machines.append(machine) @@ -160,7 +164,7 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", if machine not in completed_machines: series_upgrade(leader, machine, from_series=from_series, to_series=to_series, - origin=None, + origin=origin, post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: @@ -172,6 +176,7 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", async def async_series_upgrade_non_leaders_first(application, from_series="trusty", to_series="xenial", + origin='openstack-origin', completed_machines=[], post_upgrade_functions=None): """Series upgrade non leaders first. @@ -185,6 +190,9 @@ async def async_series_upgrade_non_leaders_first(application, :type from_series: str :param to_series: The series to which to upgrade :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str :param completed_machines: List of completed machines which do no longer require series upgrade. :type completed_machines: list @@ -209,7 +217,7 @@ async def async_series_upgrade_non_leaders_first(application, await async_series_upgrade( unit, machine, from_series=from_series, to_series=to_series, - origin=None, + origin=origin, post_upgrade_functions=post_upgrade_functions) run_post_upgrade_functions(post_upgrade_functions) completed_machines.append(machine) @@ -225,7 +233,7 @@ async def async_series_upgrade_non_leaders_first(application, await async_series_upgrade( leader, machine, from_series=from_series, to_series=to_series, - origin=None, + origin=origin, post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: From 588d38d4d7250033565f9898f582987ef449dafe Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 1 Apr 2020 08:58:36 +0000 Subject: [PATCH 355/898] Add custom assertions Add custom assertions to check that remote actions and commands run ok. --- zaza/openstack/charm_tests/test_utils.py | 56 +++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index ac8eb27..c64ff31 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -89,7 +89,53 @@ def audit_assertions(action, assert value == "PASS", "Unexpected failure: {}".format(key) -class BaseCharmTest(unittest.TestCase): +class CharmTestAssertions: + """Custom assertions to support zaza testings.""" + + def assertActionRanOK(self, action): + """Assert that the remote action ran successfully. + + Example usage:: + + self.assertActionRanOK(model.run_action( + unit, + 'pause', + model_name=self.model_name)) + + self.assertActionRanOK(model.run_action_on_leader( + unit, + 'pause', + model_name=self.model_name)) + + :param action: Action object to check. + :type action: juju.action.Action + :raises: AssertionError if the assertion fails. + """ + print("Checking action") + if action.status != 'completed': + msg = ("Action '{name}' exited with status '{status}': " + "'{message}'").format(**action.data) + raise AssertionError(msg) + + def assertRemoteRunOK(self, run_output): + """Use with zaza.model.run_on_unit. + + Example usage:: + + self.assertRemoteRunOK(zaza.model.run_on_unit( + unit, + 'ls /tmp/')) + + :param action: Dict returned from remote run. + :type action: dict + :raises: AssertionError if the assertion fails. + """ + print("Checking remote run") + if int(run_output['Code']) != 0: + raise AssertionError("Command failed: {}".format(run_output)) + + +class BaseCharmTest(unittest.TestCase, CharmTestAssertions): """Generic helpers for testing charms.""" run_resource_cleanup = False @@ -350,10 +396,10 @@ class BaseCharmTest(unittest.TestCase): self.lead_unit, 'active', model_name=self.model_name) - model.run_action( + self.assertActionRanOK(model.run_action( self.lead_unit, 'pause', - model_name=self.model_name) + model_name=self.model_name)) model.block_until_unit_wl_status( self.lead_unit, 'maintenance', @@ -366,10 +412,10 @@ class BaseCharmTest(unittest.TestCase): model_name=self.model_name, pgrep_full=pgrep_full) yield - model.run_action( + self.assertActionRanOK(model.run_action( self.lead_unit, 'resume', - model_name=self.model_name) + model_name=self.model_name)) model.block_until_unit_wl_status( self.lead_unit, 'active', From a3f1230aea5561505225543c72f1a3d692fcc6f1 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 1 Apr 2020 09:08:10 +0000 Subject: [PATCH 356/898] Remove prints --- zaza/openstack/charm_tests/test_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index c64ff31..d94ded7 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -111,7 +111,6 @@ class CharmTestAssertions: :type action: juju.action.Action :raises: AssertionError if the assertion fails. """ - print("Checking action") if action.status != 'completed': msg = ("Action '{name}' exited with status '{status}': " "'{message}'").format(**action.data) @@ -130,7 +129,6 @@ class CharmTestAssertions: :type action: dict :raises: AssertionError if the assertion fails. """ - print("Checking remote run") if int(run_output['Code']) != 0: raise AssertionError("Command failed: {}".format(run_output)) From be8889e581a09d7ebf50c34415aa22c19a0f1623 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 1 Apr 2020 10:18:30 +0000 Subject: [PATCH 357/898] Move new methods out of test class The new methods assertActionRanOK & assertRemoteRunOK are useful in setup as well as tests so it makes sense to move them out of the test class. --- zaza/openstack/charm_tests/test_utils.py | 50 ++---------------------- zaza/openstack/utilities/generic.py | 42 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 47 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index d94ded7..2a22ab1 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -89,51 +89,7 @@ def audit_assertions(action, assert value == "PASS", "Unexpected failure: {}".format(key) -class CharmTestAssertions: - """Custom assertions to support zaza testings.""" - - def assertActionRanOK(self, action): - """Assert that the remote action ran successfully. - - Example usage:: - - self.assertActionRanOK(model.run_action( - unit, - 'pause', - model_name=self.model_name)) - - self.assertActionRanOK(model.run_action_on_leader( - unit, - 'pause', - model_name=self.model_name)) - - :param action: Action object to check. - :type action: juju.action.Action - :raises: AssertionError if the assertion fails. - """ - if action.status != 'completed': - msg = ("Action '{name}' exited with status '{status}': " - "'{message}'").format(**action.data) - raise AssertionError(msg) - - def assertRemoteRunOK(self, run_output): - """Use with zaza.model.run_on_unit. - - Example usage:: - - self.assertRemoteRunOK(zaza.model.run_on_unit( - unit, - 'ls /tmp/')) - - :param action: Dict returned from remote run. - :type action: dict - :raises: AssertionError if the assertion fails. - """ - if int(run_output['Code']) != 0: - raise AssertionError("Command failed: {}".format(run_output)) - - -class BaseCharmTest(unittest.TestCase, CharmTestAssertions): +class BaseCharmTest(unittest.TestCase): """Generic helpers for testing charms.""" run_resource_cleanup = False @@ -394,7 +350,7 @@ class BaseCharmTest(unittest.TestCase, CharmTestAssertions): self.lead_unit, 'active', model_name=self.model_name) - self.assertActionRanOK(model.run_action( + generic_utils.assertActionRanOK(model.run_action( self.lead_unit, 'pause', model_name=self.model_name)) @@ -410,7 +366,7 @@ class BaseCharmTest(unittest.TestCase, CharmTestAssertions): model_name=self.model_name, pgrep_full=pgrep_full) yield - self.assertActionRanOK(model.run_action( + generic_utils.assertActionRanOK(model.run_action( self.lead_unit, 'resume', model_name=self.model_name)) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 3a22404..e342369 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -28,6 +28,48 @@ from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE +def assertActionRanOK(action): + """Assert that the remote action ran successfully. + + Example usage:: + + self.assertActionRanOK(model.run_action( + unit, + 'pause', + model_name=self.model_name)) + + self.assertActionRanOK(model.run_action_on_leader( + unit, + 'pause', + model_name=self.model_name)) + + :param action: Action object to check. + :type action: juju.action.Action + :raises: AssertionError if the assertion fails. + """ + if action.status != 'completed': + msg = ("Action '{name}' exited with status '{status}': " + "'{message}'").format(**action.data) + raise AssertionError(msg) + + +def assertRemoteRunOK(run_output): + """Use with zaza.model.run_on_unit. + + Example usage:: + + self.assertRemoteRunOK(zaza.model.run_on_unit( + unit, + 'ls /tmp/')) + + :param action: Dict returned from remote run. + :type action: dict + :raises: AssertionError if the assertion fails. + """ + if int(run_output['Code']) != 0: + raise AssertionError("Command failed: {}".format(run_output)) + + def dict_to_yaml(dict_data): """Return YAML from dictionary. From c65cbceb19e827af834845166f7673f497ac4133 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 1 Apr 2020 13:22:50 +0000 Subject: [PATCH 358/898] Add support for fqdns to get_unit_hostnames --- zaza/openstack/utilities/generic.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index e342369..9143297 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -113,11 +113,14 @@ def get_network_config(net_topology, ignore_env_vars=False, return net_info -def get_unit_hostnames(units): +def get_unit_hostnames(units, fqdn=False): """Return a dict of juju unit names to hostnames.""" host_names = {} for unit in units: - output = model.run_on_unit(unit.entity_id, 'hostname') + cmd = 'hostname' + if fqdn: + cmd = cmd + ' -f' + output = model.run_on_unit(unit.entity_id, cmd) hostname = output['Stdout'].strip() host_names[unit.entity_id] = hostname return host_names From a822f6995b6f4ae268f03fe670a8e6679379cc35 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 1 Apr 2020 13:45:47 +0000 Subject: [PATCH 359/898] Added unit tests --- .../utilities/test_zaza_utilities_generic.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 0f3ff9c..f1a648f 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -429,6 +429,20 @@ class TestGenericUtils(ut_utils.BaseTestCase): actual = generic_utils.get_unit_hostnames(_units) self.assertEqual(actual, expected) + expected_run_calls = [ + mock.call('testunit/1', 'hostname'), + mock.call('testunit/2', 'hostname')] + self._run.assert_has_calls(expected_run_calls) + + self._run.reset_mock() + self._run.side_effect = [{"Stdout": _hostname1}, + {"Stdout": _hostname2}] + expected_run_calls = [ + mock.call('testunit/1', 'hostname -f'), + mock.call('testunit/2', 'hostname -f')] + + actual = generic_utils.get_unit_hostnames(_units, fqdn=True) + self._run.assert_has_calls(expected_run_calls) def test_port_knock_units(self): self.patch( From ff3e7f64dca5e735e0487782c9f070eb0bb247d0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 26 Feb 2020 12:10:52 +0000 Subject: [PATCH 360/898] Add ceph-iscsi tests --- .../charm_tests/ceph/iscsi/__init__.py | 15 ++ .../openstack/charm_tests/ceph/iscsi/setup.py | 30 ++++ .../openstack/charm_tests/ceph/iscsi/tests.py | 146 ++++++++++++++++++ 3 files changed, 191 insertions(+) create mode 100644 zaza/openstack/charm_tests/ceph/iscsi/__init__.py create mode 100644 zaza/openstack/charm_tests/ceph/iscsi/setup.py create mode 100644 zaza/openstack/charm_tests/ceph/iscsi/tests.py diff --git a/zaza/openstack/charm_tests/ceph/iscsi/__init__.py b/zaza/openstack/charm_tests/ceph/iscsi/__init__.py new file mode 100644 index 0000000..55b06b5 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/iscsi/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ``ceph-iscsi``.""" diff --git a/zaza/openstack/charm_tests/ceph/iscsi/setup.py b/zaza/openstack/charm_tests/ceph/iscsi/setup.py new file mode 100644 index 0000000..8662f8b --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/iscsi/setup.py @@ -0,0 +1,30 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup for ceph iscsi gateway deployments.""" + +import zaza.model + + +def basic_guest_setup(): + """Run basic setup for iscsi guest.""" + unit = zaza.model.get_units('ubuntu')[0] + setup_cmds = [ + "apt install --yes open-iscsi multipath-tools", + "systemctl start iscsi", + "systemctl start iscsid"] + for cmd in setup_cmds: + zaza.model.run_on_unit( + unit.entity_id, + cmd) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py new file mode 100644 index 0000000..89b0d05 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -0,0 +1,146 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulating `ceph-iscsi` testing.""" + +import logging +import tempfile + +import zaza +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.generic as generic_utils + + +class CephISCSIGatewayTest(test_utils.BaseCharmTest): + """Class for `ceph-iscsi` tests.""" + + GW_IQN = "iqn.2003-03.com.canonical.iscsi-gw:iscsi-igw" + POOL_NAME = "iscsi" + + def get_client_initiatorname(self, unit): + """Return the initiatorname for the given unit.""" + generic_utils.assertRemoteRunOK(zaza.model.run_on_unit( + unit, + ('cp /etc/iscsi/initiatorname.iscsi /tmp; ' + 'chmod 644 /tmp/initiatorname.iscsi'))) + with tempfile.TemporaryDirectory() as tmpdirname: + tmp_file = '{}/{}'.format(tmpdirname, 'initiatorname.iscsi') + zaza.model.scp_from_unit( + unit, + '/tmp/initiatorname.iscsi', + tmp_file) + with open(tmp_file, 'r') as stream: + contents = stream.readlines() + initiatorname = None + for line in contents: + if line.startswith('InitiatorName'): + initiatorname = line.split('=')[1].rstrip() + return initiatorname + + def get_ctxt(self): + """Generate a context for running gwcli commands to create a target.""" + gw_units = zaza.model.get_units('ceph-iscsi') + client_units = zaza.model.get_units('ubuntu') + client = client_units[0] + self.get_client_initiatorname(client.entity_id) + primary_gw = gw_units[0] + secondary_gw = gw_units[1] + host_names = generic_utils.get_unit_hostnames(gw_units, fqdn=True) + ctxt = { + 'pool_name': self.POOL_NAME, + 'client_entity_id': client.entity_id, + 'gw_iqn': self.GW_IQN, + 'gw1_ip': primary_gw.public_address, + 'gw1_hostname': host_names[primary_gw.entity_id], + 'gw1_entity_id': primary_gw.entity_id, + 'gw2_ip': secondary_gw.public_address, + 'gw2_hostname': host_names[secondary_gw.entity_id], + 'gw2_entity_id': secondary_gw.entity_id, + 'img_size': '1G', + 'img_name': 'disk_1', + 'chap_username': 'myiscsiusername', + 'chap_password': 'myiscsipassword', + 'chap_creds': 'username={chap_username} password={chap_password}', + 'client_initiatorname': self.get_client_initiatorname( + client.entity_id), + 'gwcli_gw_dir': '/iscsi-targets/{gw_iqn}/gateways', + 'gwcli_hosts_dir': '/iscsi-targets/{gw_iqn}/hosts', + 'gwcli_disk_dir': '/disks', + 'gwcli_client_dir': '{gwcli_hosts_dir}/{client_initiatorname}', + } + return ctxt + + def run_commands(self, unit_name, commands, ctxt): + """Run commands on unit. + + Apply context to commands until all variables have been replaced, then + run the command on the given unit. + """ + for _cmd in commands: + cmd = _cmd.format(**ctxt) + generic_utils.assertRemoteRunOK(zaza.model.run_on_unit( + unit_name, + cmd)) + + def create_iscsi_target(self, ctxt): + """Create target on gateway.""" + generic_utils.assertActionRanOK(zaza.model.run_action_on_leader( + 'ceph-iscsi', + 'create-target', + action_params={ + 'gateway-units': '{} {}'.format( + ctxt['gw1_entity_id'], + ctxt['gw2_entity_id']), + 'iqn': self.GW_IQN, + 'image-size': ctxt['img_size'], + 'image-name': ctxt['img_name'], + 'client-initiatorname': ctxt['client_initiatorname'], + 'client-username': ctxt['chap_username'], + 'client-password': ctxt['chap_password'] + })) + + def mount_iscsi_target(self, ctxt): + """Mount iscsi target on client.""" + base_op_cmd = ('iscsiadm --mode node --targetname {gw_iqn} ' + '--op=update ').format(**ctxt) + setup_cmds = [ + 'iscsiadm -m discovery -t st -p {gw1_ip}', + base_op_cmd + '-n node.session.auth.authmethod -v CHAP', + base_op_cmd + '-n node.session.auth.username -v {chap_username}', + base_op_cmd + '-n node.session.auth.password -v {chap_password}', + 'iscsiadm --mode node --targetname {gw_iqn} --login'] + self.run_commands(ctxt['client_entity_id'], setup_cmds, ctxt) + + def check_client_device(self, ctxt): + """Wait for multipath device to appear on client.""" + async def check_device_present(): + run = await zaza.model.async_run_on_unit( + ctxt['client_entity_id'], + 'ls -l /dev/dm-0') + return '/dev/dm-0' in run['Stdout'] + zaza.model.block_until(check_device_present) + + def test_create_and_mount_volume(self): + """Test creating a target and mounting it on a client.""" + ctxt = self.get_ctxt() + self.create_iscsi_target(ctxt) + self.mount_iscsi_target(ctxt) + self.check_client_device(ctxt) + + def test_pause_resume(self): + """Test pausing and resuming a unit.""" + with self.pause_resume( + ['rbd-target-api', 'rbd-target-gw'], + pgrep_full=True): + logging.info("Testing pause resume") From 3b93ebf4924b62f8250149bc99bf57a24a0362df Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 1 Apr 2020 11:21:00 +0200 Subject: [PATCH 361/898] Add new Ceph test for Prometheus --- zaza/openstack/charm_tests/ceph/tests.py | 30 ++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 2e16617..8b1c947 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -21,6 +21,7 @@ from os import ( listdir, path ) +import requests import tempfile import tenacity @@ -760,3 +761,32 @@ class CephProxyTest(unittest.TestCase): msg = ('cinder-ceph pool restriction was not configured correctly.' ' Found: {}'.format(output)) raise zaza_exceptions.CephPoolNotConfigured(msg) + + +class CephPrometheusTest(unittest.TestCase): + """Test the Ceph <-> Prometheus relation.""" + + def test_prometheus_metrics(self): + """Validate that Prometheus has Ceph metrics.""" + try: + zaza_model.get_application( + 'prometheus2') + except KeyError: + raise unittest.SkipTest('Prometheus not present, skipping test') + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name('prometheus2')) + self.assertEqual( + '3', _get_mon_count_from_prometheus(unit.public_address)) + + +# NOTE: We might query before prometheus has fetch data +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, + min=5, max=10), + reraise=True) +def _get_mon_count_from_prometheus(prometheus_ip): + url = ('http://{}:9090/api/v1/query?query=' + 'count(ceph_mon_metadata)'.format(prometheus_ip)) + client = requests.session() + response = client.get(url) + logging.debug("Prometheus response: {}".format(response.json())) + return response.json()['data']['result'][0]['value'][1] From c712cd522284e1c88e0383491fbbff051195b21a Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 2 Apr 2020 15:42:56 +0200 Subject: [PATCH 362/898] Fix path to the noop test --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 87f6ad6..78cc9cb 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ charm_name: pacemaker-remote tests: - zaza.openstack.charm_tests.pacemaker_remote.tests.PacemakerRemoteTest configure: - - zaza.openstack.charm_tests.noop.setup.basic_setup + - zaza.charm_tests.noop.setup.basic_setup gate_bundles: - basic smoke_bundles: @@ -23,4 +23,4 @@ test-requirements.txt: ``` git+https://github.com/openstack-charmers/zaza.git#egg=zaza git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack -``` \ No newline at end of file +``` From 5d46f6df1b21a3e5abe10f10d788e2775eb726ae Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 2 Apr 2020 15:43:59 +0200 Subject: [PATCH 363/898] Revert - didn't mean to commit directly --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78cc9cb..74a1651 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ charm_name: pacemaker-remote tests: - zaza.openstack.charm_tests.pacemaker_remote.tests.PacemakerRemoteTest configure: - - zaza.charm_tests.noop.setup.basic_setup + - zaza.openstack.charm_tests.noop.setup.basic_setup gate_bundles: - basic smoke_bundles: From 8acb08a90404e54ecfa0d764ff6cb6dd03356029 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 2 Apr 2020 15:44:17 +0200 Subject: [PATCH 364/898] Update path to the noop test --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 74a1651..78cc9cb 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ charm_name: pacemaker-remote tests: - zaza.openstack.charm_tests.pacemaker_remote.tests.PacemakerRemoteTest configure: - - zaza.openstack.charm_tests.noop.setup.basic_setup + - zaza.charm_tests.noop.setup.basic_setup gate_bundles: - basic smoke_bundles: From b87c4e485b6b59e4a35f5bf1d0ebcccebcf81861 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 3 Apr 2020 08:10:56 +0200 Subject: [PATCH 365/898] Remove broken functional test folder Rationale: 1. `tox.ini` doesn't have a `func` target so probably these tests haven't been run for a long time. 2. These tests can't be run at the moment because of conflicting version ranges in python deps pulled via `requirements.txt` and `setup.py`, e.g. error: PyYAML 3.13 is installed but pyyaml<=6.0,>=5.1.2 is required by {'juju'} error: python-keystoneclient 3.21.0 is installed but python-keystoneclient>=3.22.0 is required by {'python-openstackclient'} 3. These tests don't have expectations. They just deploy a bundle. --- tests/README.md | 2 -- tests/bundles/magpie.yaml | 5 ----- tests/bundles/overlays/local-charm-overlay.yaml.j2 | 1 - tests/tests.yaml | 11 ----------- 4 files changed, 19 deletions(-) delete mode 100644 tests/README.md delete mode 100644 tests/bundles/magpie.yaml delete mode 100644 tests/bundles/overlays/local-charm-overlay.yaml.j2 delete mode 100644 tests/tests.yaml diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index e5095d0..0000000 --- a/tests/README.md +++ /dev/null @@ -1,2 +0,0 @@ -This directory contains functional test definition for functional test of Zaza -itself. diff --git a/tests/bundles/magpie.yaml b/tests/bundles/magpie.yaml deleted file mode 100644 index eb065fd..0000000 --- a/tests/bundles/magpie.yaml +++ /dev/null @@ -1,5 +0,0 @@ -series: bionic -applications: - magpie: - charm: cs:~admcleod/magpie - num_units: 2 diff --git a/tests/bundles/overlays/local-charm-overlay.yaml.j2 b/tests/bundles/overlays/local-charm-overlay.yaml.j2 deleted file mode 100644 index 185df30..0000000 --- a/tests/bundles/overlays/local-charm-overlay.yaml.j2 +++ /dev/null @@ -1 +0,0 @@ -comment: this bundle overlay intentionally left blank diff --git a/tests/tests.yaml b/tests/tests.yaml deleted file mode 100644 index 73f9e9a..0000000 --- a/tests/tests.yaml +++ /dev/null @@ -1,11 +0,0 @@ -charm_name: none -gate_bundles: -- magpie -target_deploy_status: - magpie: - workload-status: active - workload-status-message: icmp ok, local hostname ok, dns ok -configure: -- zaza.openstack.charm_tests.noop.setup.basic_setup -tests: -- zaza.openstack.charm_tests.noop.tests.NoopTest From d3974c84a33ec1f98f192a4565f4fdd0542a4f49 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 6 Apr 2020 16:12:00 +0200 Subject: [PATCH 366/898] Fix iterating over applications --- zaza/openstack/charm_tests/series_upgrade/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 239681f..d5a9163 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -38,7 +38,7 @@ def _filter_easyrsa(app, app_config, model_name=None): return False -def _filter_etcd(app, app_config, model_name=None): +def _filter_etcd(app, app_config, model_name=None):for application, app_details in applications charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) if "etcd" in charm_name: logging.warn("Skipping series upgrade of easyrsa Bug #1850124") @@ -65,7 +65,7 @@ class SeriesUpgradeTest(unittest.TestCase): applications = model.get_status().applications completed_machines = [] - for application, app_details in applications: + for application, app_details in applications.items(): # Skip subordinates if app_details["subordinate-to"]: continue From 33fa6e02dec10762237389faf48a00b112854151 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 6 Apr 2020 16:13:56 +0200 Subject: [PATCH 367/898] Fix typo --- zaza/openstack/charm_tests/series_upgrade/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index d5a9163..9e72338 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -38,7 +38,7 @@ def _filter_easyrsa(app, app_config, model_name=None): return False -def _filter_etcd(app, app_config, model_name=None):for application, app_details in applications +def _filter_etcd(app, app_config, model_name=None): charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) if "etcd" in charm_name: logging.warn("Skipping series upgrade of easyrsa Bug #1850124") From 06b9083f3c5b23e5c499f0828a2d66a83846689e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 1 Apr 2020 09:23:48 +0200 Subject: [PATCH 368/898] Validate that charm-neutron-gateway marks bridges as managed by us https://bugs.launchpad.net/charm-neutron-gateway/+bug/1809190 --- zaza/openstack/charm_tests/neutron/tests.py | 36 ++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 2ae92f6..672cc95 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -14,7 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Encapsulating `neutron-openvswitch` testing.""" +"""Encapsulating testing of some `neutron-*` charms. + +`neutron-api`, `neutron-gateway` and `neutron-openvswitch` +""" import copy @@ -110,6 +113,31 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): super(NeutronGatewayTest, cls).setUpClass(cls) cls.services = cls._get_services() + _APP_NAME = 'neutron-gateway' + + def test_800_ovs_bridges_are_managed_by_us(self): + """Checking OVS bridges' external-id. + + OVS bridges created by us should be marked as managed by us in their + external-id. See + http://docs.openvswitch.org/en/latest/topics/integration/ + """ + for unit in zaza.model.get_units(self._APP_NAME, + model_name=self.model_name): + for bridge_name in ('br-int', 'br-ex'): + logging.info( + 'Checking that the bridge {}:{}'.format( + unit.name, bridge_name + ) + ' is marked as managed by us' + ) + expected_external_id = 'charm-neutron-gateway=managed' + actual_external_id = zaza.model.run_on_unit( + unit.entity_id, + 'ovs-vsctl br-get-external-id {}'.format(bridge_name), + model_name=self.model_name + )['Stdout'].strip() + self.assertEqual(actual_external_id, expected_external_id) + def test_900_restart_on_config_change(self): """Checking restart happens on config change. @@ -118,7 +146,7 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): """ # Expected default and alternate values current_value = zaza.model.get_application_config( - 'neutron-gateway')['debug']['value'] + self._APP_NAME)['debug']['value'] new_value = str(not bool(current_value)).title() current_value = str(current_value).title() @@ -132,7 +160,7 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): # Make config change, check for service restarts logging.info( - 'Setting verbose on neutron-api {}'.format(set_alternate)) + 'Setting verbose on {} {}'.format(self._APP_NAME, set_alternate)) self.restart_on_changed( conf_file, set_default, @@ -170,7 +198,7 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): logging.debug('Remote unit timestamp {}'.format(mtime)) with self.config_change(set_default, set_alternate): - for unit in zaza.model.get_units('neutron-gateway', + for unit in zaza.model.get_units(self._APP_NAME, model_name=self.model_name): logging.info('Checking number of profiles in complain ' 'mode in {}'.format(unit.entity_id)) From f35da80f2783a39ad49e5bb9c7c59de0520ed458 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 7 Apr 2020 11:04:46 +0200 Subject: [PATCH 369/898] Unpin flake8, fix lint Add Python 3.8 env in Travis CI test matrix. At present the pinning of flake8 disallows running of lint on Python 3.8 systems. Update flake8 ignore-list to ignore W504 instead of W503, the PEP guidance is that either is ok, but there must be local consistency. There are more occurences of binary operator before line-break than after in this repository, and we have also chosen to ignore W504 in most of our other repositories. --- .travis.yml | 3 ++ requirements.txt | 2 +- tox.ini | 2 +- .../charm_tests/openstack_dashboard/tests.py | 12 ++--- zaza/openstack/charm_tests/policyd/tests.py | 8 ++-- zaza/openstack/charm_tests/quagga/tests.py | 44 +++++++++---------- zaza/openstack/charm_tests/test_utils.py | 2 +- zaza/openstack/utilities/generic.py | 40 ++++++++--------- zaza/openstack/utilities/openstack.py | 4 +- 9 files changed, 60 insertions(+), 57 deletions(-) diff --git a/.travis.yml b/.travis.yml index c1558f3..7635f2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,5 +13,8 @@ matrix: - name: "Python 3.7" python: 3.7 env: ENV=pep8,py3 + - name: "Python 3.8" + python: 3.7 + env: ENV=pep8,py3 script: - tox -c tox.ini -e $ENV diff --git a/requirements.txt b/requirements.txt index cbf31df..ed9c353 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ boto3 juju juju_wait PyYAML<=4.2,>=3.0 -flake8>=2.2.4,<=3.5.0 +flake8>=2.2.4 flake8-docstrings flake8-per-file-ignores pydocstyle<4.0.0 diff --git a/tox.ini b/tox.ini index f5be2fa..6a37cfe 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ deps = -r{toxinidir}/requirements.txt commands = /bin/true [flake8] -ignore = E402,E226,W503 +ignore = E402,E226,W504 per-file-ignores = unit_tests/**: D diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 187cdc3..7ac0dd0 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -105,12 +105,12 @@ def _login(dashboard_ip, domain, username, password): # services/information missing that horizon wants to display data # for. # Redirect to /horizon/identity/ instead. - if (openstack_utils.get_os_release() - >= openstack_utils.get_os_release('xenial_queens')): + if (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('xenial_queens')): auth['next'] = '/horizon/identity/' - if (openstack_utils.get_os_release() - >= openstack_utils.get_os_release('bionic_stein')): + if (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('bionic_stein')): auth['region'] = 'default' if api_version == 2: @@ -122,8 +122,8 @@ def _login(dashboard_ip, domain, username, password): # NOTE(ajkavanagh) there used to be a trusty/icehouse test in the # amulet test, but as the zaza tests only test from trusty/mitaka # onwards, the test has been dropped - if (openstack_utils.get_os_release() - >= openstack_utils.get_os_release('bionic_stein')): + if (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('bionic_stein')): expect = "Sign Out" # update the in dashboard seems to require region to be default in # this test configuration diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 703f0af..360736d 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -214,8 +214,8 @@ class GenericPolicydTest(PolicydTest, test_utils.OpenStackBaseTest): def setUpClass(cls, application_name=None): """Run class setup for running KeystonePolicydTest tests.""" super(GenericPolicydTest, cls).setUpClass(application_name) - if (openstack_utils.get_os_release() - < openstack_utils.get_os_release('xenial_queens')): + if (openstack_utils.get_os_release() < + openstack_utils.get_os_release('xenial_queens')): raise unittest.SkipTest( "zaza.openstack.charm_tests.policyd.tests.GenericPolicydTest " "not valid before xenial_queens") @@ -276,8 +276,8 @@ class BasePolicydSpecialization(PolicydTest, def setUpClass(cls, application_name=None): """Run class setup for running KeystonePolicydTest tests.""" super(BasePolicydSpecialization, cls).setUpClass(application_name) - if (openstack_utils.get_os_release() - < openstack_utils.get_os_release('xenial_queens')): + if (openstack_utils.get_os_release() < + openstack_utils.get_os_release('xenial_queens')): raise unittest.SkipTest( "zaza.openstack.charm_tests.policyd.tests.* " "not valid before xenial_queens") diff --git a/zaza/openstack/charm_tests/quagga/tests.py b/zaza/openstack/charm_tests/quagga/tests.py index 67a49d6..67408f2 100644 --- a/zaza/openstack/charm_tests/quagga/tests.py +++ b/zaza/openstack/charm_tests/quagga/tests.py @@ -38,26 +38,26 @@ class QuaggaTest(unittest.TestCase): 'tor2', 'peer0', 'peer1'] if app in status.applications.keys()) for application in applications: - for unit in zaza.model.get_units(application): - bgp_sum = zaza.model.run_on_unit( - unit.entity_id, - 'echo "sh bgp ipv4 unicast summary" | vtysh')['Stdout'] - r = re.compile('^(\d+\.\d+\.\d+\.\d+)') - ip_list = [] - for line in bgp_sum.splitlines(): - m = r.match(line) - if m: - ip_list.append(m.group(1)) - logging.info('unit {} neighbours {}' - .format(unit.entity_id, ip_list)) + for unit in zaza.model.get_units(application): + bgp_sum = zaza.model.run_on_unit( + unit.entity_id, + 'echo "sh bgp ipv4 unicast summary" | vtysh')['Stdout'] + r = re.compile(r'^(\d+\.\d+\.\d+\.\d+)') + ip_list = [] + for line in bgp_sum.splitlines(): + m = r.match(line) + if m: + ip_list.append(m.group(1)) + logging.info('unit {} neighbours {}' + .format(unit.entity_id, ip_list)) - if not ip_list: - raise Exception('FAILED: Unit {} has no BGP peers.' - .format(unit.entity_id)) - for ip in ip_list: - result = zaza.model.run_on_unit( - unit.entity_id, - 'ping -c 3 {}'.format(ip)) - logging.info(result['Stdout']) - if result['Code'] == '1': - raise Exception('FAILED') + if not ip_list: + raise Exception('FAILED: Unit {} has no BGP peers.' + .format(unit.entity_id)) + for ip in ip_list: + result = zaza.model.run_on_unit( + unit.entity_id, + 'ping -c 3 {}'.format(ip)) + logging.info(result['Stdout']) + if result['Code'] == '1': + raise Exception('FAILED') diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index ac8eb27..7b4cf36 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -50,7 +50,7 @@ def skipUntilVersion(service, package, release): stderr=subprocess.STDOUT, universal_newlines=True) return f(*args, **kwargs) - except subprocess.CalledProcessError as cp: + except subprocess.CalledProcessError: logging.warn("Skipping test for older ({})" "service {}, requested {}".format( package_version, service, release)) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 3a22404..64596d6 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -534,28 +534,28 @@ def get_file_contents(unit, f): def is_port_open(port, address): - """Determine if TCP port is accessible. + """Determine if TCP port is accessible. - Connect to the MySQL port on the VIP. + Connect to the MySQL port on the VIP. - :param port: Port number - :type port: str - :param address: IP address - :type port: str - :returns: True if port is reachable - :rtype: boolean - """ - try: - telnetlib.Telnet(address, port) - return True - except socket.error as e: - if e.errno == 113: - logging.error("could not connect to {}:{}" - .format(address, port)) - if e.errno == 111: - logging.error("connection refused connecting" - " to {}:{}".format(address, port)) - return False + :param port: Port number + :type port: str + :param address: IP address + :type port: str + :returns: True if port is reachable + :rtype: boolean + """ + try: + telnetlib.Telnet(address, port) + return True + except socket.error as e: + if e.errno == 113: + logging.error("could not connect to {}:{}" + .format(address, port)) + if e.errno == 111: + logging.error("connection refused connecting" + " to {}:{}".format(address, port)) + return False def port_knock_units(units, port=22, expect_success=True): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index df10e6e..f4f1374 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1411,11 +1411,11 @@ def get_os_code_info(package, pkg_version): pkg_version = pkg_version.split(':')[1:][0] if 'swift' in package: # Fully x.y.z match for swift versions - match = re.match('^(\d+)\.(\d+)\.(\d+)', pkg_version) + match = re.match(r'^(\d+)\.(\d+)\.(\d+)', pkg_version) else: # x.y match only for 20XX.X # and ignore patch level for other packages - match = re.match('^(\d+)\.(\d+)', pkg_version) + match = re.match(r'^(\d+)\.(\d+)', pkg_version) if match: vers = match.group(0) From 116da9ca6cf5130ef21f94e7caf9e995fe9a4014 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 7 Apr 2020 10:55:09 +0200 Subject: [PATCH 370/898] octavia: tear down resources created during LBaasV2 test Fixes #221 --- zaza/openstack/charm_tests/octavia/tests.py | 69 ++++++++++++++++----- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index bd23cc9..a5265e4 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -18,6 +18,8 @@ import logging import subprocess import tenacity +import osc_lib.exceptions + import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -47,15 +49,52 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): """Run class setup for running LBaaSv2 service tests.""" super(LBAASv2Test, cls).setUpClass() + cls.keystone_session = openstack_utils.get_overcloud_keystone_session() + cls.neutron_client = openstack_utils.get_neutron_session_client( + cls.keystone_session) + cls.octavia_client = openstack_utils.get_octavia_session_client( + cls.keystone_session) + + # NOTE(fnordahl): in the event of a test failure we do not want to run + # tear down code as it will make debugging a problem virtually + # impossible. To alleviate each test method will set the + # `run_tearDown` instance variable at the end which will let us run + # tear down only when there were no failure. + cls.run_tearDown = False + # List of load balancers created by this test + cls.loadbalancers = [] + # LIst of floating IPs created by this test + cls.fips = [] + + @classmethod + def tearDown(cls): + """Remove resources created during test execution. + + Note that resources created in the configure step prior to executing + the test should not be touched here. + """ + for lb in cls.loadbalancers: + cls.octavia_client.load_balancer_delete(lb['id'], cascade=True) + try: + cls.wait_for_lb_resource( + cls.octavia_client.load_balancer_show, lb['id'], + provisioning_status='DELETED') + except osc_lib.exceptions.NotFound: + pass + for fip in cls.fips: + cls.neutron_client.delete_floatingip(fip) + @staticmethod - @tenacity.retry(wait=tenacity.wait_fixed(1), - reraise=True, stop=tenacity.stop_after_delay(900)) + @tenacity.retry(retry=tenacity.retry_if_exception_type(AssertionError), + wait=tenacity.wait_fixed(1), reraise=True, + stop=tenacity.stop_after_delay(900)) def wait_for_lb_resource(octavia_show_func, resource_id, - operating_status=None): + provisioning_status=None, operating_status=None): """Wait for loadbalancer resource to reach expected status.""" + provisioning_status = provisioning_status or 'ACTIVE' resp = octavia_show_func(resource_id) logging.info(resp['provisioning_status']) - assert resp['provisioning_status'] == 'ACTIVE', ( + assert resp['provisioning_status'] == provisioning_status, ( 'load balancer resource has not reached ' 'expected provisioning status: {}' .format(resp)) @@ -197,11 +236,8 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): def test_create_loadbalancer(self): """Create load balancer.""" - keystone_session = openstack_utils.get_overcloud_keystone_session() - neutron_client = openstack_utils.get_neutron_session_client( - keystone_session) nova_client = openstack_utils.get_nova_session_client( - keystone_session) + self.keystone_session) # Get IP of the prepared payload instances payload_ips = [] @@ -209,24 +245,24 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): payload_ips.append(server.networks['private'][0]) self.assertTrue(len(payload_ips) > 0) - resp = neutron_client.list_networks(name='private') + resp = self.neutron_client.list_networks(name='private') subnet_id = resp['networks'][0]['subnets'][0] if openstack_utils.dvr_enabled(): - resp = neutron_client.list_networks(name='private_lb_fip_network') + resp = self.neutron_client.list_networks( + name='private_lb_fip_network') vip_subnet_id = resp['networks'][0]['subnets'][0] else: vip_subnet_id = subnet_id - octavia_client = openstack_utils.get_octavia_session_client( - keystone_session) - for provider in self.get_lb_providers(octavia_client).keys(): + for provider in self.get_lb_providers(self.octavia_client).keys(): logging.info('Creating loadbalancer with provider {}' .format(provider)) - lb = self._create_lb_resources(octavia_client, provider, + lb = self._create_lb_resources(self.octavia_client, provider, vip_subnet_id, subnet_id, payload_ips) + self.loadbalancers.append(lb) lb_fp = openstack_utils.create_floating_ip( - neutron_client, 'ext_net', port={'id': lb['vip_port_id']}) + self.neutron_client, 'ext_net', port={'id': lb['vip_port_id']}) snippet = 'This is the default welcome page' assert snippet in self._get_payload(lb_fp['floating_ip_address']) @@ -234,3 +270,6 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): ' (provider="{}") at "http://{}/"' .format(snippet, provider, lb_fp['floating_ip_address'])) + + # If we get here, it means the tests passed + self.run_tearDown = True From 91c120a650283c91a5b92bdf00ccb930f60bcc08 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 8 Apr 2020 07:35:14 +0200 Subject: [PATCH 371/898] octavia: only run tearDown on success --- zaza/openstack/charm_tests/octavia/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index a5265e4..b189484 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -73,6 +73,8 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): Note that resources created in the configure step prior to executing the test should not be touched here. """ + if not cls.run_tearDown: + return for lb in cls.loadbalancers: cls.octavia_client.load_balancer_delete(lb['id'], cascade=True) try: From 147b94e1f6606e11cb0ec8ac711a5b4d7d0f2713 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 8 Apr 2020 10:50:53 +0100 Subject: [PATCH 372/898] Extend resource wait timeouts for cinder tests (juju storage) As the cinder charm now has Juju storage support, the cinder tests needs their timeouts extended as Juju storage on serverstack is MUCH, MUCH slower than ephemeral storage (which is local to the host). --- zaza/openstack/charm_tests/cinder/tests.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 1809cb9..1fa6a30 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -63,6 +63,8 @@ class CinderTests(test_utils.OpenStackBaseTest): openstack_utils.resource_reaches_status( self.cinder_client.volumes, vol_new.id, + wait_iteration_max_time=1200, + stop_after_attempt=20, expected_status="available", msg="Volume status wait") self.cinder_client.volumes.extend( @@ -71,6 +73,8 @@ class CinderTests(test_utils.OpenStackBaseTest): openstack_utils.resource_reaches_status( self.cinder_client.volumes, vol_new.id, + wait_iteration_max_time=1200, + stop_after_attempt=20, expected_status="available", msg="Volume status wait") @@ -85,6 +89,8 @@ class CinderTests(test_utils.OpenStackBaseTest): openstack_utils.resource_reaches_status( self.cinder_client.volumes, vol_img.id, + wait_iteration_max_time=1200, + stop_after_attempt=20, expected_status="available", msg="Volume status wait") @@ -97,6 +103,8 @@ class CinderTests(test_utils.OpenStackBaseTest): openstack_utils.resource_reaches_status( self.cinder_client.volumes, vol_new.id, + wait_iteration_max_time=1200, + stop_after_attempt=20, expected_status="available", msg="Volume status wait") @@ -107,6 +115,8 @@ class CinderTests(test_utils.OpenStackBaseTest): openstack_utils.resource_reaches_status( self.cinder_client.volume_snapshots, snap_new.id, + wait_iteration_max_time=1200, + stop_after_attempt=20, expected_status="available", msg="Volume status wait") @@ -118,6 +128,8 @@ class CinderTests(test_utils.OpenStackBaseTest): openstack_utils.resource_reaches_status( self.cinder_client.volumes, vol_from_snap.id, + wait_iteration_max_time=1200, + stop_after_attempt=20, expected_status="available", msg="Volume status wait") @@ -129,6 +141,8 @@ class CinderTests(test_utils.OpenStackBaseTest): openstack_utils.resource_reaches_status( self.cinder_client.volumes, vol_new.id, + wait_iteration_max_time=1200, + stop_after_attempt=20, expected_status="available", msg="Volume status wait") vol_new.force_delete() From 4dfcd931fa19ba021f75f6c84e54240a4ea1555a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 9 Apr 2020 08:21:07 +0000 Subject: [PATCH 373/898] Create new pool for tests --- zaza/openstack/charm_tests/ceph/iscsi/tests.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index 89b0d05..73ea3ed 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -26,7 +26,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): """Class for `ceph-iscsi` tests.""" GW_IQN = "iqn.2003-03.com.canonical.iscsi-gw:iscsi-igw" - POOL_NAME = "iscsi" + DATA_POOL_NAME = 'superssd' def get_client_initiatorname(self, unit): """Return the initiatorname for the given unit.""" @@ -58,7 +58,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): secondary_gw = gw_units[1] host_names = generic_utils.get_unit_hostnames(gw_units, fqdn=True) ctxt = { - 'pool_name': self.POOL_NAME, + 'pool_name': self.DATA_POOL_NAME, 'client_entity_id': client.entity_id, 'gw_iqn': self.GW_IQN, 'gw1_ip': primary_gw.public_address, @@ -103,6 +103,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): ctxt['gw1_entity_id'], ctxt['gw2_entity_id']), 'iqn': self.GW_IQN, + 'pool-name': self.DATA_POOL_NAME, 'image-size': ctxt['img_size'], 'image-name': ctxt['img_name'], 'client-initiatorname': ctxt['client_initiatorname'], @@ -131,8 +132,16 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): return '/dev/dm-0' in run['Stdout'] zaza.model.block_until(check_device_present) + def create_data_pool(self): + generic_utils.assertActionRanOK(zaza.model.run_action_on_leader( + 'ceph-mon', + 'create-pool', + action_params={ + 'name': self.DATA_POOL_NAME})) + def test_create_and_mount_volume(self): """Test creating a target and mounting it on a client.""" + self.create_data_pool() ctxt = self.get_ctxt() self.create_iscsi_target(ctxt) self.mount_iscsi_target(ctxt) From 2b9d27b1f0ddaf68207068865439f5b5cc21c69c Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 9 Apr 2020 17:31:34 +0200 Subject: [PATCH 374/898] Add coverage reporting for unit tests --- .travis.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c1558f3..938efa4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,9 @@ sudo: true dist: xenial language: python -install: pip install tox-travis +install: + - pip install tox-travis + - pip install codecov matrix: include: - name: "Python 3.5" @@ -15,3 +17,5 @@ matrix: env: ENV=pep8,py3 script: - tox -c tox.ini -e $ENV +after_success: + - codecov --verbose --gcov-glob unit_tests/* \ No newline at end of file From a47640cf6971f3fec482b461337f09f56a553d5f Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 10 Apr 2020 10:02:31 +0200 Subject: [PATCH 375/898] Add codecov configuration to exclude the charm tests This does not, however, exclude the configure code in charm_tests as that can, and should, be unit tested :) --- codecov.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..d583fd7 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,2 @@ +ignore: + - "zaza/openstack/charm_tests/**/*tests.py" From cf1ea4c71b9bcc1bec13362f33f28dbf768133a5 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 8 Apr 2020 16:10:38 +0200 Subject: [PATCH 376/898] First pass at batched parallel series upgrade --- .../test_zaza_utilities_upgrade_utils.py | 4 +- .../series_upgrade/parallel_tests.py | 202 +++++++ .../utilities/parallel_series_upgrade.py | 528 ++++++++++++++++++ zaza/openstack/utilities/upgrade_utils.py | 6 +- 4 files changed, 737 insertions(+), 3 deletions(-) create mode 100644 zaza/openstack/charm_tests/series_upgrade/parallel_tests.py create mode 100755 zaza/openstack/utilities/parallel_series_upgrade.py diff --git a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py index 1f21b49..814772e 100644 --- a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py +++ b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py @@ -90,6 +90,7 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): def test_get_upgrade_groups(self): expected = collections.OrderedDict([ + ('Stateful Services', []), ('Core Identity', []), ('Control Plane', ['cinder']), ('Data Plane', ['nova-compute']), @@ -103,10 +104,11 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): def test_get_series_upgrade_groups(self): expected = collections.OrderedDict([ + ('Stateful Services', ['mydb']), ('Core Identity', []), ('Control Plane', ['cinder']), ('Data Plane', ['nova-compute']), - ('sweep_up', ['mydb', 'ntp'])]) + ('sweep_up', ['ntp'])]) actual = openstack_upgrade.get_series_upgrade_groups() pprint.pprint(expected) pprint.pprint(actual) diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py new file mode 100644 index 0000000..1b64994 --- /dev/null +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define class for Series Upgrade.""" + +import asyncio +import logging +import os +import sys +import unittest + +from zaza import model +from zaza.openstack.utilities import ( + cli as cli_utils, + upgrade_utils as upgrade_utils, +) +from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest +from zaza.openstack.utilities.parallel_series_upgrade import ( + parallel_series_upgrade, +) + + +def _filter_easyrsa(app, app_config, model_name=None): + charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) + if "easyrsa" in charm_name: + logging.warn("Skipping series upgrade of easyrsa Bug #1850121") + return True + return False + + +def _filter_etcd(app, app_config, model_name=None): + charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) + if "etcd" in charm_name: + logging.warn("Skipping series upgrade of easyrsa Bug #1850124") + return True + return False + + +class ParallelSeriesUpgradeTest(unittest.TestCase): + """Class to encapsulate Series Upgrade Tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + cli_utils.setup_logging() + cls.from_series = None + cls.to_series = None + cls.workaround_script = None + cls.files = [] + + def test_200_run_series_upgrade(self): + """Run series upgrade.""" + # Set Feature Flag + os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" + upgrade_groups = upgrade_utils.get_series_upgrade_groups( + extra_filters=[_filter_etcd, _filter_easyrsa]) + from_series = self.from_series + to_series = self.to_series + completed_machines = [] + workaround_script = None + files = [] + applications = model.get_status().applications + for group_name, apps in upgrade_groups.items(): + logging.info("About to upgrade {} from {} to {}".format( + group_name, from_series, to_series)) + upgrade_functions = [] + if group_name in ["Stateful Services", "Data Plane", "sweep_up"]: + logging.info("Going to upgrade {} unit by unit".format(apps)) + upgrade_function = \ + parallel_series_upgrade.serial_series_upgrade + else: + logging.info("Going to upgrade {} all at once".format(apps)) + upgrade_function = \ + parallel_series_upgrade.parallel_series_upgrade + + for charm_name in apps: + charm = applications[charm_name]['charm'] + name = upgrade_utils.extract_charm_name_from_url(charm) + upgrade_config = parallel_series_upgrade.app_config(name) + upgrade_functions.append( + upgrade_function( + charm_name, + **upgrade_config, + from_series=from_series, + to_series=to_series, + completed_machines=completed_machines, + workaround_script=workaround_script, + files=files)) + asyncio.get_event_loop().run_until_complete( + asyncio.gather(*upgrade_functions)) + + if "rabbitmq-server" in apps: + logging.info( + "Running complete-cluster-series-upgrade action on leader") + model.run_action_on_leader( + 'rabbitmq-server', + 'complete-cluster-series-upgrade', + action_params={}) + + if "percona-cluster" in apps: + logging.info( + "Running complete-cluster-series-upgrade action on leader") + model.run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) + model.block_until_all_units_idle() + logging.info("Finished {}".format(group_name)) + logging.info("Done!") + + +class OpenStackParallelSeriesUpgrade(ParallelSeriesUpgradeTest): + """OpenStack Series Upgrade. + + Full OpenStack series upgrade with VM launch before and after the series + upgrade. + + This test requires a full OpenStack including at least: keystone, glance, + nova-cloud-controller, nova-compute, neutron-gateway, neutron-api and + neutron-openvswitch. + """ + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + super(OpenStackParallelSeriesUpgrade, cls).setUpClass() + cls.lts = LTSGuestCreateTest() + cls.lts.setUpClass() + + def test_100_validate_pre_series_upgrade_cloud(self): + """Validate pre series upgrade.""" + logging.info("Validate pre-series-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + def test_500_validate_series_upgraded_cloud(self): + """Validate post series upgrade.""" + logging.info("Validate post-series-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + +class TrustyXenialSeriesUpgrade(OpenStackParallelSeriesUpgrade): + """OpenStack Trusty to Xenial Series Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Trusty to Xenial Series Upgrades.""" + super(TrustyXenialSeriesUpgrade, cls).setUpClass() + cls.from_series = "trusty" + cls.to_series = "xenial" + + +class XenialBionicSeriesUpgrade(OpenStackParallelSeriesUpgrade): + """OpenStack Xenial to Bionic Series Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(XenialBionicSeriesUpgrade, cls).setUpClass() + cls.from_series = "xenial" + cls.to_series = "bionic" + + +class BionicFocalSeriesUpgrade(OpenStackParallelSeriesUpgrade): + """OpenStack Bionic to FocalSeries Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(BionicFocalSeriesUpgrade, cls).setUpClass() + cls.from_series = "bionic" + cls.to_series = "focal" + + +if __name__ == "__main__": + from_series = os.environ.get("FROM_SERIES") + if from_series == "trusty": + to_series = "xenial" + series_upgrade_test = TrustyXenialSeriesUpgrade() + elif from_series == "xenial": + to_series = "bionic" + series_upgrade_test = XenialBionicSeriesUpgrade() + elif from_series == "bionic": + to_series = "focal" + series_upgrade_test = BionicFocalSeriesUpgrade() + + else: + raise Exception("FROM_SERIES is not set to a vailid LTS series") + series_upgrade_test.setUpClass() + sys.exit(series_upgrade_test.test_200_run_series_upgrade()) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py new file mode 100755 index 0000000..a959838 --- /dev/null +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -0,0 +1,528 @@ +#!/usr/bin/env python3 +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of functions for testing series upgrade in parallel.""" + + +import asyncio +import collections +import copy +import logging +import subprocess + +from zaza import model +import zaza.openstack.utilities.generic as os_utils +import zaza.openstack.utilities.series_upgrade as series_upgrade_utils +from zaza.openstack.utilities.series_upgrade import ( + SUBORDINATE_PAUSE_RESUME_BLACKLIST, +) + + +def app_config(charm_name): + """Return a dict with the upgrade config for an application. + + :param charm_name: Name of the charm about to upgrade + :type charm_name: str + :param async: Whether the upgreade functions should be async + :type async: bool + :returns: A dicitonary of the upgrade config for the application + :rtype: Dict + """ + default = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'follower_first': False, } + _app_settings = collections.defaultdict(lambda: default) + ceph = { + 'origin': "source", + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + } + exceptions = { + 'rabbitmq-server': { + 'origin': 'source', + 'pause_non_leader_subordinate': False, }, + 'percona-cluster': {'origin': 'source', }, + 'nova-compute': { + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, }, + 'ceph': ceph, + 'ceph-mon': ceph, + 'ceph-osd': ceph, + 'designate-bind': {'origin': None, }, + 'tempest': {'origin': None, }, + 'memcached': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': False, + }, + 'vault': { + 'origin': None, + 'pause_non_leader_primary': False, + 'pause_non_leader_subordinate': True, + 'post_upgrade_functions': [ + ('zaza.openstack.charm_tests.vault.setup.' + 'mojo_unseal_by_unit')] + }, + 'mongodb': { + 'origin': None, + 'follower_first': True, + } + } + for key, value in exceptions.items(): + _app_settings[key] = copy.deepcopy(default) + _app_settings[key].update(value) + return _app_settings[charm_name] + + +def upgrade_ubuntu_lite(from_series='xenial', to_series='bionic'): + """Validate that we can upgrade the ubuntu-lite charm. + + :param from_series: What series are we upgrading from + :type from_series: str + :param to_series: What series are we upgrading to + :type to_series: str + """ + completed_machines = [] + asyncio.get_event_loop().run_until_complete( + parallel_series_upgrade( + 'ubuntu-lite', pause_non_leader_primary=False, + pause_non_leader_subordinate=False, + completed_machines=completed_machines, origin=None) + ) + + +async def parallel_series_upgrade( + application, from_series='xenial', to_series='bionic', + origin='openstack-origin', pause_non_leader_primary=True, + pause_non_leader_subordinate=True, post_upgrade_functions=None, + completed_machines=None, files=None, workaround_script=None, + follower_first=False): + """Perform series upgrade on an application in parallel. + + :param unit_name: Unit Name + :type unit_name: str + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :param follower_first: Should the follower(s) be upgraded first + :type follower_first: bool + :returns: None + :rtype: None + """ + if completed_machines is None: + completed_machines = [] + if files is None: + files = [] + if post_upgrade_functions is None: + post_upgrade_functions = [] + if follower_first: + logging.error("leader_first is ignored for parallel upgrade") + logging.info( + "About to upgrade the units of {} in parallel (follower first: {})" + .format(application, follower_first)) + # return + status = (await model.async_get_status()).applications[application] + leader, non_leaders = await get_leader_and_non_leaders(application) + for leader_name, leader_unit in leader.items(): + leader_machine = leader_unit["machine"] + leader = leader_name + machines = [ + unit["machine"] for name, unit + in non_leaders.items() + if unit['machine'] not in completed_machines] + await maybe_pause_things( + status, + non_leaders, + pause_non_leader_subordinate, + pause_non_leader_primary) + await series_upgrade_utils.async_set_series( + application, to_series=to_series) + + prepare_group = [ + prepare_series_upgrade(machine, to_series=to_series) + for machine in machines] + asyncio.gather(*prepare_group) + await prepare_series_upgrade(leader_machine, to_series=to_series) + if leader_machine not in completed_machines: + machines.append(leader_machine) + # do the dist upgrade + await dist_upgrades(machines) + # do a do-release-upgrade + await do_release_upgrades(machines) + # do a reboot + await reboots(machines) + + await complete_series_upgrade(machines, to_series) + if origin: + await os_utils.async_set_origin(application, origin) + + +async def serial_series_upgrade( + application, from_series='xenial', to_series='bionic', + origin='openstack-origin', pause_non_leader_primary=True, + pause_non_leader_subordinate=True, post_upgrade_functions=None, + completed_machines=None, files=None, workaround_script=None, + follower_first=False,): + """Perform series upgrade on an application in series. + + :param unit_name: Unit Name + :type unit_name: str + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param origin: The configuration setting variable name for changing origin + source. (openstack-origin or source) + :type origin: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :param follower_first: Should the follower(s) be upgraded first + :type follower_first: bool + :returns: None + :rtype: None + """ + if completed_machines is None: + completed_machines = [] + if files is None: + files = [] + if post_upgrade_functions is None: + post_upgrade_functions = [] + logging.info( + "About to upgrade the units of {} in serial (follower first: {})" + .format(application, follower_first)) + # return + status = (await model.async_get_status()).applications[application] + leader, non_leaders = await get_leader_and_non_leaders(application) + for leader_name, leader_unit in leader.items(): + leader_machine = leader_unit["machine"] + leader = leader_name + + machines = [ + unit["machine"] for name, unit + in non_leaders.items() + if unit['machine'] not in completed_machines] + + await maybe_pause_things( + status, + non_leaders, + pause_non_leader_subordinate, + pause_non_leader_primary) + await series_upgrade_utils.async_set_series( + application, to_series=to_series) + + if not follower_first and leader_machine not in completed_machines: + await prepare_series_upgrade(leader_machine, to_series) + logging.info( + "About to dist-upgrade the leader ({})".format(leader_machine)) + # upgrade the leader + await dist_upgrades([leader_machine]) + # do a do-release-upgrade + await do_release_upgrades([leader_machine]) + # do a reboot + await reboots([leader_machine]) + await complete_series_upgrade([leader_machine], to_series) + + for machine in machines: + await prepare_series_upgrade(machine, to_series) + logging.debug( + "About to dist-upgrade follower {}".format(machine)) + await dist_upgrades([machine]) + # do a do-release-upgrade + await do_release_upgrades([machine]) + # do a reboot + await reboots([machine]) + await complete_series_upgrade([machine], to_series) + + if follower_first and leader_machine not in completed_machines: + await prepare_series_upgrade(leader_machine, to_series) + logging.info( + "About to dist-upgrade the leader ({})".format(leader_machine)) + # upgrade the leader + await dist_upgrades([leader_machine]) + # do a do-release-upgrade + await do_release_upgrades([leader_machine]) + # do a reboot + await reboots([leader_machine]) + await complete_series_upgrade([leader_machine], to_series) + if origin: + await os_utils.async_set_origin(application, origin) + + +async def maybe_pause_things( + status, units, pause_non_leader_subordinate=True, + pause_non_leader_primary=True): + """Pause the non-leaders, based on the run configuration. + + :param status: Juju status for an application + :type status: juju.applications + :param units: List of units to paybe pause + :type units: LIst[str] + :param pause_non_leader_subordinate: Should the non leader + subordinate be paused + :type pause_non_leader_subordinate: bool + :param pause_non_leader_primary: Should the non leaders be paused + :type pause_non_leader_primary: bool + :returns: Nothing + :trype: None + """ + subordinate_pauses = [] + leader_pauses = [] + for unit in units: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + _app = subordinate.split('/')[0] + if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: + logging.info("Skipping pausing {} - blacklisted" + .format(subordinate)) + else: + logging.info("Pausing {}".format(subordinate)) + subordinate_pauses.append(model.async_run_action( + subordinate, "pause", action_params={})) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + leader_pauses.append( + model.async_run_action(unit, "pause", action_params={})) + await asyncio.gather(*subordinate_pauses) + await asyncio.gather(*leader_pauses) + + +async def dist_upgrades(machines): + """Run dist-upgrade on unit after update package db. + + :param machines: List of machines to upgrade + :type machines: List[str] + :returns: None + :rtype: None + """ + upgrade_group = [] + for machine in machines: + upgrade_group.append(async_dist_upgrade(machine)) + logging.info( + "About to await the dist upgrades of {}".format(machines)) + await asyncio.gather(*upgrade_group) + + +async def do_release_upgrades(machines): + """Run do-release-upgrade noninteractive. + + :param machines: List of machines to upgrade + :type machines: List[str] + :returns: None + :rtype: None + """ + upgrade_group = [] + for machine in machines: + upgrade_group.append(async_do_release_upgrade(machine)) + logging.info( + "About to await the do release upgrades of {}".format(machines)) + await asyncio.gather(*upgrade_group) + + +async def reboots(machines): + """Reboot all of the listed machines. + + :param machines: A list of machines, ex: ['1', '2'] + :type machines: List[str] + :returns: None + :rtype: None + """ + upgrade_group = [] + for machine in machines: + upgrade_group.append(reboot(machine)) + logging.info("About to await the reboots of {}".format(machines)) + await asyncio.gather(*upgrade_group) + + +async def get_units(application): + """Get all units for the named application. + + :param application: The application to get units of + :type application: str + :returns: The units for a specified application + :rtype: Dict[str, juju.Unit] + """ + status = (await model.async_get_status()).applications[application] + return status["units"] + + +async def get_leader_and_non_leaders(application): + """Get the leader and non-leader Juju units. + + This function returns a tuple that looks like: + + ({ + 'unit/1': juju.Unit, + }, + { + 'unit/0': juju.Unit, + 'unit/2': juju.unit, + }) + + The first entry of this tuple is the leader, and the second is + all non-leader units. + + :param application: Application to fetch details for + :type application: str + :returns: A tuple of dicts identifying leader and non-leaders + :rtype: Dict[str, List[juju.Unit]] + """ + logging.info( + "Configuring leader / non leaders for {}".format(application)) + # if completed_machines is None: + # completed_machines = [] + leader = None + non_leaders = {} + for name, unit in (await get_units(application)).items(): + if unit.get("leader"): + leader = {name: unit} + # leader = status["units"][unit]["machine"] + else: + non_leaders[name] = unit + # machine = status["units"][unit]["machine"] + # if machine not in completed_machines: + # non_leaders.append(machine) + return (leader, non_leaders) + + +async def prepare_series_upgrade(machine, to_series): + """Execute juju series-upgrade prepare on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + :param machine_num: Machine number + :type machine_num: str + :param to_series: The series to which to upgrade + :type to_series: str + :returns: None + :rtype: None + """ + logging.debug("Preparing series upgrade for: {}".format(machine)) + await series_upgrade_utils.async_prepare_series_upgrade( + machine, to_series=to_series) + + +async def reboot(unit): + """Reboot the named machine. + + :param unit: Machine to reboot + :type unit: str + :returns: Nothing + :rtype: None + """ + try: + await run_on_machine(unit, 'shutdown --reboot now & exit') + # await run_on_machine(unit, "sudo reboot && exit") + except subprocess.CalledProcessError as e: + logging.warn("Error doing reboot: {}".format(e)) + pass + + +async def complete_series_upgrade(machines, to_series): + """Execute juju series-upgrade complete on machine. + + NOTE: This is a new feature in juju behind a feature flag and not yet in + libjuju. + export JUJU_DEV_FEATURE_FLAGS=upgrade-series + :param machine_num: Machine number + :type machine_num: str + :returns: None + :rtype: None + """ + logging.info("Completing series upgrade for {}".format(machines)) + group = [] + for machine in machines: + # This can fail on the non-leaders if the leader goes first? + group.append( + series_upgrade_utils.async_complete_series_upgrade(machine)) + await asyncio.gather(*group) + + +async def run_on_machine(machine, command, model_name=None, timeout=None): + """Juju run on unit. + + :param model_name: Name of model unit is in + :type model_name: str + :param unit_name: Name of unit to match + :type unit: str + :param command: Command to execute + :type command: str + :param timeout: How long in seconds to wait for command to complete + :type timeout: int + :returns: action.data['results'] {'Code': '', 'Stderr': '', 'Stdout': ''} + :rtype: dict + """ + cmd = ['juju', 'run', '--machine={}'.format(machine)] + if model_name: + cmd.append('--model={}'.format(model_name)) + if timeout: + cmd.append('--timeout={}'.format(timeout)) + cmd.append(command) + logging.debug("About to call '{}'".format(cmd)) + await os_utils.check_call(cmd) + + +async def async_dist_upgrade(machine): + """Run dist-upgrade on unit after update package db. + + :param machine: Machine Number + :type machine: str + :returns: None + :rtype: None + """ + logging.info('Updating package db ' + machine) + update_cmd = 'sudo apt update' + await run_on_machine(machine, update_cmd) + + logging.info('Updating existing packages ' + machine) + dist_upgrade_cmd = ( + """yes | sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + await run_on_machine(machine, dist_upgrade_cmd) + + +async def async_do_release_upgrade(machine): + """Run do-release-upgrade noninteractive. + + :param machine: Machine Name + :type machine: str + :returns: None + :rtype: None + """ + logging.info('Upgrading ' + machine) + do_release_upgrade_cmd = ( + 'yes | sudo DEBIAN_FRONTEND=noninteractive ' + 'do-release-upgrade -f DistUpgradeViewNonInteractive') + + await run_on_machine(machine, do_release_upgrade_cmd, timeout="120m") diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 062e9b1..b24aec7 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -20,15 +20,17 @@ import zaza.model SERVICE_GROUPS = collections.OrderedDict([ + ('Stateful Services', ['percona-cluster', 'rabbitmq-server']), ('Core Identity', ['keystone']), ('Control Plane', [ - 'aodh', 'barbican', 'ceilometer', 'ceph-mon', 'ceph-fs', + 'aodh', 'barbican', 'ceilometer', 'ceph-fs', 'ceph-radosgw', 'cinder', 'designate', 'designate-bind', 'glance', 'gnocchi', 'heat', 'manila', 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', 'nova-cloud-controller', 'openstack-dashboard']), ('Data Plane', [ - 'nova-compute', 'ceph-osd', 'swift-proxy', 'swift-storage']) + 'nova-compute', 'ceph-mon', 'ceph-osd', + 'swift-proxy', 'swift-storage']) ]) UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] From 186708e88353bbb99819c6ce695a4315791337cf Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 9 Apr 2020 09:12:14 +0200 Subject: [PATCH 377/898] Change upgrade process to handle each machine fully Updating each machine in a fully self-contained way makes it easy to add per-machine and (possibly) per-application post-upgrade function calls. This still performs the actual series upgrades fully in parallel but will, for example, start the do-release-upgrade on one unit while another is still performing the initial dist-upgrade --- .../series_upgrade/parallel_tests.py | 2 +- .../utilities/parallel_series_upgrade.py | 113 ++++++++++++------ 2 files changed, 80 insertions(+), 35 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py index 1b64994..7c9dba0 100644 --- a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -28,7 +28,7 @@ from zaza.openstack.utilities import ( upgrade_utils as upgrade_utils, ) from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest -from zaza.openstack.utilities.parallel_series_upgrade import ( +from zaza.openstack.utilities import ( parallel_series_upgrade, ) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index a959838..1585ef7 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -125,6 +125,16 @@ async def parallel_series_upgrade( :param origin: The configuration setting variable name for changing origin source. (openstack-origin or source) :type origin: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param post_upgrade_functions: A list of Zaza functions to call when + the upgrade is complete on each machine + :type post_upgrade_functions: List[str] :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade @@ -170,14 +180,14 @@ async def parallel_series_upgrade( await prepare_series_upgrade(leader_machine, to_series=to_series) if leader_machine not in completed_machines: machines.append(leader_machine) - # do the dist upgrade - await dist_upgrades(machines) - # do a do-release-upgrade - await do_release_upgrades(machines) - # do a reboot - await reboots(machines) - - await complete_series_upgrade(machines, to_series) + upgrade_group = [ + series_upgrade_machine( + machine, from_series=from_series, to_series=to_series, + files=files, workaround_script=None, + post_upgrade_functions=post_upgrade_functions) + for machine in machines + ] + await asyncio.gather(*upgrade_group) if origin: await os_utils.async_set_origin(application, origin) @@ -201,6 +211,16 @@ async def serial_series_upgrade( :param origin: The configuration setting variable name for changing origin source. (openstack-origin or source) :type origin: str + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param post_upgrade_functions: A list of Zaza functions to call when + the upgrade is complete on each machine + :type post_upgrade_functions: List[str] :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade @@ -241,42 +261,67 @@ async def serial_series_upgrade( if not follower_first and leader_machine not in completed_machines: await prepare_series_upgrade(leader_machine, to_series) - logging.info( - "About to dist-upgrade the leader ({})".format(leader_machine)) - # upgrade the leader - await dist_upgrades([leader_machine]) - # do a do-release-upgrade - await do_release_upgrades([leader_machine]) - # do a reboot - await reboots([leader_machine]) - await complete_series_upgrade([leader_machine], to_series) + logging.info("About to upgrade leader of {}: {}" + .format(application, leader_machine)) + await series_upgrade_machine( + leader_machine, from_series=from_series, to_series=to_series, + files=files, workaround_script=None, + post_upgrade_functions=post_upgrade_functions) for machine in machines: await prepare_series_upgrade(machine, to_series) - logging.debug( - "About to dist-upgrade follower {}".format(machine)) - await dist_upgrades([machine]) - # do a do-release-upgrade - await do_release_upgrades([machine]) - # do a reboot - await reboots([machine]) - await complete_series_upgrade([machine], to_series) + logging.info("About to upgrade follower of {}: {}" + .format(application, machine)) + await series_upgrade_machine( + machine, from_series=from_series, to_series=to_series, + files=files, workaround_script=None, + post_upgrade_functions=post_upgrade_functions) if follower_first and leader_machine not in completed_machines: await prepare_series_upgrade(leader_machine, to_series) - logging.info( - "About to dist-upgrade the leader ({})".format(leader_machine)) - # upgrade the leader - await dist_upgrades([leader_machine]) - # do a do-release-upgrade - await do_release_upgrades([leader_machine]) - # do a reboot - await reboots([leader_machine]) - await complete_series_upgrade([leader_machine], to_series) + logging.info("About to upgrade leader of {}: {}" + .format(application, leader_machine)) + await series_upgrade_machine( + leader_machine, from_series=from_series, to_series=to_series, + files=files, workaround_script=None, + post_upgrade_functions=post_upgrade_functions) if origin: await os_utils.async_set_origin(application, origin) +async def series_upgrade_machine( + machine, from_series='xenial', to_series='bionic', + files=None, workaround_script=None, post_upgrade_functions=None): + """Perform series upgrade on an machine. + + :param machine_num: Machine number + :type machine_num: str + :param from_series: The series from which to upgrade + :type from_series: str + :param to_series: The series to which to upgrade + :type to_series: str + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str + :param post_upgrade_functions: A list of Zaza functions to call when + the upgrade is complete on each machine + :type post_upgrade_functions: List[str] + :returns: None + :rtype: None + """ + logging.info( + "About to dist-upgrade ({})".format(machine)) + # upgrade the do the dist upgrade + await async_dist_upgrade(machine) + # do a do-release-upgrade + await async_do_release_upgrade(machine) + # do a reboot + await reboot(machine) + await series_upgrade_utils.async_complete_series_upgrade(machine) + series_upgrade_utils.run_post_upgrade_functions(post_upgrade_functions) + + async def maybe_pause_things( status, units, pause_non_leader_subordinate=True, pause_non_leader_primary=True): From 2a98274d67589a5a251c0fa9082f28e27abc6833 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 9 Apr 2020 10:56:31 +0200 Subject: [PATCH 378/898] Add a few more function hooks to the series upgrade process By including pre and pos-application functions, charm series upgrades can be handled in a more generic way, even when they require running additional actions before a unit is upgraded, or after the whole application is upgraded. --- zaza/openstack/charm_tests/mysql/utils.py | 26 +++++ .../charm_tests/rabbitmq_server/utils.py | 8 ++ .../series_upgrade/parallel_tests.py | 16 ---- .../utilities/parallel_series_upgrade.py | 94 +++++++++++++++++-- zaza/openstack/utilities/upgrade_utils.py | 4 +- 5 files changed, 123 insertions(+), 25 deletions(-) create mode 100644 zaza/openstack/charm_tests/mysql/utils.py diff --git a/zaza/openstack/charm_tests/mysql/utils.py b/zaza/openstack/charm_tests/mysql/utils.py new file mode 100644 index 0000000..2622903 --- /dev/null +++ b/zaza/openstack/charm_tests/mysql/utils.py @@ -0,0 +1,26 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module of functions for interfacing with the percona-cluster charm.""" + +import zaza.model as model + + +def complete_cluster_series_upgrade(): + """Run the complete-cluster-series-upgrade action on the lead unit.""" + # TODO: Make this work across either mysql or percona-cluster names + model.run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 9826add..62db7e6 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -584,3 +584,11 @@ def _post_check_unit_cluster_nodes(unit, nodes, unit_node_names): 'after unit removal.\n' ''.format(unit_name, node)) return errors + + +def complete_cluster_series_upgrade(): + """Run the complete-cluster-series-upgrade action on the lead unit.""" + zaza.model.run_action_on_leader( + 'rabbitmq-server', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py index 7c9dba0..5307339 100644 --- a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -101,22 +101,6 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): files=files)) asyncio.get_event_loop().run_until_complete( asyncio.gather(*upgrade_functions)) - - if "rabbitmq-server" in apps: - logging.info( - "Running complete-cluster-series-upgrade action on leader") - model.run_action_on_leader( - 'rabbitmq-server', - 'complete-cluster-series-upgrade', - action_params={}) - - if "percona-cluster" in apps: - logging.info( - "Running complete-cluster-series-upgrade action on leader") - model.run_action_on_leader( - 'mysql', - 'complete-cluster-series-upgrade', - action_params={}) model.block_until_all_units_idle() logging.info("Finished {}".format(group_name)) logging.info("Done!") diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 1585ef7..622c5ac 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -23,6 +23,7 @@ import logging import subprocess from zaza import model +from zaza.charm_lifecycle import utils as cl_utils import zaza.openstack.utilities.generic as os_utils import zaza.openstack.utilities.series_upgrade as series_upgrade_utils from zaza.openstack.utilities.series_upgrade import ( @@ -45,6 +46,8 @@ def app_config(charm_name): 'pause_non_leader_subordinate': True, 'pause_non_leader_primary': True, 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], 'follower_first': False, } _app_settings = collections.defaultdict(lambda: default) ceph = { @@ -55,11 +58,25 @@ def app_config(charm_name): exceptions = { 'rabbitmq-server': { 'origin': 'source', - 'pause_non_leader_subordinate': False, }, - 'percona-cluster': {'origin': 'source', }, + 'pause_non_leader_subordinate': False, + 'post_application_upgrade_functions': [ + ('zaza.openstack.charm_tests.rabbitmq_server.utils.' + 'complete_cluster_series_upgrade')] + }, + 'percona-cluster': { + 'origin': 'source', + 'post_application_upgrade_functions': [ + ('zaza.openstack.charm_tests.mysql.utils.' + 'complete_cluster_series_upgrade')] + }, 'nova-compute': { 'pause_non_leader_primary': False, - 'pause_non_leader_subordinate': False, }, + 'pause_non_leader_subordinate': False, + # TODO + # 'pre_upgrade_functions': [ + # 'zaza.openstack.charm_tests.nova_compute.setup.evacuate' + # ] + }, 'ceph': ceph, 'ceph-mon': ceph, 'ceph-osd': ceph, @@ -111,7 +128,8 @@ async def parallel_series_upgrade( origin='openstack-origin', pause_non_leader_primary=True, pause_non_leader_subordinate=True, post_upgrade_functions=None, completed_machines=None, files=None, workaround_script=None, - follower_first=False): + follower_first=False, pre_upgrade_functions=None, + post_application_upgrade_functions=None): """Perform series upgrade on an application in parallel. :param unit_name: Unit Name @@ -132,9 +150,17 @@ async def parallel_series_upgrade( hacluster applications should be paused :type pause_non_leader_subordinate: bool + + :param pre_upgrade_functions: A list of Zaza functions to call before + the upgrade is started on each machine + :type pre_upgrade_functions: List[str] :param post_upgrade_functions: A list of Zaza functions to call when the upgrade is complete on each machine :type post_upgrade_functions: List[str] + :param post_application_upgrade_functions: A list of Zaza functions + to call when the upgrade is complete + on all machine in the application + :type post_application_upgrade_functions: List[str] :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade @@ -150,12 +176,16 @@ async def parallel_series_upgrade( files = [] if post_upgrade_functions is None: post_upgrade_functions = [] + if pre_upgrade_functions is None: + pre_upgrade_functions = [] + if post_application_upgrade_functions is None: + post_application_upgrade_functions = [] if follower_first: logging.error("leader_first is ignored for parallel upgrade") logging.info( "About to upgrade the units of {} in parallel (follower first: {})" .format(application, follower_first)) - # return + status = (await model.async_get_status()).applications[application] leader, non_leaders = await get_leader_and_non_leaders(application) for leader_name, leader_unit in leader.items(): @@ -165,6 +195,7 @@ async def parallel_series_upgrade( unit["machine"] for name, unit in non_leaders.items() if unit['machine'] not in completed_machines] + await maybe_pause_things( status, non_leaders, @@ -190,6 +221,7 @@ async def parallel_series_upgrade( await asyncio.gather(*upgrade_group) if origin: await os_utils.async_set_origin(application, origin) + run_post_application_upgrade_functions(post_application_upgrade_functions) async def serial_series_upgrade( @@ -197,7 +229,8 @@ async def serial_series_upgrade( origin='openstack-origin', pause_non_leader_primary=True, pause_non_leader_subordinate=True, post_upgrade_functions=None, completed_machines=None, files=None, workaround_script=None, - follower_first=False,): + follower_first=False, pre_upgrade_functions=None, + post_application_upgrade_functions=None): """Perform series upgrade on an application in series. :param unit_name: Unit Name @@ -218,9 +251,16 @@ async def serial_series_upgrade( hacluster applications should be paused :type pause_non_leader_subordinate: bool + :param pre_upgrade_functions: A list of Zaza functions to call before + the upgrade is started on each machine + :type pre_upgrade_functions: List[str] :param post_upgrade_functions: A list of Zaza functions to call when the upgrade is complete on each machine :type post_upgrade_functions: List[str] + :param post_application_upgrade_functions: A list of Zaza functions + to call when the upgrade is complete + on all machine in the application + :type post_application_upgrade_functions: List[str] :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade @@ -236,6 +276,10 @@ async def serial_series_upgrade( files = [] if post_upgrade_functions is None: post_upgrade_functions = [] + if pre_upgrade_functions is None: + pre_upgrade_functions = [] + if post_application_upgrade_functions is None: + post_application_upgrade_functions = [] logging.info( "About to upgrade the units of {} in serial (follower first: {})" .format(application, follower_first)) @@ -287,11 +331,13 @@ async def serial_series_upgrade( post_upgrade_functions=post_upgrade_functions) if origin: await os_utils.async_set_origin(application, origin) + run_post_application_upgrade_functions(post_application_upgrade_functions) async def series_upgrade_machine( machine, from_series='xenial', to_series='bionic', - files=None, workaround_script=None, post_upgrade_functions=None): + files=None, workaround_script=None, post_upgrade_functions=None, + pre_upgrade_functions=None): """Perform series upgrade on an machine. :param machine_num: Machine number @@ -304,6 +350,9 @@ async def series_upgrade_machine( :type files: list :param workaround_script: Workaround script to run during series upgrade :type workaround_script: str + :param pre_upgrade_functions: A list of Zaza functions to call before + the upgrade is started on each machine + :type pre_upgrade_functions: List[str] :param post_upgrade_functions: A list of Zaza functions to call when the upgrade is complete on each machine :type post_upgrade_functions: List[str] @@ -312,6 +361,8 @@ async def series_upgrade_machine( """ logging.info( "About to dist-upgrade ({})".format(machine)) + + run_pre_upgrade_functions(pre_upgrade_functions) # upgrade the do the dist upgrade await async_dist_upgrade(machine) # do a do-release-upgrade @@ -322,6 +373,35 @@ async def series_upgrade_machine( series_upgrade_utils.run_post_upgrade_functions(post_upgrade_functions) +def run_pre_upgrade_functions(machine, pre_upgrade_functions): + """Execute list supplied functions. + + Each of the supplied functions will be called with a single + argument of the machine that is about to be upgraded. + + :param machine: Machine that is about to be upgraded + :type machine: str + :param pre_upgrade_functions: List of functions + :type pre_upgrade_functions: [function, function, ...] + """ + if pre_upgrade_functions: + for func in pre_upgrade_functions: + logging.info("Running {}".format(func)) + cl_utils.get_class(func)(machine) + + +def run_post_application_upgrade_functions(post_upgrade_functions): + """Execute list supplied functions. + + :param post_upgrade_functions: List of functions + :type post_upgrade_functions: [function, function, ...] + """ + if post_upgrade_functions: + for func in post_upgrade_functions: + logging.info("Running {}".format(func)) + cl_utils.get_class(func)() + + async def maybe_pause_things( status, units, pause_non_leader_subordinate=True, pause_non_leader_primary=True): diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index b24aec7..c864b9b 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -21,7 +21,7 @@ import zaza.model SERVICE_GROUPS = collections.OrderedDict([ ('Stateful Services', ['percona-cluster', 'rabbitmq-server']), - ('Core Identity', ['keystone']), + ('Core Identity', ['keystone', 'ceph-mon']), ('Control Plane', [ 'aodh', 'barbican', 'ceilometer', 'ceph-fs', 'ceph-radosgw', 'cinder', 'designate', @@ -29,7 +29,7 @@ SERVICE_GROUPS = collections.OrderedDict([ 'manila-generic', 'neutron-api', 'neutron-gateway', 'placement', 'nova-cloud-controller', 'openstack-dashboard']), ('Data Plane', [ - 'nova-compute', 'ceph-mon', 'ceph-osd', + 'nova-compute', 'ceph-osd', 'swift-proxy', 'swift-storage']) ]) From 392470e946d45caa3a3f503bd805727f883ff112 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 9 Apr 2020 15:05:53 +0200 Subject: [PATCH 379/898] Cleanup based on feedback This change removes some redundant checking at setup time and splits the long parameter lists into one argument per line lists --- .../utilities/parallel_series_upgrade.py | 142 ++++++------------ 1 file changed, 47 insertions(+), 95 deletions(-) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 622c5ac..e08ede9 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -124,12 +124,20 @@ def upgrade_ubuntu_lite(from_series='xenial', to_series='bionic'): async def parallel_series_upgrade( - application, from_series='xenial', to_series='bionic', - origin='openstack-origin', pause_non_leader_primary=True, - pause_non_leader_subordinate=True, post_upgrade_functions=None, - completed_machines=None, files=None, workaround_script=None, - follower_first=False, pre_upgrade_functions=None, - post_application_upgrade_functions=None): + application, + from_series='xenial', + to_series='bionic', + origin='openstack-origin', + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + pre_upgrade_functions=None, + post_upgrade_functions=None, + post_application_upgrade_functions=None, + completed_machines=None, + follower_first=False, + files=None, + workaround_script=None +): """Perform series upgrade on an application in parallel. :param unit_name: Unit Name @@ -160,26 +168,18 @@ async def parallel_series_upgrade( :param post_application_upgrade_functions: A list of Zaza functions to call when the upgrade is complete on all machine in the application + :param follower_first: Should the follower(s) be upgraded first + :type follower_first: bool :type post_application_upgrade_functions: List[str] :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade :type workaround_script: str - :param follower_first: Should the follower(s) be upgraded first - :type follower_first: bool :returns: None :rtype: None """ if completed_machines is None: completed_machines = [] - if files is None: - files = [] - if post_upgrade_functions is None: - post_upgrade_functions = [] - if pre_upgrade_functions is None: - pre_upgrade_functions = [] - if post_application_upgrade_functions is None: - post_application_upgrade_functions = [] if follower_first: logging.error("leader_first is ignored for parallel upgrade") logging.info( @@ -213,8 +213,8 @@ async def parallel_series_upgrade( machines.append(leader_machine) upgrade_group = [ series_upgrade_machine( - machine, from_series=from_series, to_series=to_series, - files=files, workaround_script=None, + machine, + files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) for machine in machines ] @@ -225,13 +225,21 @@ async def parallel_series_upgrade( async def serial_series_upgrade( - application, from_series='xenial', to_series='bionic', - origin='openstack-origin', pause_non_leader_primary=True, - pause_non_leader_subordinate=True, post_upgrade_functions=None, - completed_machines=None, files=None, workaround_script=None, - follower_first=False, pre_upgrade_functions=None, - post_application_upgrade_functions=None): - """Perform series upgrade on an application in series. + application, + from_series='xenial', + to_series='bionic', + origin='openstack-origin', + pause_non_leader_primary=True, + pause_non_leader_subordinate=True, + pre_upgrade_functions=None, + post_upgrade_functions=None, + post_application_upgrade_functions=None, + completed_machines=None, + follower_first=False, + files=None, + workaround_script=None +): + """Perform series upgrade on an application in parallel. :param unit_name: Unit Name :type unit_name: str @@ -251,6 +259,7 @@ async def serial_series_upgrade( hacluster applications should be paused :type pause_non_leader_subordinate: bool + :param pre_upgrade_functions: A list of Zaza functions to call before the upgrade is started on each machine :type pre_upgrade_functions: List[str] @@ -260,26 +269,18 @@ async def serial_series_upgrade( :param post_application_upgrade_functions: A list of Zaza functions to call when the upgrade is complete on all machine in the application + :param follower_first: Should the follower(s) be upgraded first + :type follower_first: bool :type post_application_upgrade_functions: List[str] :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade :type workaround_script: str - :param follower_first: Should the follower(s) be upgraded first - :type follower_first: bool :returns: None :rtype: None """ if completed_machines is None: completed_machines = [] - if files is None: - files = [] - if post_upgrade_functions is None: - post_upgrade_functions = [] - if pre_upgrade_functions is None: - pre_upgrade_functions = [] - if post_application_upgrade_functions is None: - post_application_upgrade_functions = [] logging.info( "About to upgrade the units of {} in serial (follower first: {})" .format(application, follower_first)) @@ -308,8 +309,8 @@ async def serial_series_upgrade( logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) await series_upgrade_machine( - leader_machine, from_series=from_series, to_series=to_series, - files=files, workaround_script=None, + leader_machine, + files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) for machine in machines: @@ -317,8 +318,8 @@ async def serial_series_upgrade( logging.info("About to upgrade follower of {}: {}" .format(application, machine)) await series_upgrade_machine( - machine, from_series=from_series, to_series=to_series, - files=files, workaround_script=None, + machine, + files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) if follower_first and leader_machine not in completed_machines: @@ -326,8 +327,8 @@ async def serial_series_upgrade( logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) await series_upgrade_machine( - leader_machine, from_series=from_series, to_series=to_series, - files=files, workaround_script=None, + leader_machine, + files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) if origin: await os_utils.async_set_origin(application, origin) @@ -335,17 +336,15 @@ async def serial_series_upgrade( async def series_upgrade_machine( - machine, from_series='xenial', to_series='bionic', - files=None, workaround_script=None, post_upgrade_functions=None, - pre_upgrade_functions=None): + machine, + post_upgrade_functions=None, + pre_upgrade_functions=None, + files=None, + workaround_script=None): """Perform series upgrade on an machine. :param machine_num: Machine number :type machine_num: str - :param from_series: The series from which to upgrade - :type from_series: str - :param to_series: The series to which to upgrade - :type to_series: str :param files: Workaround files to scp to unit under upgrade :type files: list :param workaround_script: Workaround script to run during series upgrade @@ -441,53 +440,6 @@ async def maybe_pause_things( await asyncio.gather(*leader_pauses) -async def dist_upgrades(machines): - """Run dist-upgrade on unit after update package db. - - :param machines: List of machines to upgrade - :type machines: List[str] - :returns: None - :rtype: None - """ - upgrade_group = [] - for machine in machines: - upgrade_group.append(async_dist_upgrade(machine)) - logging.info( - "About to await the dist upgrades of {}".format(machines)) - await asyncio.gather(*upgrade_group) - - -async def do_release_upgrades(machines): - """Run do-release-upgrade noninteractive. - - :param machines: List of machines to upgrade - :type machines: List[str] - :returns: None - :rtype: None - """ - upgrade_group = [] - for machine in machines: - upgrade_group.append(async_do_release_upgrade(machine)) - logging.info( - "About to await the do release upgrades of {}".format(machines)) - await asyncio.gather(*upgrade_group) - - -async def reboots(machines): - """Reboot all of the listed machines. - - :param machines: A list of machines, ex: ['1', '2'] - :type machines: List[str] - :returns: None - :rtype: None - """ - upgrade_group = [] - for machine in machines: - upgrade_group.append(reboot(machine)) - logging.info("About to await the reboots of {}".format(machines)) - await asyncio.gather(*upgrade_group) - - async def get_units(application): """Get all units for the named application. From 9cd4b32aa3de43830107239976a7722756c3b8f8 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 9 Apr 2020 15:08:36 +0200 Subject: [PATCH 380/898] Remove leftover development comments, useless function --- .../utilities/parallel_series_upgrade.py | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index e08ede9..87deb51 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -187,7 +187,7 @@ async def parallel_series_upgrade( .format(application, follower_first)) status = (await model.async_get_status()).applications[application] - leader, non_leaders = await get_leader_and_non_leaders(application) + leader, non_leaders = await get_leader_and_non_leaders(status, application) for leader_name, leader_unit in leader.items(): leader_machine = leader_unit["machine"] leader = leader_name @@ -284,7 +284,6 @@ async def serial_series_upgrade( logging.info( "About to upgrade the units of {} in serial (follower first: {})" .format(application, follower_first)) - # return status = (await model.async_get_status()).applications[application] leader, non_leaders = await get_leader_and_non_leaders(application) for leader_name, leader_unit in leader.items(): @@ -362,11 +361,8 @@ async def series_upgrade_machine( "About to dist-upgrade ({})".format(machine)) run_pre_upgrade_functions(pre_upgrade_functions) - # upgrade the do the dist upgrade await async_dist_upgrade(machine) - # do a do-release-upgrade await async_do_release_upgrade(machine) - # do a reboot await reboot(machine) await series_upgrade_utils.async_complete_series_upgrade(machine) series_upgrade_utils.run_post_upgrade_functions(post_upgrade_functions) @@ -440,19 +436,7 @@ async def maybe_pause_things( await asyncio.gather(*leader_pauses) -async def get_units(application): - """Get all units for the named application. - - :param application: The application to get units of - :type application: str - :returns: The units for a specified application - :rtype: Dict[str, juju.Unit] - """ - status = (await model.async_get_status()).applications[application] - return status["units"] - - -async def get_leader_and_non_leaders(application): +async def get_leader_and_non_leaders(status, application): """Get the leader and non-leader Juju units. This function returns a tuple that looks like: @@ -475,19 +459,13 @@ async def get_leader_and_non_leaders(application): """ logging.info( "Configuring leader / non leaders for {}".format(application)) - # if completed_machines is None: - # completed_machines = [] leader = None non_leaders = {} - for name, unit in (await get_units(application)).items(): + for name, unit in status["units"].items(): if unit.get("leader"): leader = {name: unit} - # leader = status["units"][unit]["machine"] else: non_leaders[name] = unit - # machine = status["units"][unit]["machine"] - # if machine not in completed_machines: - # non_leaders.append(machine) return (leader, non_leaders) From 74652f2523bd77d9411e7f08f0a97abe97faabd8 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 9 Apr 2020 17:29:05 +0200 Subject: [PATCH 381/898] Add unit test coverage for series upgrades This also includes some tidying --- ..._zaza_utilities_parallel_series_upgrade.py | 382 ++++++++++++++++++ .../utilities/parallel_series_upgrade.py | 59 +-- zaza/openstack/utilities/upgrade_utils.py | 4 +- 3 files changed, 406 insertions(+), 39 deletions(-) create mode 100644 unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py new file mode 100644 index 0000000..966b54e --- /dev/null +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -0,0 +1,382 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import mock +import unit_tests.utils as ut_utils +import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.series_upgrade as series_upgrade +import zaza.openstack.utilities.parallel_series_upgrade as upgrade_utils + +FAKE_STATUS = { + 'can-upgrade-to': '', + 'charm': 'local:trusty/app-136', + 'subordinate-to': [], + 'units': {'app/0': {'leader': True, + 'machine': '0', + 'subordinates': { + 'app-hacluster/0': { + 'charm': 'local:trusty/hacluster-0', + 'leader': True}}}, + 'app/1': {'machine': '1', + 'subordinates': { + 'app-hacluster/1': { + 'charm': 'local:trusty/hacluster-0'}}}, + 'app/2': {'machine': '2', + 'subordinates': { + 'app-hacluster/2': { + 'charm': 'local:trusty/hacluster-0'}}}}} + + +class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase): + def setUp(self): + super(Test_ParallelSeriesUpgradeSync, self).setUp() + # Juju Status Object and data + # self.juju_status = mock.MagicMock() + # self.juju_status.applications.__getitem__.return_value = FAKE_STATUS + # self.patch_object(upgrade_utils, "model") + # self.model.get_status.return_value = self.juju_status + + def test_get_leader_and_non_leaders(self): + expected = ({ + 'app/0': { + 'leader': True, + 'machine': '0', + 'subordinates': { + 'app-hacluster/0': { + 'charm': 'local:trusty/hacluster-0', + 'leader': True}}}}, { + 'app/1': { + 'machine': '1', + 'subordinates': { + 'app-hacluster/1': { + 'charm': 'local:trusty/hacluster-0'}}}, + 'app/2': { + 'machine': '2', + 'subordinates': { + 'app-hacluster/2': { + 'charm': 'local:trusty/hacluster-0'}}}}) + + self.assertEqual( + expected, + upgrade_utils.get_leader_and_non_leaders(FAKE_STATUS) + ) + + def test_app_config_openstack_charm(self): + expected = { + 'origin': 'openstack-origin', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], + 'follower_first': False, } + config = upgrade_utils.app_config('keystone') + self.assertEqual(expected, config) + + def test_app_config_mongo(self): + expected = { + 'origin': None, + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], + 'follower_first': True, } + config = upgrade_utils.app_config('mongodb') + self.assertEqual(expected, config) + + def test_app_config_ceph(self): + expected = { + 'origin': 'source', + 'pause_non_leader_subordinate': False, + 'pause_non_leader_primary': False, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [], + 'follower_first': False, } + config = upgrade_utils.app_config('ceph-mon') + self.assertEqual(expected, config) + + def test_app_config_percona(self): + expected = { + 'origin': 'source', + 'pause_non_leader_subordinate': True, + 'pause_non_leader_primary': True, + 'post_upgrade_functions': [], + 'pre_upgrade_functions': [], + 'post_application_upgrade_functions': [ + ('zaza.openstack.charm_tests.mysql.utils.' + 'complete_cluster_series_upgrade') + ], + 'follower_first': False, } + config = upgrade_utils.app_config('percona-cluster') + self.assertEqual(expected, config) + + +class AioTestCase(ut_utils.BaseTestCase): + def __init__(self, methodName='runTest', loop=None): + self.loop = loop or asyncio.get_event_loop() + self._function_cache = {} + super(AioTestCase, self).__init__(methodName=methodName) + + def coroutine_function_decorator(self, func): + def wrapper(*args, **kw): + return self.loop.run_until_complete(func(*args, **kw)) + return wrapper + + def __getattribute__(self, item): + attr = object.__getattribute__(self, item) + if asyncio.iscoroutinefunction(attr) and item.startswith('test_'): + if item not in self._function_cache: + self._function_cache[item] = ( + self.coroutine_function_decorator(attr)) + return self._function_cache[item] + return attr + + +class TestParallelSeriesUpgrade(AioTestCase): + def setUp(self): + super(TestParallelSeriesUpgrade, self).setUp() + self.patch_object(series_upgrade, "async_prepare_series_upgrade") + self.patch_object(generic_utils, 'check_call') + # Juju Status Object and data + + self.juju_status = mock.AsyncMock() + self.juju_status.return_value.applications.__getitem__.return_value = \ + FAKE_STATUS + self.patch_object(upgrade_utils, "model") + self.model.async_get_status = self.juju_status + self.async_run_action = mock.AsyncMock() + self.model.async_run_action = self.async_run_action + + @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') + @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') + @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') + @mock.patch.object(upgrade_utils, 'maybe_pause_things') + @mock.patch.object(upgrade_utils, 'series_upgrade_machine') + async def test_parallel_series_upgrade( + self, + mock_series_upgrade_machine, + mock_maybe_pause_things, + mock_async_set_series, + mock_async_prepare_series_upgrade, + mock_post_application_upgrade_functions, + mock_async_set_origin, + ): + await upgrade_utils.parallel_series_upgrade( + 'app', + from_series='trusty', + to_series='xenial', + ) + mock_async_set_series.assert_called_once_with( + 'app', to_series='xenial') + self.juju_status.assert_called() + mock_async_prepare_series_upgrade.assert_has_calls([ + mock.call('1', to_series='xenial'), + mock.call('2', to_series='xenial'), + mock.call('0', to_series='xenial'), + ]) + mock_maybe_pause_things.assert_called() + mock_series_upgrade_machine.assert_has_calls([ + mock.call( + '1', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '2', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '0', + files=None, + workaround_script=None, + post_upgrade_functions=None), + ]) + mock_async_set_origin.assert_called_once_with( + 'app', 'openstack-origin') + mock_post_application_upgrade_functions.assert_called_once_with(None) + + @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') + @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') + @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') + @mock.patch.object(upgrade_utils, 'maybe_pause_things') + @mock.patch.object(upgrade_utils, 'series_upgrade_machine') + async def test_serial_series_upgrade( + self, + mock_series_upgrade_machine, + mock_maybe_pause_things, + mock_async_set_series, + mock_async_prepare_series_upgrade, + mock_post_application_upgrade_functions, + mock_async_set_origin, + ): + await upgrade_utils.serial_series_upgrade( + 'app', + from_series='trusty', + to_series='xenial', + ) + mock_async_set_series.assert_called_once_with( + 'app', to_series='xenial') + self.juju_status.assert_called() + mock_async_prepare_series_upgrade.assert_has_calls([ + mock.call('0', to_series='xenial'), + mock.call('1', to_series='xenial'), + mock.call('2', to_series='xenial'), + ]) + mock_maybe_pause_things.assert_called() + mock_series_upgrade_machine.assert_has_calls([ + mock.call( + '0', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '1', + files=None, + workaround_script=None, + post_upgrade_functions=None), + mock.call( + '2', + files=None, + workaround_script=None, + post_upgrade_functions=None), + ]) + mock_async_set_origin.assert_called_once_with( + 'app', 'openstack-origin') + mock_post_application_upgrade_functions.assert_called_once_with(None) + + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_complete_series_upgrade') + @mock.patch.object(upgrade_utils, 'reboot') + @mock.patch.object(upgrade_utils, 'async_do_release_upgrade') + @mock.patch.object(upgrade_utils, 'async_dist_upgrade') + async def test_series_upgrade_machine( + self, + mock_async_dist_upgrade, + mock_async_do_release_upgrade, + mock_reboot, + mock_async_complete_series_upgrade + ): + await upgrade_utils.series_upgrade_machine( + '1', + post_upgrade_functions=None, + pre_upgrade_functions=None, + files=None, + workaround_script=None) + mock_async_dist_upgrade.assert_called_once_with('1') + mock_async_do_release_upgrade.assert_called_once_with('1') + mock_reboot.assert_called_once_with('1') + mock_async_complete_series_upgrade.assert_called_once_with('1') + + async def test_maybe_pause_things_primary(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=False, + pause_non_leader_primary=True) + self.async_run_action.assert_has_calls([ + mock.call('app/1', "pause", action_params={}), + mock.call('app/2', "pause", action_params={}), + ]) + + async def test_maybe_pause_things_subordinates(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=True, + pause_non_leader_primary=False) + self.async_run_action.assert_has_calls([ + mock.call('app-hacluster/1', "pause", action_params={}), + mock.call('app-hacluster/2', "pause", action_params={}), + ]) + + async def test_maybe_pause_things_all(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=True, + pause_non_leader_primary=True) + self.async_run_action.assert_has_calls([ + mock.call('app-hacluster/1', "pause", action_params={}), + mock.call('app/1', "pause", action_params={}), + mock.call('app-hacluster/2', "pause", action_params={}), + mock.call('app/2', "pause", action_params={}), + ]) + + async def test_maybe_pause_things_none(self): + await upgrade_utils.maybe_pause_things( + FAKE_STATUS, + ['app/1', 'app/2'], + pause_non_leader_subordinate=False, + pause_non_leader_primary=False) + self.async_run_action.assert_not_called() + + @mock.patch.object(upgrade_utils, 'run_on_machine') + async def test_async_do_release_upgrade(self, mock_run_on_machine): + await upgrade_utils.async_do_release_upgrade('1') + do_release_upgrade_cmd = ( + 'yes | sudo DEBIAN_FRONTEND=noninteractive ' + 'do-release-upgrade -f DistUpgradeViewNonInteractive') + mock_run_on_machine.assert_called_once_with( + '1', do_release_upgrade_cmd, timeout='120m' + ) + + async def test_prepare_series_upgrade(self): + await upgrade_utils.prepare_series_upgrade( + '1', to_series='xenial' + ) + self.async_prepare_series_upgrade.assert_called_once_with( + '1', to_series='xenial' + ) + + @mock.patch.object(upgrade_utils, 'run_on_machine') + async def test_reboot(self, mock_run_on_machine): + await upgrade_utils.reboot('1') + mock_run_on_machine.assert_called_once_with( + '1', 'shutdown --reboot now & exit' + ) + + async def test_run_on_machine(self): + await upgrade_utils.run_on_machine('1', 'test') + self.check_call.assert_called_once_with( + ['juju', 'run', '--machine=1', 'test']) + + async def test_run_on_machine_with_timeout(self): + await upgrade_utils.run_on_machine('1', 'test', timeout='20m') + self.check_call.assert_called_once_with( + ['juju', 'run', '--machine=1', '--timeout=20m', 'test']) + + async def test_run_on_machine_with_model(self): + await upgrade_utils.run_on_machine('1', 'test', model_name='test') + self.check_call.assert_called_once_with( + ['juju', 'run', '--machine=1', '--model=test', 'test']) + + @mock.patch.object(upgrade_utils, 'run_on_machine') + async def test_async_dist_upgrade(self, mock_run_on_machine): + await upgrade_utils.async_dist_upgrade('1') + apt_update_command = ( + """yes | sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """-o "Dpkg::Options::=--force-confdef" """ + """-o "Dpkg::Options::=--force-confold" dist-upgrade""") + mock_run_on_machine.assert_has_calls([ + mock.call('1', 'sudo apt update'), + mock.call('1', apt_update_command), + ]) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 87deb51..7e6693e 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -187,7 +187,9 @@ async def parallel_series_upgrade( .format(application, follower_first)) status = (await model.async_get_status()).applications[application] - leader, non_leaders = await get_leader_and_non_leaders(status, application) + logging.info( + "Configuring leader / non leaders for {}".format(application)) + leader, non_leaders = get_leader_and_non_leaders(status) for leader_name, leader_unit in leader.items(): leader_machine = leader_unit["machine"] leader = leader_name @@ -205,10 +207,12 @@ async def parallel_series_upgrade( application, to_series=to_series) prepare_group = [ - prepare_series_upgrade(machine, to_series=to_series) + series_upgrade_utils.async_prepare_series_upgrade( + machine, to_series=to_series) for machine in machines] asyncio.gather(*prepare_group) - await prepare_series_upgrade(leader_machine, to_series=to_series) + await series_upgrade_utils.async_prepare_series_upgrade( + leader_machine, to_series=to_series) if leader_machine not in completed_machines: machines.append(leader_machine) upgrade_group = [ @@ -285,7 +289,9 @@ async def serial_series_upgrade( "About to upgrade the units of {} in serial (follower first: {})" .format(application, follower_first)) status = (await model.async_get_status()).applications[application] - leader, non_leaders = await get_leader_and_non_leaders(application) + logging.info( + "Configuring leader / non leaders for {}".format(application)) + leader, non_leaders = get_leader_and_non_leaders(status) for leader_name, leader_unit in leader.items(): leader_machine = leader_unit["machine"] leader = leader_name @@ -304,7 +310,8 @@ async def serial_series_upgrade( application, to_series=to_series) if not follower_first and leader_machine not in completed_machines: - await prepare_series_upgrade(leader_machine, to_series) + await series_upgrade_utils.async_prepare_series_upgrade( + leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) await series_upgrade_machine( @@ -313,7 +320,8 @@ async def serial_series_upgrade( post_upgrade_functions=post_upgrade_functions) for machine in machines: - await prepare_series_upgrade(machine, to_series) + await series_upgrade_utils.async_prepare_series_upgrade( + machine, to_series=to_series) logging.info("About to upgrade follower of {}: {}" .format(application, machine)) await series_upgrade_machine( @@ -322,7 +330,8 @@ async def serial_series_upgrade( post_upgrade_functions=post_upgrade_functions) if follower_first and leader_machine not in completed_machines: - await prepare_series_upgrade(leader_machine, to_series) + await series_upgrade_utils.async_prepare_series_upgrade( + leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) await series_upgrade_machine( @@ -360,7 +369,7 @@ async def series_upgrade_machine( logging.info( "About to dist-upgrade ({})".format(machine)) - run_pre_upgrade_functions(pre_upgrade_functions) + run_pre_upgrade_functions(machine, pre_upgrade_functions) await async_dist_upgrade(machine) await async_do_release_upgrade(machine) await reboot(machine) @@ -436,7 +445,7 @@ async def maybe_pause_things( await asyncio.gather(*leader_pauses) -async def get_leader_and_non_leaders(status, application): +def get_leader_and_non_leaders(status): """Get the leader and non-leader Juju units. This function returns a tuple that looks like: @@ -452,13 +461,9 @@ async def get_leader_and_non_leaders(status, application): The first entry of this tuple is the leader, and the second is all non-leader units. - :param application: Application to fetch details for - :type application: str :returns: A tuple of dicts identifying leader and non-leaders :rtype: Dict[str, List[juju.Unit]] """ - logging.info( - "Configuring leader / non leaders for {}".format(application)) leader = None non_leaders = {} for name, unit in status["units"].items(): @@ -487,42 +492,22 @@ async def prepare_series_upgrade(machine, to_series): machine, to_series=to_series) -async def reboot(unit): +async def reboot(machine): """Reboot the named machine. - :param unit: Machine to reboot - :type unit: str + :param machine: Machine to reboot + :type machine: str :returns: Nothing :rtype: None """ try: - await run_on_machine(unit, 'shutdown --reboot now & exit') + await run_on_machine(machine, 'shutdown --reboot now & exit') # await run_on_machine(unit, "sudo reboot && exit") except subprocess.CalledProcessError as e: logging.warn("Error doing reboot: {}".format(e)) pass -async def complete_series_upgrade(machines, to_series): - """Execute juju series-upgrade complete on machine. - - NOTE: This is a new feature in juju behind a feature flag and not yet in - libjuju. - export JUJU_DEV_FEATURE_FLAGS=upgrade-series - :param machine_num: Machine number - :type machine_num: str - :returns: None - :rtype: None - """ - logging.info("Completing series upgrade for {}".format(machines)) - group = [] - for machine in machines: - # This can fail on the non-leaders if the leader goes first? - group.append( - series_upgrade_utils.async_complete_series_upgrade(machine)) - await asyncio.gather(*group) - - async def run_on_machine(machine, command, model_name=None, timeout=None): """Juju run on unit. diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index c864b9b..4066d19 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -20,8 +20,8 @@ import zaza.model SERVICE_GROUPS = collections.OrderedDict([ - ('Stateful Services', ['percona-cluster', 'rabbitmq-server']), - ('Core Identity', ['keystone', 'ceph-mon']), + ('Stateful Services', ['percona-cluster', 'rabbitmq-server', 'ceph-mon']), + ('Core Identity', ['keystone']), ('Control Plane', [ 'aodh', 'barbican', 'ceilometer', 'ceph-fs', 'ceph-radosgw', 'cinder', 'designate', From 7649bcb10e364374ea6537a5cd7b2802aac5927a Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 10 Apr 2020 09:29:07 +0200 Subject: [PATCH 382/898] Skip the async unit tests on py35 --- .../utilities/test_zaza_utilities_parallel_series_upgrade.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 966b54e..3935175 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -14,6 +14,8 @@ import asyncio import mock +import sys +import unittest import unit_tests.utils as ut_utils import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.series_upgrade as series_upgrade @@ -149,6 +151,8 @@ class AioTestCase(ut_utils.BaseTestCase): class TestParallelSeriesUpgrade(AioTestCase): def setUp(self): super(TestParallelSeriesUpgrade, self).setUp() + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") self.patch_object(series_upgrade, "async_prepare_series_upgrade") self.patch_object(generic_utils, 'check_call') # Juju Status Object and data From e2d6a5c0810e6d0ba653816230e06e81cf1cb606 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 10 Apr 2020 09:58:56 +0200 Subject: [PATCH 383/898] Add test coverage for follower-first upgrades --- ..._zaza_utilities_parallel_series_upgrade.py | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 3935175..7ec7df6 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -40,6 +40,18 @@ FAKE_STATUS = { 'app-hacluster/2': { 'charm': 'local:trusty/hacluster-0'}}}}} +FAKE_STATUS_MONGO = { + 'can-upgrade-to': '', + 'charm': 'local:trusty/mongodb-10', + 'subordinate-to': [], + 'units': {'mongo/0': {'leader': True, + 'machine': '0', + 'subordinates': {}}, + 'mongo/1': {'machine': '1', + 'subordinates': {}}, + 'mongo/2': {'machine': '2', + 'subordinates': {}}}} + class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase): def setUp(self): @@ -165,6 +177,114 @@ class TestParallelSeriesUpgrade(AioTestCase): self.async_run_action = mock.AsyncMock() self.model.async_run_action = self.async_run_action + @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') + @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') + @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') + @mock.patch.object(upgrade_utils, 'maybe_pause_things') + @mock.patch.object(upgrade_utils, 'series_upgrade_machine') + async def test_parallel_series_upgrade_mongo( + self, + mock_series_upgrade_machine, + mock_maybe_pause_things, + mock_async_set_series, + mock_async_prepare_series_upgrade, + mock_post_application_upgrade_functions, + mock_async_set_origin, + ): + self.juju_status.return_value.applications.__getitem__.return_value = \ + FAKE_STATUS_MONGO + upgrade_config = upgrade_utils.app_config('mongodb') + await upgrade_utils.parallel_series_upgrade( + 'mongodb', + from_series='trusty', + to_series='xenial', + **upgrade_config + ) + mock_async_set_series.assert_called_once_with( + 'mongodb', to_series='xenial') + self.juju_status.assert_called() + mock_async_prepare_series_upgrade.assert_has_calls([ + mock.call('1', to_series='xenial'), + mock.call('2', to_series='xenial'), + mock.call('0', to_series='xenial'), + ]) + mock_maybe_pause_things.assert_called() + mock_series_upgrade_machine.assert_has_calls([ + mock.call( + '1', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + mock.call( + '2', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + mock.call( + '0', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + ]) + mock_async_set_origin.assert_not_called() + mock_post_application_upgrade_functions.assert_called_once_with([]) + + @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') + @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') + @mock.patch.object(upgrade_utils.series_upgrade_utils, 'async_set_series') + @mock.patch.object(upgrade_utils, 'maybe_pause_things') + @mock.patch.object(upgrade_utils, 'series_upgrade_machine') + async def test_serial_series_upgrade_mongo( + self, + mock_series_upgrade_machine, + mock_maybe_pause_things, + mock_async_set_series, + mock_async_prepare_series_upgrade, + mock_post_application_upgrade_functions, + mock_async_set_origin, + ): + self.juju_status.return_value.applications.__getitem__.return_value = \ + FAKE_STATUS_MONGO + upgrade_config = upgrade_utils.app_config('mongodb') + await upgrade_utils.serial_series_upgrade( + 'mongodb', + from_series='trusty', + to_series='xenial', + **upgrade_config + ) + mock_async_set_series.assert_called_once_with( + 'mongodb', to_series='xenial') + self.juju_status.assert_called() + mock_async_prepare_series_upgrade.assert_has_calls([ + mock.call('1', to_series='xenial'), + mock.call('2', to_series='xenial'), + mock.call('0', to_series='xenial'), + ]) + mock_maybe_pause_things.assert_called() + mock_series_upgrade_machine.assert_has_calls([ + mock.call( + '1', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + mock.call( + '2', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + mock.call( + '0', + files=None, + workaround_script=None, + post_upgrade_functions=[]), + ]) + mock_async_set_origin.assert_not_called() + mock_post_application_upgrade_functions.assert_called_once_with([]) + @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') @mock.patch.object( From 4d38a158f0fd070390e5872697aeb6e8ba5aa3e8 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 10 Apr 2020 11:02:22 +0200 Subject: [PATCH 384/898] Tox coverage reporting shoulg be zaza.openstack only --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6a37cfe..7d67e9f 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ setenv = VIRTUAL_ENV={envdir} install_command = pip install {opts} {packages} -commands = nosetests --with-coverage --cover-package=zaza {posargs} {toxinidir}/unit_tests +commands = nosetests --with-coverage --cover-package=zaza.openstack {posargs} {toxinidir}/unit_tests [testenv:py3] basepython = python3 From 183b3c012fec36a6c9d183b2d7d8dba27f23f28a Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 10 Apr 2020 11:02:38 +0200 Subject: [PATCH 385/898] Add unit tests for new rabbit and mysql helpers --- unit_tests/charm_tests/__init__.py | 13 ++++++++ unit_tests/charm_tests/test_mysql.py | 32 +++++++++++++++++++ .../charm_tests/test_rabbitmq_server.py | 32 +++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 unit_tests/charm_tests/__init__.py create mode 100644 unit_tests/charm_tests/test_mysql.py create mode 100644 unit_tests/charm_tests/test_rabbitmq_server.py diff --git a/unit_tests/charm_tests/__init__.py b/unit_tests/charm_tests/__init__.py new file mode 100644 index 0000000..6131624 --- /dev/null +++ b/unit_tests/charm_tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/unit_tests/charm_tests/test_mysql.py b/unit_tests/charm_tests/test_mysql.py new file mode 100644 index 0000000..3493ce9 --- /dev/null +++ b/unit_tests/charm_tests/test_mysql.py @@ -0,0 +1,32 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unittest + +import zaza.openstack.charm_tests.mysql.utils as mysql_utils + + +class TestMysqlUtils(unittest.TestCase): + """Test class to encapsulate testing Mysql test utils.""" + + @mock.patch.object(mysql_utils, 'model') + def test_mysql_complete_cluster_series_upgrade(self, mock_model): + run_action_on_leader = mock.MagicMock() + mock_model.run_action_on_leader = run_action_on_leader + mysql_utils.complete_cluster_series_upgrade() + run_action_on_leader.assert_called_once_with( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/unit_tests/charm_tests/test_rabbitmq_server.py b/unit_tests/charm_tests/test_rabbitmq_server.py new file mode 100644 index 0000000..8e6ae0d --- /dev/null +++ b/unit_tests/charm_tests/test_rabbitmq_server.py @@ -0,0 +1,32 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unittest + +import zaza.openstack.charm_tests.rabbitmq_server.utils as rabbit_utils + + +class TestRabbitUtils(unittest.TestCase): + """Test class to encapsulate testing Mysql test utils.""" + + @mock.patch.object(rabbit_utils.zaza, 'model') + def test_rabbit_complete_cluster_series_upgrade(self, mock_model): + run_action_on_leader = mock.MagicMock() + mock_model.run_action_on_leader = run_action_on_leader + rabbit_utils.complete_cluster_series_upgrade() + run_action_on_leader.assert_called_once_with( + 'rabbitmq-server', + 'complete-cluster-series-upgrade', + action_params={}) From 208439cdf1aec531473518c0957c1c0f8ebb31b3 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 10 Apr 2020 11:42:21 +0200 Subject: [PATCH 386/898] Add unit tests for pre and post-app upgrade functions --- ...est_zaza_utilities_parallel_series_upgrade.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 7ec7df6..97371de 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -62,6 +62,22 @@ class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase): # self.patch_object(upgrade_utils, "model") # self.model.get_status.return_value = self.juju_status + @mock.patch.object(upgrade_utils.cl_utils, 'get_class') + def test_run_post_application_upgrade_functions(self, mock_get_class): + called = mock.MagicMock() + mock_get_class.return_value = called + upgrade_utils.run_post_application_upgrade_functions(['my.thing']) + mock_get_class.assert_called_once_with('my.thing') + called.assert_called() + + @mock.patch.object(upgrade_utils.cl_utils, 'get_class') + def test_run_pre_upgrade_functions(self, mock_get_class): + called = mock.MagicMock() + mock_get_class.return_value = called + upgrade_utils.run_pre_upgrade_functions('1', ['my.thing']) + mock_get_class.assert_called_once_with('my.thing') + called.assert_called_once_with('1') + def test_get_leader_and_non_leaders(self): expected = ({ 'app/0': { From a7df42a9df9845ec7478819dd9ff6517fbe80c5a Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 10 Apr 2020 16:27:03 +0200 Subject: [PATCH 387/898] According to the series upgrade doc, source should update early --- zaza/openstack/utilities/parallel_series_upgrade.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 7e6693e..66583c8 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -205,7 +205,8 @@ async def parallel_series_upgrade( pause_non_leader_primary) await series_upgrade_utils.async_set_series( application, to_series=to_series) - + if origin: + await os_utils.async_set_origin(application, origin) prepare_group = [ series_upgrade_utils.async_prepare_series_upgrade( machine, to_series=to_series) @@ -223,8 +224,6 @@ async def parallel_series_upgrade( for machine in machines ] await asyncio.gather(*upgrade_group) - if origin: - await os_utils.async_set_origin(application, origin) run_post_application_upgrade_functions(post_application_upgrade_functions) @@ -308,7 +307,8 @@ async def serial_series_upgrade( pause_non_leader_primary) await series_upgrade_utils.async_set_series( application, to_series=to_series) - + if origin: + await os_utils.async_set_origin(application, origin) if not follower_first and leader_machine not in completed_machines: await series_upgrade_utils.async_prepare_series_upgrade( leader_machine, to_series=to_series) @@ -338,8 +338,6 @@ async def serial_series_upgrade( leader_machine, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) - if origin: - await os_utils.async_set_origin(application, origin) run_post_application_upgrade_functions(post_application_upgrade_functions) From 27e5f7d8bc094a498c6000152849e2c365bb4e13 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 10 Apr 2020 16:51:41 +0200 Subject: [PATCH 388/898] We must have async functions for the before/after callables --- unit_tests/charm_tests/test_mysql.py | 14 ++++++-- .../charm_tests/test_rabbitmq_server.py | 14 ++++++-- ..._zaza_utilities_parallel_series_upgrade.py | 36 ++++++++++--------- zaza/openstack/charm_tests/mysql/utils.py | 4 +-- .../charm_tests/rabbitmq_server/utils.py | 4 +-- zaza/openstack/charm_tests/vault/setup.py | 14 ++++++++ .../utilities/parallel_series_upgrade.py | 27 ++++++++++---- 7 files changed, 80 insertions(+), 33 deletions(-) diff --git a/unit_tests/charm_tests/test_mysql.py b/unit_tests/charm_tests/test_mysql.py index 3493ce9..e38460b 100644 --- a/unit_tests/charm_tests/test_mysql.py +++ b/unit_tests/charm_tests/test_mysql.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import mock import unittest +import sys import zaza.openstack.charm_tests.mysql.utils as mysql_utils @@ -21,11 +23,17 @@ import zaza.openstack.charm_tests.mysql.utils as mysql_utils class TestMysqlUtils(unittest.TestCase): """Test class to encapsulate testing Mysql test utils.""" + def setUp(self): + super(TestMysqlUtils, self).setUp() + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + @mock.patch.object(mysql_utils, 'model') def test_mysql_complete_cluster_series_upgrade(self, mock_model): - run_action_on_leader = mock.MagicMock() - mock_model.run_action_on_leader = run_action_on_leader - mysql_utils.complete_cluster_series_upgrade() + run_action_on_leader = mock.AsyncMock() + mock_model.async_run_action_on_leader = run_action_on_leader + asyncio.get_event_loop().run_until_complete( + mysql_utils.complete_cluster_series_upgrade()) run_action_on_leader.assert_called_once_with( 'mysql', 'complete-cluster-series-upgrade', diff --git a/unit_tests/charm_tests/test_rabbitmq_server.py b/unit_tests/charm_tests/test_rabbitmq_server.py index 8e6ae0d..e092122 100644 --- a/unit_tests/charm_tests/test_rabbitmq_server.py +++ b/unit_tests/charm_tests/test_rabbitmq_server.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import mock import unittest +import sys import zaza.openstack.charm_tests.rabbitmq_server.utils as rabbit_utils @@ -21,11 +23,17 @@ import zaza.openstack.charm_tests.rabbitmq_server.utils as rabbit_utils class TestRabbitUtils(unittest.TestCase): """Test class to encapsulate testing Mysql test utils.""" + def setUp(self): + super(TestRabbitUtils, self).setUp() + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + @mock.patch.object(rabbit_utils.zaza, 'model') def test_rabbit_complete_cluster_series_upgrade(self, mock_model): - run_action_on_leader = mock.MagicMock() - mock_model.run_action_on_leader = run_action_on_leader - rabbit_utils.complete_cluster_series_upgrade() + run_action_on_leader = mock.AsyncMock() + mock_model.async_run_action_on_leader = run_action_on_leader + asyncio.get_event_loop().run_until_complete( + rabbit_utils.complete_cluster_series_upgrade()) run_action_on_leader.assert_called_once_with( 'rabbitmq-server', 'complete-cluster-series-upgrade', diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 97371de..8c17cbb 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -62,22 +62,6 @@ class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase): # self.patch_object(upgrade_utils, "model") # self.model.get_status.return_value = self.juju_status - @mock.patch.object(upgrade_utils.cl_utils, 'get_class') - def test_run_post_application_upgrade_functions(self, mock_get_class): - called = mock.MagicMock() - mock_get_class.return_value = called - upgrade_utils.run_post_application_upgrade_functions(['my.thing']) - mock_get_class.assert_called_once_with('my.thing') - called.assert_called() - - @mock.patch.object(upgrade_utils.cl_utils, 'get_class') - def test_run_pre_upgrade_functions(self, mock_get_class): - called = mock.MagicMock() - mock_get_class.return_value = called - upgrade_utils.run_pre_upgrade_functions('1', ['my.thing']) - mock_get_class.assert_called_once_with('my.thing') - called.assert_called_once_with('1') - def test_get_leader_and_non_leaders(self): expected = ({ 'app/0': { @@ -193,6 +177,26 @@ class TestParallelSeriesUpgrade(AioTestCase): self.async_run_action = mock.AsyncMock() self.model.async_run_action = self.async_run_action + @mock.patch.object(upgrade_utils.cl_utils, 'get_class') + async def test_run_post_application_upgrade_functions( + self, + mock_get_class + ): + called = mock.AsyncMock() + mock_get_class.return_value = called + await upgrade_utils.run_post_application_upgrade_functions( + ['my.thing']) + mock_get_class.assert_called_once_with('my.thing') + called.assert_called() + + @mock.patch.object(upgrade_utils.cl_utils, 'get_class') + async def test_run_pre_upgrade_functions(self, mock_get_class): + called = mock.AsyncMock() + mock_get_class.return_value = called + await upgrade_utils.run_pre_upgrade_functions('1', ['my.thing']) + mock_get_class.assert_called_once_with('my.thing') + called.assert_called_once_with('1') + @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') @mock.patch.object( diff --git a/zaza/openstack/charm_tests/mysql/utils.py b/zaza/openstack/charm_tests/mysql/utils.py index 2622903..1fe5114 100644 --- a/zaza/openstack/charm_tests/mysql/utils.py +++ b/zaza/openstack/charm_tests/mysql/utils.py @@ -17,10 +17,10 @@ import zaza.model as model -def complete_cluster_series_upgrade(): +async def complete_cluster_series_upgrade(): """Run the complete-cluster-series-upgrade action on the lead unit.""" # TODO: Make this work across either mysql or percona-cluster names - model.run_action_on_leader( + await model.async_run_action_on_leader( 'mysql', 'complete-cluster-series-upgrade', action_params={}) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 62db7e6..3994178 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -586,9 +586,9 @@ def _post_check_unit_cluster_nodes(unit, nodes, unit_node_names): return errors -def complete_cluster_series_upgrade(): +async def complete_cluster_series_upgrade(): """Run the complete-cluster-series-upgrade action on the lead unit.""" - zaza.model.run_action_on_leader( + await zaza.model.async_run_action_on_leader( 'rabbitmq-server', 'complete-cluster-series-upgrade', action_params={}) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 96c45cb..21c3793 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -66,6 +66,20 @@ def mojo_unseal_by_unit(): zaza.model.run_on_unit(unit_name, './hooks/update-status') +async def async_mojo_unseal_by_unit(): + """Unseal any units reported as sealed using mojo cacert.""" + cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path() + vault_creds = vault_utils.get_credentails() + for client in vault_utils.get_clients(cacert=cacert): + if client.hvac_client.is_sealed(): + client.hvac_client.unseal(vault_creds['keys'][0]) + unit_name = await juju_utils.async_get_unit_name_from_ip_address( + client.addr, + 'vault') + await zaza.model.async_run_on_unit( + unit_name, './hooks/update-status') + + def auto_initialize(cacert=None, validation_application='keystone'): """Auto initialize vault for testing. diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 66583c8..9e29d6d 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -93,7 +93,7 @@ def app_config(charm_name): 'pause_non_leader_subordinate': True, 'post_upgrade_functions': [ ('zaza.openstack.charm_tests.vault.setup.' - 'mojo_unseal_by_unit')] + 'async_mojo_unseal_by_unit')] }, 'mongodb': { 'origin': None, @@ -338,7 +338,8 @@ async def serial_series_upgrade( leader_machine, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) - run_post_application_upgrade_functions(post_application_upgrade_functions) + await run_post_application_upgrade_functions( + post_application_upgrade_functions) async def series_upgrade_machine( @@ -372,10 +373,10 @@ async def series_upgrade_machine( await async_do_release_upgrade(machine) await reboot(machine) await series_upgrade_utils.async_complete_series_upgrade(machine) - series_upgrade_utils.run_post_upgrade_functions(post_upgrade_functions) + await run_post_upgrade_functions(post_upgrade_functions) -def run_pre_upgrade_functions(machine, pre_upgrade_functions): +async def run_pre_upgrade_functions(machine, pre_upgrade_functions): """Execute list supplied functions. Each of the supplied functions will be called with a single @@ -389,10 +390,10 @@ def run_pre_upgrade_functions(machine, pre_upgrade_functions): if pre_upgrade_functions: for func in pre_upgrade_functions: logging.info("Running {}".format(func)) - cl_utils.get_class(func)(machine) + await cl_utils.get_class(func)(machine) -def run_post_application_upgrade_functions(post_upgrade_functions): +async def run_post_upgrade_functions(post_upgrade_functions): """Execute list supplied functions. :param post_upgrade_functions: List of functions @@ -401,7 +402,19 @@ def run_post_application_upgrade_functions(post_upgrade_functions): if post_upgrade_functions: for func in post_upgrade_functions: logging.info("Running {}".format(func)) - cl_utils.get_class(func)() + await cl_utils.get_class(func)() + + +async def run_post_application_upgrade_functions(post_upgrade_functions): + """Execute list supplied functions. + + :param post_upgrade_functions: List of functions + :type post_upgrade_functions: [function, function, ...] + """ + if post_upgrade_functions: + for func in post_upgrade_functions: + logging.info("Running {}".format(func)) + await cl_utils.get_class(func)() async def maybe_pause_things( From 6c70ee5171b8193b4f834275215b315e0e288cfe Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 14 Apr 2020 12:42:18 +0200 Subject: [PATCH 389/898] The units in an application need to be idle before we try to series-upgrade them --- .../utilities/parallel_series_upgrade.py | 63 ++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 9e29d6d..de740ce 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -17,6 +17,8 @@ import asyncio + +import concurrent import collections import copy import logging @@ -207,11 +209,15 @@ async def parallel_series_upgrade( application, to_series=to_series) if origin: await os_utils.async_set_origin(application, origin) + app_idle = [ + wait_for_unit_idle(unit) for unit in status["units"] + ] + await asyncio.gather(*app_idle) prepare_group = [ series_upgrade_utils.async_prepare_series_upgrade( machine, to_series=to_series) for machine in machines] - asyncio.gather(*prepare_group) + await asyncio.gather(*prepare_group) await series_upgrade_utils.async_prepare_series_upgrade( leader_machine, to_series=to_series) if leader_machine not in completed_machines: @@ -224,6 +230,7 @@ async def parallel_series_upgrade( for machine in machines ] await asyncio.gather(*upgrade_group) + completed_machines.extend(machines) run_post_application_upgrade_functions(post_application_upgrade_functions) @@ -310,6 +317,7 @@ async def serial_series_upgrade( if origin: await os_utils.async_set_origin(application, origin) if not follower_first and leader_machine not in completed_machines: + await wait_for_unit_idle(leader) await series_upgrade_utils.async_prepare_series_upgrade( leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" @@ -318,8 +326,14 @@ async def serial_series_upgrade( leader_machine, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) + completed_machines.append(leader_machine) - for machine in machines: + # for machine in machines: + for unit_name, unit in non_leaders.items(): + machine = unit['machine'] + if machine in completed_machines: + continue + await wait_for_unit_idle(unit_name) await series_upgrade_utils.async_prepare_series_upgrade( machine, to_series=to_series) logging.info("About to upgrade follower of {}: {}" @@ -328,8 +342,10 @@ async def serial_series_upgrade( machine, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) + completed_machines.append(machine) if follower_first and leader_machine not in completed_machines: + await wait_for_unit_idle(leader) await series_upgrade_utils.async_prepare_series_upgrade( leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" @@ -338,6 +354,7 @@ async def serial_series_upgrade( leader_machine, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) + completed_machines.append(leader_machine) await run_post_application_upgrade_functions( post_application_upgrade_functions) @@ -577,3 +594,45 @@ async def async_do_release_upgrade(machine): 'do-release-upgrade -f DistUpgradeViewNonInteractive') await run_on_machine(machine, do_release_upgrade_cmd, timeout="120m") + + +# TODO: Move these functions into zaza.model +async def wait_for_unit_idle(unit_name, timeout=600): + """Wait until the unit's agent is idle. + + :param unit_name: The unit name of the application, ex: mysql/0 + :type unit_name: str + :param timeout: How long to wait before timing out + :type timeout: int + :returns: None + :rtype: None + """ + app = unit_name.split('/')[0] + try: + await model.async_block_until( + _unit_idle(app, unit_name), + timeout=timeout) + except concurrent.futures._base.TimeoutError: + raise model.ModelTimeout("Zaza has timed out waiting on {} to " + "reach idle state.".format(unit_name)) + + +def _unit_idle(app, unit_name): + async def f(): + x = await get_agent_status(app, unit_name) + return x == "idle" + return f + + +async def get_agent_status(app, unit_name): + """Get the current status of the specified unit. + + :param app: The name of the Juju application, ex: mysql + :type app: str + :param unit_name: The unit name of the application, ex: mysql/0 + :type unit_name: str + :returns: The agent status, either active / idle, returned by Juju + :rtype: str + """ + return (await model.async_get_status()). \ + applications[app]['units'][unit_name]['agent-status']['status'] From 1538db58ffe89c0a022c6b6c44e6f98005452fac Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 14 Apr 2020 12:45:37 +0200 Subject: [PATCH 390/898] Ensure that upgrading non-leader first can handle pause arguments --- zaza/openstack/utilities/series_upgrade.py | 80 +++++++++++++++++++--- 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 7f3efa3..605aca9 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -107,11 +107,15 @@ def run_post_upgrade_functions(post_upgrade_functions): cl_utils.get_class(func)() -def series_upgrade_non_leaders_first(application, from_series="trusty", - to_series="xenial", - origin='openstack-origin', - completed_machines=[], - post_upgrade_functions=None): +def series_upgrade_non_leaders_first( + application, from_series="trusty", + to_series="xenial", + origin='openstack-origin', + completed_machines=[], + post_upgrade_functions=None, + pause_non_leader_primary=False, + pause_non_leader_subordinate=False +): """Series upgrade non leaders first. Wrap all the functionality to handle series upgrade for charms @@ -129,6 +133,14 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", :param completed_machines: List of completed machines which do no longer require series upgrade. :type completed_machines: list + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param from_series: The series from which to upgrade :returns: None :rtype: None """ @@ -141,6 +153,23 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", else: non_leaders.append(unit) + # Pause the non-leaders + for unit in non_leaders: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + _app = subordinate.split('/')[0] + if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: + logging.info("Skipping pausing {} - blacklisted" + .format(subordinate)) + else: + logging.info("Pausing {}".format(subordinate)) + model.run_action( + subordinate, "pause", action_params={}) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + model.run_action(unit, "pause", action_params={}) + # Series upgrade the non-leaders first for unit in non_leaders: machine = status["units"][unit]["machine"] @@ -173,12 +202,16 @@ def series_upgrade_non_leaders_first(application, from_series="trusty", model.block_until_all_units_idle() -async def async_series_upgrade_non_leaders_first(application, - from_series="trusty", - to_series="xenial", - origin='openstack-origin', - completed_machines=[], - post_upgrade_functions=None): +async def async_series_upgrade_non_leaders_first( + application, + from_series="trusty", + to_series="xenial", + origin='openstack-origin', + completed_machines=[], + post_upgrade_functions=None, + pause_non_leader_primary=False, + pause_non_leader_subordinate=False +): """Series upgrade non leaders first. Wrap all the functionality to handle series upgrade for charms @@ -196,6 +229,14 @@ async def async_series_upgrade_non_leaders_first(application, :param completed_machines: List of completed machines which do no longer require series upgrade. :type completed_machines: list + :param pause_non_leader_primary: Whether the non-leader applications should + be paused + :type pause_non_leader_primary: bool + :param pause_non_leader_subordinate: Whether the non-leader subordinate + hacluster applications should be + paused + :type pause_non_leader_subordinate: bool + :param from_series: The series from which to upgrade :returns: None :rtype: None """ @@ -208,6 +249,23 @@ async def async_series_upgrade_non_leaders_first(application, else: non_leaders.append(unit) + # Pause the non-leaders + for unit in non_leaders: + if pause_non_leader_subordinate: + if status["units"][unit].get("subordinates"): + for subordinate in status["units"][unit]["subordinates"]: + _app = subordinate.split('/')[0] + if _app in SUBORDINATE_PAUSE_RESUME_BLACKLIST: + logging.info("Skipping pausing {} - blacklisted" + .format(subordinate)) + else: + logging.info("Pausing {}".format(subordinate)) + await model.async_run_action( + subordinate, "pause", action_params={}) + if pause_non_leader_primary: + logging.info("Pausing {}".format(unit)) + await model.async_run_action(unit, "pause", action_params={}) + # Series upgrade the non-leaders first for unit in non_leaders: machine = status["units"][unit]["machine"] From 83246a93050b3d6f1bd591aaeae11c4ab67fc256 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 9 Apr 2020 17:02:49 +0200 Subject: [PATCH 391/898] Print stderr when failing to ping new instance --- .../utilities/test_zaza_utilities_openstack.py | 15 +++++++++------ zaza/openstack/configure/guest.py | 9 ++++++++- zaza/openstack/utilities/openstack.py | 3 ++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 915d661..46efb0d 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -16,6 +16,7 @@ import copy import datetime import io import mock +import subprocess import tenacity import unit_tests.utils as ut_utils @@ -664,17 +665,19 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): []) def test_ping_response(self): - self.patch_object(openstack_utils.subprocess, 'check_call') + self.patch_object(openstack_utils.subprocess, 'run') openstack_utils.ping_response('10.0.0.10') - self.check_call.assert_called_once_with( - ['ping', '-c', '1', '-W', '1', '10.0.0.10'], stdout=-3) + self.run.assert_called_once_with( + ['ping', '-c', '1', '-W', '1', '10.0.0.10'], check=True, + stdout=mock.ANY, stderr=mock.ANY) def test_ping_response_fail(self): openstack_utils.ping_response.retry.wait = \ tenacity.wait_none() - self.patch_object(openstack_utils.subprocess, 'check_call') - self.check_call.side_effect = Exception() - with self.assertRaises(Exception): + self.patch_object(openstack_utils.subprocess, 'run') + self.run.side_effect = subprocess.CalledProcessError(returncode=42, + cmd='mycmd') + with self.assertRaises(subprocess.CalledProcessError): openstack_utils.ping_response('10.0.0.10') def test_ssh_test(self): diff --git a/zaza/openstack/configure/guest.py b/zaza/openstack/configure/guest.py index 920b539..bd70bca 100644 --- a/zaza/openstack/configure/guest.py +++ b/zaza/openstack/configure/guest.py @@ -16,6 +16,7 @@ """Encapsulate nova testing.""" +import subprocess import logging import time @@ -132,7 +133,13 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, external_network_name, port=port)['floating_ip_address'] logging.info('Assigned floating IP {} to {}'.format(ip, vm_name)) - openstack_utils.ping_response(ip) + try: + openstack_utils.ping_response(ip) + except subprocess.CalledProcessError as e: + logging.error('Pinging {} failed with {}'.format(ip, e.returncode)) + logging.error('stdout: {}'.format(e.stdout)) + logging.error('stderr: {}'.format(e.stderr)) + raise # Check ssh'ing to instance. logging.info('Testing ssh access.') diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index f4f1374..de324e4 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2258,7 +2258,8 @@ def ping_response(ip): :raises: subprocess.CalledProcessError """ cmd = ['ping', '-c', '1', '-W', '1', ip] - subprocess.check_call(cmd, stdout=subprocess.DEVNULL) + subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + check=True) def ssh_test(username, ip, vm_name, password=None, privkey=None): From 592622935cae297983932792b643170a0eddfa27 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 15 Apr 2020 09:40:37 +0200 Subject: [PATCH 392/898] tidy up and add more debugability --- ..._zaza_utilities_parallel_series_upgrade.py | 15 ++++-- zaza/openstack/utilities/generic.py | 7 +++ .../utilities/parallel_series_upgrade.py | 53 ++++++++----------- zaza/openstack/utilities/series_upgrade.py | 2 + 4 files changed, 43 insertions(+), 34 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 8c17cbb..36638b1 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -177,6 +177,9 @@ class TestParallelSeriesUpgrade(AioTestCase): self.async_run_action = mock.AsyncMock() self.model.async_run_action = self.async_run_action + self.async_block_until = mock.AsyncMock() + self.model.async_block_until = self.async_block_until + @mock.patch.object(upgrade_utils.cl_utils, 'get_class') async def test_run_post_application_upgrade_functions( self, @@ -225,11 +228,14 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_async_set_series.assert_called_once_with( 'mongodb', to_series='xenial') self.juju_status.assert_called() + + # The below is using `any_order=True` because the ordering is + # undetermined and differs between python versions mock_async_prepare_series_upgrade.assert_has_calls([ mock.call('1', to_series='xenial'), mock.call('2', to_series='xenial'), mock.call('0', to_series='xenial'), - ]) + ], any_order=True) mock_maybe_pause_things.assert_called() mock_series_upgrade_machine.assert_has_calls([ mock.call( @@ -495,7 +501,7 @@ class TestParallelSeriesUpgrade(AioTestCase): async def test_reboot(self, mock_run_on_machine): await upgrade_utils.reboot('1') mock_run_on_machine.assert_called_once_with( - '1', 'shutdown --reboot now & exit' + '1', 'sudo init 6 & exit' ) async def test_run_on_machine(self): @@ -517,10 +523,11 @@ class TestParallelSeriesUpgrade(AioTestCase): async def test_async_dist_upgrade(self, mock_run_on_machine): await upgrade_utils.async_dist_upgrade('1') apt_update_command = ( - """yes | sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """yes | sudo DEBIAN_FRONTEND=noninteractive """ + """apt-get --assume-yes """ """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") mock_run_on_machine.assert_has_calls([ - mock.call('1', 'sudo apt update'), + mock.call('1', 'sudo apt-get update'), mock.call('1', apt_update_command), ]) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 50dcc69..0874cc6 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -385,10 +385,17 @@ async def check_call(cmd): stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) stdout, stderr = await proc.communicate() + stdout = stdout.decode('utf-8') + stderr = stderr.decode('utf-8') if proc.returncode != 0: logging.warn("STDOUT: {}".format(stdout)) logging.warn("STDERR: {}".format(stderr)) raise subprocess.CalledProcessError(proc.returncode, cmd) + else: + if stderr: + logging.info("STDERR: {} ({})".format(stderr, ' '.join(cmd))) + if stdout: + logging.info("STDOUT: {} ({})".format(stdout, ' '.join(cmd))) def set_dpkg_non_interactive_on_unit( diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index de740ce..dfbd36e 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -214,12 +214,10 @@ async def parallel_series_upgrade( ] await asyncio.gather(*app_idle) prepare_group = [ - series_upgrade_utils.async_prepare_series_upgrade( - machine, to_series=to_series) + prepare_series_upgrade(machine, to_series=to_series) for machine in machines] await asyncio.gather(*prepare_group) - await series_upgrade_utils.async_prepare_series_upgrade( - leader_machine, to_series=to_series) + await prepare_series_upgrade(leader_machine, to_series=to_series) if leader_machine not in completed_machines: machines.append(leader_machine) upgrade_group = [ @@ -231,7 +229,8 @@ async def parallel_series_upgrade( ] await asyncio.gather(*upgrade_group) completed_machines.extend(machines) - run_post_application_upgrade_functions(post_application_upgrade_functions) + await run_post_application_upgrade_functions( + post_application_upgrade_functions) async def serial_series_upgrade( @@ -302,11 +301,6 @@ async def serial_series_upgrade( leader_machine = leader_unit["machine"] leader = leader_name - machines = [ - unit["machine"] for name, unit - in non_leaders.items() - if unit['machine'] not in completed_machines] - await maybe_pause_things( status, non_leaders, @@ -318,8 +312,7 @@ async def serial_series_upgrade( await os_utils.async_set_origin(application, origin) if not follower_first and leader_machine not in completed_machines: await wait_for_unit_idle(leader) - await series_upgrade_utils.async_prepare_series_upgrade( - leader_machine, to_series=to_series) + await prepare_series_upgrade(leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) await series_upgrade_machine( @@ -334,8 +327,7 @@ async def serial_series_upgrade( if machine in completed_machines: continue await wait_for_unit_idle(unit_name) - await series_upgrade_utils.async_prepare_series_upgrade( - machine, to_series=to_series) + await prepare_series_upgrade(machine, to_series=to_series) logging.info("About to upgrade follower of {}: {}" .format(application, machine)) await series_upgrade_machine( @@ -346,8 +338,7 @@ async def serial_series_upgrade( if follower_first and leader_machine not in completed_machines: await wait_for_unit_idle(leader) - await series_upgrade_utils.async_prepare_series_upgrade( - leader_machine, to_series=to_series) + await prepare_series_upgrade(leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) await series_upgrade_machine( @@ -383,9 +374,8 @@ async def series_upgrade_machine( :rtype: None """ logging.info( - "About to dist-upgrade ({})".format(machine)) - - run_pre_upgrade_functions(machine, pre_upgrade_functions) + "About to series-upgrade ({})".format(machine)) + await run_pre_upgrade_functions(machine, pre_upgrade_functions) await async_dist_upgrade(machine) await async_do_release_upgrade(machine) await reboot(machine) @@ -401,37 +391,40 @@ async def run_pre_upgrade_functions(machine, pre_upgrade_functions): :param machine: Machine that is about to be upgraded :type machine: str - :param pre_upgrade_functions: List of functions + :param pre_upgrade_functions: List of awaitable functions :type pre_upgrade_functions: [function, function, ...] """ if pre_upgrade_functions: for func in pre_upgrade_functions: logging.info("Running {}".format(func)) - await cl_utils.get_class(func)(machine) + m = cl_utils.get_class(func) + await m(machine) async def run_post_upgrade_functions(post_upgrade_functions): """Execute list supplied functions. - :param post_upgrade_functions: List of functions + :param post_upgrade_functions: List of awaitable functions :type post_upgrade_functions: [function, function, ...] """ if post_upgrade_functions: for func in post_upgrade_functions: logging.info("Running {}".format(func)) - await cl_utils.get_class(func)() + m = cl_utils.get_class(func) + await m() async def run_post_application_upgrade_functions(post_upgrade_functions): """Execute list supplied functions. - :param post_upgrade_functions: List of functions + :param post_upgrade_functions: List of awaitable functions :type post_upgrade_functions: [function, function, ...] """ if post_upgrade_functions: for func in post_upgrade_functions: logging.info("Running {}".format(func)) - await cl_utils.get_class(func)() + m = cl_utils.get_class(func) + await m() async def maybe_pause_things( @@ -515,7 +508,7 @@ async def prepare_series_upgrade(machine, to_series): :returns: None :rtype: None """ - logging.debug("Preparing series upgrade for: {}".format(machine)) + logging.info("Preparing series upgrade for: {}".format(machine)) await series_upgrade_utils.async_prepare_series_upgrade( machine, to_series=to_series) @@ -529,7 +522,7 @@ async def reboot(machine): :rtype: None """ try: - await run_on_machine(machine, 'shutdown --reboot now & exit') + await run_on_machine(machine, 'sudo init 6 & exit') # await run_on_machine(unit, "sudo reboot && exit") except subprocess.CalledProcessError as e: logging.warn("Error doing reboot: {}".format(e)) @@ -556,7 +549,7 @@ async def run_on_machine(machine, command, model_name=None, timeout=None): if timeout: cmd.append('--timeout={}'.format(timeout)) cmd.append(command) - logging.debug("About to call '{}'".format(cmd)) + logging.info("About to call '{}'".format(cmd)) await os_utils.check_call(cmd) @@ -569,12 +562,12 @@ async def async_dist_upgrade(machine): :rtype: None """ logging.info('Updating package db ' + machine) - update_cmd = 'sudo apt update' + update_cmd = 'sudo apt-get update' await run_on_machine(machine, update_cmd) logging.info('Updating existing packages ' + machine) dist_upgrade_cmd = ( - """yes | sudo DEBIAN_FRONTEND=noninteractive apt --assume-yes """ + """yes | sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes """ """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") await run_on_machine(machine, dist_upgrade_cmd) diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 7f3efa3..f7e79a8 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -687,6 +687,7 @@ async def async_complete_series_upgrade(machine_num): juju_model = await model.async_get_juju_model() cmd = ["juju", "upgrade-series", "-m", juju_model, machine_num, "complete"] + logging.info("About to call '{}'".format(cmd)) await os_utils.check_call(cmd) @@ -704,6 +705,7 @@ async def async_set_series(application, to_series): juju_model = await model.async_get_juju_model() cmd = ["juju", "set-series", "-m", juju_model, application, to_series] + logging.info("About to call '{}'".format(cmd)) await os_utils.check_call(cmd) From c492ecdcac3b2724833c347e978de97ea2e626d7 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 15 Apr 2020 15:48:37 +0200 Subject: [PATCH 393/898] Add functional test that covers Ubuntu Lite in Parallel --- .../series_upgrade/parallel_tests.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py index 5307339..6408cf0 100644 --- a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -168,6 +168,59 @@ class BionicFocalSeriesUpgrade(OpenStackParallelSeriesUpgrade): cls.to_series = "focal" +class UbuntuLiteParallelSeriesUpgrade(unittest.TestCase): + """ubuntu Lite Parallel Series Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Series Upgrades.""" + cli_utils.setup_logging() + cls.from_series = None + cls.to_series = None + + def test_200_run_series_upgrade(self): + """Run series upgrade.""" + # Set Feature Flag + os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" + parallel_series_upgrade.upgrade_ubuntu_lite( + from_series=self.from_series, + to_series=self.to_series + ) + + +class TrustyXenialSeriesUpgradeUbuntu(UbuntuLiteParallelSeriesUpgrade): + """OpenStack Trusty to Xenial Series Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Trusty to Xenial Series Upgrades.""" + super(TrustyXenialSeriesUpgradeUbuntu, cls).setUpClass() + cls.from_series = "trusty" + cls.to_series = "xenial" + + +class XenialBionicSeriesUpgradeUbuntu(UbuntuLiteParallelSeriesUpgrade): + """OpenStack Xenial to Bionic Series Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(XenialBionicSeriesUpgradeUbuntu, cls).setUpClass() + cls.from_series = "xenial" + cls.to_series = "bionic" + + +class BionicFocalSeriesUpgradeUbuntu(UbuntuLiteParallelSeriesUpgrade): + """OpenStack Bionic to FocalSeries Upgrade.""" + + @classmethod + def setUpClass(cls): + """Run setup for Xenial to Bionic Series Upgrades.""" + super(BionicFocalSeriesUpgradeUbuntu, cls).setUpClass() + cls.from_series = "bionic" + cls.to_series = "focal" + + if __name__ == "__main__": from_series = os.environ.get("FROM_SERIES") if from_series == "trusty": From fa4587f3668aa1adcd7e4dfadff01c8d5768f848 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 15 Apr 2020 16:34:21 +0200 Subject: [PATCH 394/898] Ensure that origin is set only after the first machine is rebooting --- ..._zaza_utilities_parallel_series_upgrade.py | 71 +++++++++++++++---- .../utilities/parallel_series_upgrade.py | 18 +++-- 2 files changed, 69 insertions(+), 20 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 36638b1..616c590 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -200,7 +200,6 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_get_class.assert_called_once_with('my.thing') called.assert_called_once_with('1') - @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') @mock.patch.object( upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') @@ -214,7 +213,6 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_async_set_series, mock_async_prepare_series_upgrade, mock_post_application_upgrade_functions, - mock_async_set_origin, ): self.juju_status.return_value.applications.__getitem__.return_value = \ FAKE_STATUS_MONGO @@ -240,24 +238,28 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_series_upgrade_machine.assert_has_calls([ mock.call( '1', + origin=None, + application='mongodb', files=None, workaround_script=None, post_upgrade_functions=[]), mock.call( '2', + origin=None, + application='mongodb', files=None, workaround_script=None, post_upgrade_functions=[]), mock.call( '0', + origin=None, + application='mongodb', files=None, workaround_script=None, post_upgrade_functions=[]), ]) - mock_async_set_origin.assert_not_called() mock_post_application_upgrade_functions.assert_called_once_with([]) - @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') @mock.patch.object( upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') @@ -271,7 +273,6 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_async_set_series, mock_async_prepare_series_upgrade, mock_post_application_upgrade_functions, - mock_async_set_origin, ): self.juju_status.return_value.applications.__getitem__.return_value = \ FAKE_STATUS_MONGO @@ -294,24 +295,28 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_series_upgrade_machine.assert_has_calls([ mock.call( '1', + origin=None, + application='mongodb', files=None, workaround_script=None, post_upgrade_functions=[]), mock.call( '2', + origin=None, + application='mongodb', files=None, workaround_script=None, post_upgrade_functions=[]), mock.call( '0', + origin=None, + application='mongodb', files=None, workaround_script=None, post_upgrade_functions=[]), ]) - mock_async_set_origin.assert_not_called() mock_post_application_upgrade_functions.assert_called_once_with([]) - @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') @mock.patch.object( upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') @@ -325,7 +330,6 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_async_set_series, mock_async_prepare_series_upgrade, mock_post_application_upgrade_functions, - mock_async_set_origin, ): await upgrade_utils.parallel_series_upgrade( 'app', @@ -335,34 +339,39 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_async_set_series.assert_called_once_with( 'app', to_series='xenial') self.juju_status.assert_called() + # The below is using `any_order=True` because the ordering is + # undetermined and differs between python versions mock_async_prepare_series_upgrade.assert_has_calls([ mock.call('1', to_series='xenial'), mock.call('2', to_series='xenial'), mock.call('0', to_series='xenial'), - ]) + ], any_order=True) mock_maybe_pause_things.assert_called() mock_series_upgrade_machine.assert_has_calls([ mock.call( '1', + origin='openstack-origin', + application='app', files=None, workaround_script=None, post_upgrade_functions=None), mock.call( '2', + origin='openstack-origin', + application='app', files=None, workaround_script=None, post_upgrade_functions=None), mock.call( '0', + origin='openstack-origin', + application='app', files=None, workaround_script=None, post_upgrade_functions=None), ]) - mock_async_set_origin.assert_called_once_with( - 'app', 'openstack-origin') mock_post_application_upgrade_functions.assert_called_once_with(None) - @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') @mock.patch.object(upgrade_utils, 'run_post_application_upgrade_functions') @mock.patch.object( upgrade_utils.series_upgrade_utils, 'async_prepare_series_upgrade') @@ -376,7 +385,6 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_async_set_series, mock_async_prepare_series_upgrade, mock_post_application_upgrade_functions, - mock_async_set_origin, ): await upgrade_utils.serial_series_upgrade( 'app', @@ -395,22 +403,26 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_series_upgrade_machine.assert_has_calls([ mock.call( '0', + origin='openstack-origin', + application='app', files=None, workaround_script=None, post_upgrade_functions=None), mock.call( '1', + origin='openstack-origin', + application='app', files=None, workaround_script=None, post_upgrade_functions=None), mock.call( '2', + origin='openstack-origin', + application='app', files=None, workaround_script=None, post_upgrade_functions=None), ]) - mock_async_set_origin.assert_called_once_with( - 'app', 'openstack-origin') mock_post_application_upgrade_functions.assert_called_once_with(None) @mock.patch.object( @@ -436,6 +448,35 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_reboot.assert_called_once_with('1') mock_async_complete_series_upgrade.assert_called_once_with('1') + @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') + @mock.patch.object( + upgrade_utils.series_upgrade_utils, 'async_complete_series_upgrade') + @mock.patch.object(upgrade_utils, 'reboot') + @mock.patch.object(upgrade_utils, 'async_do_release_upgrade') + @mock.patch.object(upgrade_utils, 'async_dist_upgrade') + async def test_series_upgrade_machine_with_source( + self, + mock_async_dist_upgrade, + mock_async_do_release_upgrade, + mock_reboot, + mock_async_complete_series_upgrade, + mock_async_set_origin + ): + await upgrade_utils.series_upgrade_machine( + '1', + origin='openstack-origin', + application='app', + post_upgrade_functions=None, + pre_upgrade_functions=None, + files=None, + workaround_script=None) + mock_async_dist_upgrade.assert_called_once_with('1') + mock_async_do_release_upgrade.assert_called_once_with('1') + mock_reboot.assert_called_once_with('1') + mock_async_complete_series_upgrade.assert_called_once_with('1') + mock_async_set_origin.assert_called_once_with( + 'app', 'openstack-origin') + async def test_maybe_pause_things_primary(self): await upgrade_utils.maybe_pause_things( FAKE_STATUS, diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index dfbd36e..980f556 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -207,8 +207,6 @@ async def parallel_series_upgrade( pause_non_leader_primary) await series_upgrade_utils.async_set_series( application, to_series=to_series) - if origin: - await os_utils.async_set_origin(application, origin) app_idle = [ wait_for_unit_idle(unit) for unit in status["units"] ] @@ -223,6 +221,8 @@ async def parallel_series_upgrade( upgrade_group = [ series_upgrade_machine( machine, + origin=origin, + application=application, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) for machine in machines @@ -308,8 +308,6 @@ async def serial_series_upgrade( pause_non_leader_primary) await series_upgrade_utils.async_set_series( application, to_series=to_series) - if origin: - await os_utils.async_set_origin(application, origin) if not follower_first and leader_machine not in completed_machines: await wait_for_unit_idle(leader) await prepare_series_upgrade(leader_machine, to_series=to_series) @@ -317,6 +315,8 @@ async def serial_series_upgrade( .format(application, leader_machine)) await series_upgrade_machine( leader_machine, + origin=origin, + application=application, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) completed_machines.append(leader_machine) @@ -332,6 +332,8 @@ async def serial_series_upgrade( .format(application, machine)) await series_upgrade_machine( machine, + origin=origin, + application=application, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) @@ -343,6 +345,8 @@ async def serial_series_upgrade( .format(application, leader_machine)) await series_upgrade_machine( leader_machine, + origin=origin, + application=application, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) completed_machines.append(leader_machine) @@ -352,6 +356,8 @@ async def serial_series_upgrade( async def series_upgrade_machine( machine, + origin=None, + application=None, post_upgrade_functions=None, pre_upgrade_functions=None, files=None, @@ -379,6 +385,8 @@ async def series_upgrade_machine( await async_dist_upgrade(machine) await async_do_release_upgrade(machine) await reboot(machine) + if origin: + await os_utils.async_set_origin(application, origin) await series_upgrade_utils.async_complete_series_upgrade(machine) await run_post_upgrade_functions(post_upgrade_functions) @@ -462,8 +470,8 @@ async def maybe_pause_things( logging.info("Pausing {}".format(unit)) leader_pauses.append( model.async_run_action(unit, "pause", action_params={})) - await asyncio.gather(*subordinate_pauses) await asyncio.gather(*leader_pauses) + await asyncio.gather(*subordinate_pauses) def get_leader_and_non_leaders(status): From 94434f62ac3f2022275882ba965192f2ad554340 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 14 Apr 2020 14:05:30 +0200 Subject: [PATCH 395/898] Make nova.SecurityTests runable against nova-cloud-controller Needed for https://bugs.launchpad.net/charm-nova-cloud-controller/+bug/1828424 --- zaza/openstack/charm_tests/nova/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index f750b5a..be3c6d6 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -157,11 +157,11 @@ class NovaCompute(test_utils.OpenStackBaseTest): class SecurityTests(test_utils.OpenStackBaseTest): - """Nova Compute security tests tests.""" + """nova-compute and nova-cloud-controller security tests.""" @classmethod def setUpClass(cls): - """Run class setup for running Nova Compute SecurityTests.""" + """Run class setup for running Nova SecurityTests.""" super(SecurityTests, cls).setUpClass() def test_security_checklist(self): @@ -181,7 +181,7 @@ class SecurityTests(test_utils.OpenStackBaseTest): 'validate-uses-keystone', ] - for unit in zaza.model.get_units('nova-compute', + for unit in zaza.model.get_units(self.application_name, model_name=self.model_name): logging.info('Running `security-checklist` action' ' on unit {}'.format(unit.entity_id)) From 2bc2234cad71b31253d8d17e59a1abb3be590f56 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 16 Apr 2020 11:53:35 +0200 Subject: [PATCH 396/898] Update to migrate bits out to Zaza --- ..._zaza_utilities_parallel_series_upgrade.py | 33 ++------ .../utilities/parallel_series_upgrade.py | 84 ++----------------- 2 files changed, 18 insertions(+), 99 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 616c590..aefa00b 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -179,6 +179,9 @@ class TestParallelSeriesUpgrade(AioTestCase): self.async_block_until = mock.AsyncMock() self.model.async_block_until = self.async_block_until + self.model.async_wait_for_unit_idle = mock.AsyncMock() + self.async_run_on_machine = mock.AsyncMock() + self.model.async_run_on_machine = self.async_run_on_machine @mock.patch.object(upgrade_utils.cl_utils, 'get_class') async def test_run_post_application_upgrade_functions( @@ -520,13 +523,12 @@ class TestParallelSeriesUpgrade(AioTestCase): pause_non_leader_primary=False) self.async_run_action.assert_not_called() - @mock.patch.object(upgrade_utils, 'run_on_machine') - async def test_async_do_release_upgrade(self, mock_run_on_machine): + async def test_async_do_release_upgrade(self): await upgrade_utils.async_do_release_upgrade('1') do_release_upgrade_cmd = ( 'yes | sudo DEBIAN_FRONTEND=noninteractive ' 'do-release-upgrade -f DistUpgradeViewNonInteractive') - mock_run_on_machine.assert_called_once_with( + self.async_run_on_machine.assert_called_once_with( '1', do_release_upgrade_cmd, timeout='120m' ) @@ -538,37 +540,20 @@ class TestParallelSeriesUpgrade(AioTestCase): '1', to_series='xenial' ) - @mock.patch.object(upgrade_utils, 'run_on_machine') - async def test_reboot(self, mock_run_on_machine): + async def test_reboot(self): await upgrade_utils.reboot('1') - mock_run_on_machine.assert_called_once_with( + self.async_run_on_machine.assert_called_once_with( '1', 'sudo init 6 & exit' ) - async def test_run_on_machine(self): - await upgrade_utils.run_on_machine('1', 'test') - self.check_call.assert_called_once_with( - ['juju', 'run', '--machine=1', 'test']) - - async def test_run_on_machine_with_timeout(self): - await upgrade_utils.run_on_machine('1', 'test', timeout='20m') - self.check_call.assert_called_once_with( - ['juju', 'run', '--machine=1', '--timeout=20m', 'test']) - - async def test_run_on_machine_with_model(self): - await upgrade_utils.run_on_machine('1', 'test', model_name='test') - self.check_call.assert_called_once_with( - ['juju', 'run', '--machine=1', '--model=test', 'test']) - - @mock.patch.object(upgrade_utils, 'run_on_machine') - async def test_async_dist_upgrade(self, mock_run_on_machine): + async def test_async_dist_upgrade(self): await upgrade_utils.async_dist_upgrade('1') apt_update_command = ( """yes | sudo DEBIAN_FRONTEND=noninteractive """ """apt-get --assume-yes """ """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") - mock_run_on_machine.assert_has_calls([ + self.async_run_on_machine.assert_has_calls([ mock.call('1', 'sudo apt-get update'), mock.call('1', apt_update_command), ]) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 980f556..eea2edb 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -18,7 +18,6 @@ import asyncio -import concurrent import collections import copy import logging @@ -208,7 +207,7 @@ async def parallel_series_upgrade( await series_upgrade_utils.async_set_series( application, to_series=to_series) app_idle = [ - wait_for_unit_idle(unit) for unit in status["units"] + model.async_wait_for_unit_idle(unit) for unit in status["units"] ] await asyncio.gather(*app_idle) prepare_group = [ @@ -309,7 +308,7 @@ async def serial_series_upgrade( await series_upgrade_utils.async_set_series( application, to_series=to_series) if not follower_first and leader_machine not in completed_machines: - await wait_for_unit_idle(leader) + await model.async_wait_for_unit_idle(leader) await prepare_series_upgrade(leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) @@ -326,7 +325,7 @@ async def serial_series_upgrade( machine = unit['machine'] if machine in completed_machines: continue - await wait_for_unit_idle(unit_name) + await model.async_wait_for_unit_idle(unit_name) await prepare_series_upgrade(machine, to_series=to_series) logging.info("About to upgrade follower of {}: {}" .format(application, machine)) @@ -339,7 +338,7 @@ async def serial_series_upgrade( completed_machines.append(machine) if follower_first and leader_machine not in completed_machines: - await wait_for_unit_idle(leader) + await model.async_wait_for_unit_idle(leader) await prepare_series_upgrade(leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) @@ -530,37 +529,13 @@ async def reboot(machine): :rtype: None """ try: - await run_on_machine(machine, 'sudo init 6 & exit') + await model.async_run_on_machine(machine, 'sudo init 6 & exit') # await run_on_machine(unit, "sudo reboot && exit") except subprocess.CalledProcessError as e: logging.warn("Error doing reboot: {}".format(e)) pass -async def run_on_machine(machine, command, model_name=None, timeout=None): - """Juju run on unit. - - :param model_name: Name of model unit is in - :type model_name: str - :param unit_name: Name of unit to match - :type unit: str - :param command: Command to execute - :type command: str - :param timeout: How long in seconds to wait for command to complete - :type timeout: int - :returns: action.data['results'] {'Code': '', 'Stderr': '', 'Stdout': ''} - :rtype: dict - """ - cmd = ['juju', 'run', '--machine={}'.format(machine)] - if model_name: - cmd.append('--model={}'.format(model_name)) - if timeout: - cmd.append('--timeout={}'.format(timeout)) - cmd.append(command) - logging.info("About to call '{}'".format(cmd)) - await os_utils.check_call(cmd) - - async def async_dist_upgrade(machine): """Run dist-upgrade on unit after update package db. @@ -571,14 +546,14 @@ async def async_dist_upgrade(machine): """ logging.info('Updating package db ' + machine) update_cmd = 'sudo apt-get update' - await run_on_machine(machine, update_cmd) + await model.async_run_on_machine(machine, update_cmd) logging.info('Updating existing packages ' + machine) dist_upgrade_cmd = ( """yes | sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes """ """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") - await run_on_machine(machine, dist_upgrade_cmd) + await model.async_run_on_machine(machine, dist_upgrade_cmd) async def async_do_release_upgrade(machine): @@ -594,46 +569,5 @@ async def async_do_release_upgrade(machine): 'yes | sudo DEBIAN_FRONTEND=noninteractive ' 'do-release-upgrade -f DistUpgradeViewNonInteractive') - await run_on_machine(machine, do_release_upgrade_cmd, timeout="120m") - - -# TODO: Move these functions into zaza.model -async def wait_for_unit_idle(unit_name, timeout=600): - """Wait until the unit's agent is idle. - - :param unit_name: The unit name of the application, ex: mysql/0 - :type unit_name: str - :param timeout: How long to wait before timing out - :type timeout: int - :returns: None - :rtype: None - """ - app = unit_name.split('/')[0] - try: - await model.async_block_until( - _unit_idle(app, unit_name), - timeout=timeout) - except concurrent.futures._base.TimeoutError: - raise model.ModelTimeout("Zaza has timed out waiting on {} to " - "reach idle state.".format(unit_name)) - - -def _unit_idle(app, unit_name): - async def f(): - x = await get_agent_status(app, unit_name) - return x == "idle" - return f - - -async def get_agent_status(app, unit_name): - """Get the current status of the specified unit. - - :param app: The name of the Juju application, ex: mysql - :type app: str - :param unit_name: The unit name of the application, ex: mysql/0 - :type unit_name: str - :returns: The agent status, either active / idle, returned by Juju - :rtype: str - """ - return (await model.async_get_status()). \ - applications[app]['units'][unit_name]['agent-status']['status'] + await model.async_run_on_machine( + machine, do_release_upgrade_cmd, timeout="120m") From de028d7eb2b9fe1b35b43a8796c284fdf91be537 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 16 Apr 2020 11:58:54 +0200 Subject: [PATCH 397/898] fix copy-pasta error --- zaza/openstack/utilities/parallel_series_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index eea2edb..29e276d 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -247,7 +247,7 @@ async def serial_series_upgrade( files=None, workaround_script=None ): - """Perform series upgrade on an application in parallel. + """Perform series upgrade on an application in serial. :param unit_name: Unit Name :type unit_name: str From ac227eab08d750b4ced1bef6db4a51a14254c822 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 15 Apr 2020 16:33:57 +0200 Subject: [PATCH 398/898] Fix copy-paste mistakes and typos and remove dead code --- zaza/openstack/charm_tests/aodh/tests.py | 2 +- zaza/openstack/charm_tests/ceilometer/tests.py | 14 +++----------- zaza/openstack/charm_tests/cinder/tests.py | 6 +++--- zaza/openstack/charm_tests/cinder_backup/tests.py | 2 +- zaza/openstack/charm_tests/designate/tests.py | 2 +- zaza/openstack/charm_tests/glance/tests.py | 2 +- zaza/openstack/charm_tests/gnocchi/tests.py | 2 +- zaza/openstack/charm_tests/heat/tests.py | 2 +- zaza/openstack/charm_tests/mysql/tests.py | 4 ++-- zaza/openstack/charm_tests/neutron/tests.py | 4 ++-- zaza/openstack/charm_tests/nova/tests.py | 2 +- zaza/openstack/charm_tests/test_utils.py | 2 +- 12 files changed, 18 insertions(+), 26 deletions(-) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index f43615f..a8c7ee8 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -111,7 +111,7 @@ class AodhTest(test_utils.OpenStackBaseTest): def test_900_restart_on_config_change(self): """Checking restart happens on config change. - Change disk format and assert then change propagates to the correct + Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ # Expected default and alternate values diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index a7f2618..53c3594 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -34,7 +34,6 @@ class CeilometerTest(test_utils.OpenStackBaseTest): XENIAL_NEWTON = openstack_utils.get_os_release('xenial_newton') XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') TRUSTY_MITAKA = openstack_utils.get_os_release('trusty_mitaka') - TRUSTY_LIBERTY = openstack_utils.get_os_release('trusty_liberty') @classmethod def setUpClass(cls): @@ -44,7 +43,7 @@ class CeilometerTest(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list of services for Openstack Release.""" + """Return a list of services for the selected Openstack release.""" services = [] if self.application_name == 'ceilometer-agent': @@ -81,21 +80,14 @@ class CeilometerTest(test_utils.OpenStackBaseTest): services.append('ceilometer-alarm-notifier') services.append('ceilometer-alarm-evaluator') - # if self.current_release >= CeilometerTest.TRUSTY_LIBERTY: - # Liberty and later - # services.append('ceilometer-polling') - # else: - # Juno and earlier - # services.append('ceilometer-agent-central') - return services @property def restartable_services(self): """Return a list of services that are known to be restartable. - For Openstack Release these services are known to be able to be stopped - and started with no issues. + For the selected Openstack release these services are known to be able + to be stopped and started with no issues. """ # Due to Bug #1861321 ceilometer-collector does not reliably # restart. diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 1fa6a30..ea75fe9 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -153,7 +153,7 @@ class CinderTests(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list services for OpenStack release.""" + """Return a list services for the selected Openstack release.""" services = ['cinder-scheduler', 'cinder-volume'] if (openstack_utils.get_os_release() >= openstack_utils.get_os_release('xenial_ocata')): @@ -165,7 +165,7 @@ class CinderTests(test_utils.OpenStackBaseTest): def test_900_restart_on_config_change(self): """Checking restart happens on config change. - Change disk format and assert then change propagates to the correct + Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ # Expected default and alternate values @@ -176,7 +176,7 @@ class CinderTests(test_utils.OpenStackBaseTest): conf_file = '/etc/cinder/cinder.conf' # Make config change, check for service restarts - logging.debug('Setting disk format glance...') + logging.debug('Setting debug mode...') self.restart_on_changed( conf_file, set_default, diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 087bbac..ae1ca50 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -40,7 +40,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list services for Openstack Release.""" + """Return a list services for the selected Openstack release.""" current_release = openstack_utils.get_os_release() services = ['cinder-scheduler', 'cinder-volume'] if (current_release >= diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 8e57273..5066dd4 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -95,7 +95,7 @@ class DesignateTests(BaseDesignateTest): def test_900_restart_on_config_change(self): """Checking restart happens on config change. - Change disk format and assert then change propagates to the correct + Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ # Expected default and alternate values diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index 4f25d81..033ec28 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -44,7 +44,7 @@ class GlanceTest(test_utils.OpenStackBaseTest): def test_411_set_disk_format(self): """Change disk format and check. - Change disk format and assert then change propagates to the correct + Change disk format and assert that change propagates to the correct file and that services are restarted as a result """ # Expected default and alternate values diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 6811880..e73bab7 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -28,7 +28,7 @@ class GnocchiTest(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list of services for Openstack Release.""" + """Return a list of services for the selected Openstack release.""" return ['haproxy', 'gnocchi-metricd', 'apache2'] def test_200_api_connection(self): diff --git a/zaza/openstack/charm_tests/heat/tests.py b/zaza/openstack/charm_tests/heat/tests.py index 040aa02..a0e2a23 100644 --- a/zaza/openstack/charm_tests/heat/tests.py +++ b/zaza/openstack/charm_tests/heat/tests.py @@ -54,7 +54,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list services for OpenStack release. + """Return a list services for the selected Openstack release. :returns: List of services :rtype: [str] diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index a800712..77bc966 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -78,7 +78,7 @@ class MySQLCommonTests(MySQLBaseTest): def test_910_restart_on_config_change(self): """Checking restart happens on config change. - Change disk format and assert then change propagates to the correct + Change max connections and assert that change propagates to the correct file and that services are restarted as a result """ # Expected default and alternate values @@ -86,7 +86,7 @@ class MySQLCommonTests(MySQLBaseTest): set_alternate = {"max-connections": "1000"} # Make config change, check for service restarts - logging.debug("Setting peer timeout ...") + logging.debug("Setting max connections ...") self.restart_on_changed( self.conf_file, set_default, diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 2ae92f6..f4519f8 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -113,7 +113,7 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): def test_900_restart_on_config_change(self): """Checking restart happens on config change. - Change disk format and assert then change propagates to the correct + Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ # Expected default and alternate values @@ -285,7 +285,7 @@ class NeutronApiTest(NeutronCreateNetworkTest): def test_900_restart_on_config_change(self): """Checking restart happens on config change. - Change disk format and assert then change propagates to the correct + Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ # Expected default and alternate values diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index f750b5a..2dc9f6e 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -75,7 +75,7 @@ class NovaCompute(test_utils.OpenStackBaseTest): def test_900_restart_on_config_change(self): """Checking restart happens on config change. - Change disk format and assert then change propagates to the correct + Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ # Expected default and alternate values diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 2abfa62..82fed63 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -259,7 +259,7 @@ class BaseCharmTest(unittest.TestCase): pgrep_full=False): """Run restart on change tests. - Test that changing config results in config file being updates and + Test that changing config results in config file being updated and services restarted. Return config to default_config afterwards :param config_file: Config file to check for settings From 8e9ecc53e8c9cad699fa145dfc8dc581703c5058 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 16 Apr 2020 18:59:07 +0200 Subject: [PATCH 399/898] Openstack -> OpenStack --- .../openstack/charm_tests/ceilometer/tests.py | 4 +-- zaza/openstack/charm_tests/cinder/tests.py | 2 +- .../charm_tests/cinder_backup/tests.py | 2 +- zaza/openstack/charm_tests/gnocchi/tests.py | 2 +- zaza/openstack/charm_tests/heat/tests.py | 2 +- zaza/openstack/charm_tests/kerberos/tests.py | 6 ++--- zaza/openstack/charm_tests/neutron/setup.py | 2 +- zaza/openstack/configure/network.py | 2 +- zaza/openstack/utilities/openstack.py | 26 +++++++++---------- 9 files changed, 24 insertions(+), 24 deletions(-) diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index 53c3594..eb50cb0 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -43,7 +43,7 @@ class CeilometerTest(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list of services for the selected Openstack release.""" + """Return a list of services for the selected OpenStack release.""" services = [] if self.application_name == 'ceilometer-agent': @@ -86,7 +86,7 @@ class CeilometerTest(test_utils.OpenStackBaseTest): def restartable_services(self): """Return a list of services that are known to be restartable. - For the selected Openstack release these services are known to be able + For the selected OpenStack release these services are known to be able to be stopped and started with no issues. """ # Due to Bug #1861321 ceilometer-collector does not reliably diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index ea75fe9..cdccce6 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -153,7 +153,7 @@ class CinderTests(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list services for the selected Openstack release.""" + """Return a list services for the selected OpenStack release.""" services = ['cinder-scheduler', 'cinder-volume'] if (openstack_utils.get_os_release() >= openstack_utils.get_os_release('xenial_ocata')): diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index ae1ca50..b3f7ce7 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -40,7 +40,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list services for the selected Openstack release.""" + """Return a list services for the selected OpenStack release.""" current_release = openstack_utils.get_os_release() services = ['cinder-scheduler', 'cinder-volume'] if (current_release >= diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index e73bab7..f789cff 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -28,7 +28,7 @@ class GnocchiTest(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list of services for the selected Openstack release.""" + """Return a list of services for the selected OpenStack release.""" return ['haproxy', 'gnocchi-metricd', 'apache2'] def test_200_api_connection(self): diff --git a/zaza/openstack/charm_tests/heat/tests.py b/zaza/openstack/charm_tests/heat/tests.py index a0e2a23..d69217f 100644 --- a/zaza/openstack/charm_tests/heat/tests.py +++ b/zaza/openstack/charm_tests/heat/tests.py @@ -54,7 +54,7 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): @property def services(self): - """Return a list services for the selected Openstack release. + """Return a list services for the selected OpenStack release. :returns: List of services :rtype: [str] diff --git a/zaza/openstack/charm_tests/kerberos/tests.py b/zaza/openstack/charm_tests/kerberos/tests.py index e122352..0f627b7 100644 --- a/zaza/openstack/charm_tests/kerberos/tests.py +++ b/zaza/openstack/charm_tests/kerberos/tests.py @@ -31,7 +31,7 @@ class CharmKeystoneKerberosTest(BaseKeystoneTest): super(CharmKeystoneKerberosTest, cls).setUpClass() def test_keystone_kerberos_authentication(self): - """Validate auth to Openstack through the kerberos method.""" + """Validate auth to OpenStack through the kerberos method.""" logging.info('Retrieving a kerberos token with kinit for admin user') ubuntu_test_host = zaza.model.get_units('ubuntu-test-host')[0] @@ -46,7 +46,7 @@ class CharmKeystoneKerberosTest(BaseKeystoneTest): ) assert result['Code'] == '0', result['Stderr'] - logging.info('Fetching user/project info in Openstack') + logging.info('Fetching user/project info in OpenStack') domain_name = 'k8s' project_name = 'k8s' keystone_session = openstack_utils.get_overcloud_keystone_session() @@ -56,7 +56,7 @@ class CharmKeystoneKerberosTest(BaseKeystoneTest): project_id = keystone_client.projects.find(name=project_name).id keystone_hostname = get_unit_full_hostname('keystone') - logging.info('Retrieving an Openstack token to validate auth') + logging.info('Retrieving an OpenStack token to validate auth') cmd = 'openstack token issue -f value -c id ' \ '--os-auth-url http://{}:5000/krb/v3 ' \ '--os-project-id {} ' \ diff --git a/zaza/openstack/charm_tests/neutron/setup.py b/zaza/openstack/charm_tests/neutron/setup.py index e6f19a5..e8f2439 100644 --- a/zaza/openstack/charm_tests/neutron/setup.py +++ b/zaza/openstack/charm_tests/neutron/setup.py @@ -81,7 +81,7 @@ def basic_overcloud_network(limit_gws=None): # Get keystone session keystone_session = openstack_utils.get_overcloud_keystone_session() - # Handle network for Openstack-on-Openstack scenarios + # Handle network for OpenStack-on-OpenStack scenarios if juju_utils.get_provider_type() == "openstack": undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session() network.setup_gateway_ext_port(network_config, diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 12b8d9e..649992c 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -284,7 +284,7 @@ def run_from_cli(**kwargs): network_config = generic_utils.get_network_config( net_topology, ignore_env_vars, net_topology_file) - # Handle network for Openstack-on-Openstack scenarios + # Handle network for OpenStack-on-OpenStack scenarios if juju_utils.get_provider_type() == "openstack": undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session( verify=cacert) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index de324e4..875071b 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -14,7 +14,7 @@ """Module for interacting with OpenStack. -This module contains a number of functions for interacting with Openstack. +This module contains a number of functions for interacting with OpenStack. """ from .os_versions import ( OPENSTACK_CODENAMES, @@ -168,11 +168,11 @@ def get_cacert(): return KEYSTONE_LOCAL_CACERT -# Openstack Client helpers +# OpenStack Client helpers def get_ks_creds(cloud_creds, scope='PROJECT'): """Return the credentials for authenticating against keystone. - :param cloud_creds: Openstack RC environment credentials + :param cloud_creds: OpenStack RC environment credentials :type cloud_creds: dict :param scope: Authentication scope: PROJECT or DOMAIN :type scope: string @@ -375,7 +375,7 @@ def get_keystone_scope(model_name=None): def get_keystone_session(openrc_creds, scope='PROJECT', verify=None): """Return keystone session. - :param openrc_creds: Openstack RC credentials + :param openrc_creds: OpenStack RC credentials :type openrc_creds: dict :param verify: Control TLS certificate verification behaviour :type verify: any (True - use system certs, @@ -444,7 +444,7 @@ def get_keystone_session_client(session, client_api_version=3): def get_keystone_client(openrc_creds, verify=None): """Return authenticated keystoneclient and set auth_ref for service_catalog. - :param openrc_creds: Openstack RC credentials + :param openrc_creds: OpenStack RC credentials :type openrc_creds: dict :param verify: Control TLS certificate verification behaviour :type verify: any @@ -1602,7 +1602,7 @@ def get_undercloud_auth(): return auth_settings -# Openstack Client helpers +# OpenStack Client helpers def get_keystone_ip(model_name=None): """Return the IP address to use when communicating with keystone api. @@ -2115,7 +2115,7 @@ def create_ssh_key(nova_client, keypair_name, replace=False): :param nova_client: Authenticated nova client :type nova_client: novaclient.v2.client.Client - :param keypair_name: Label to apply to keypair in Openstack. + :param keypair_name: Label to apply to keypair in OpenStack. :type keypair_name: str :param replace: Whether to replace the existing keypair if it already exists. @@ -2138,7 +2138,7 @@ def create_ssh_key(nova_client, keypair_name, replace=False): def get_private_key_file(keypair_name): """Location of the file containing the private key with the given label. - :param keypair_name: Label of keypair in Openstack. + :param keypair_name: Label of keypair in OpenStack. :type keypair_name: str :returns: Path to file containing key :rtype: str @@ -2149,7 +2149,7 @@ def get_private_key_file(keypair_name): def write_private_key(keypair_name, key): """Store supplied private key in file. - :param keypair_name: Label of keypair in Openstack. + :param keypair_name: Label of keypair in OpenStack. :type keypair_name: str :param key: PEM Encoded Private Key :type key: str @@ -2161,7 +2161,7 @@ def write_private_key(keypair_name, key): def get_private_key(keypair_name): """Return private key. - :param keypair_name: Label of keypair in Openstack. + :param keypair_name: Label of keypair in OpenStack. :type keypair_name: str :returns: PEM Encoded Private Key :rtype: str @@ -2175,11 +2175,11 @@ def get_private_key(keypair_name): def get_public_key(nova_client, keypair_name): - """Return public key from Openstack. + """Return public key from OpenStack. :param nova_client: Authenticated nova client :type nova_client: novaclient.v2.client.Client - :param keypair_name: Label of keypair in Openstack. + :param keypair_name: Label of keypair in OpenStack. :type keypair_name: str :returns: OpenSSH Encoded Public Key :rtype: str or None @@ -2196,7 +2196,7 @@ def valid_key_exists(nova_client, keypair_name): :param nova_client: Authenticated nova client :type nova_client: novaclient.v2.client.Client - :param keypair_name: Label of keypair in Openstack. + :param keypair_name: Label of keypair in OpenStack. :type keypair_name: str """ pub_key = get_public_key(nova_client, keypair_name) From 49730180f606cec76f4817f86e18c68d5433030c Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 17 Apr 2020 06:56:57 +0200 Subject: [PATCH 400/898] Temporarily disable test_800_ovs_bridges_are_managed_by_us --- zaza/openstack/charm_tests/neutron/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 24e2037..cfbb32b 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -115,12 +115,16 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): _APP_NAME = 'neutron-gateway' + @unittest.expectedFailure def test_800_ovs_bridges_are_managed_by_us(self): """Checking OVS bridges' external-id. OVS bridges created by us should be marked as managed by us in their external-id. See http://docs.openvswitch.org/en/latest/topics/integration/ + + NOTE(lourot): this test is expected to fail as long as this feature + hasn't landed yet: https://review.opendev.org/717074 """ for unit in zaza.model.get_units(self._APP_NAME, model_name=self.model_name): From 53b81209a6af46aa202d07918a125117f3015ef5 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 17 Apr 2020 07:17:49 +0000 Subject: [PATCH 401/898] Add missing doc string --- zaza/openstack/charm_tests/ceph/iscsi/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index 73ea3ed..3c904a6 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -133,6 +133,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): zaza.model.block_until(check_device_present) def create_data_pool(self): + """Create data pool to back iscsi targets.""" generic_utils.assertActionRanOK(zaza.model.run_action_on_leader( 'ceph-mon', 'create-pool', From ea345c6c5587cadcc6fec4d80cec5ffaa397d753 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 15 Apr 2020 22:15:29 +0200 Subject: [PATCH 402/898] First NovaCloudController test cases Not yet covering the same amount as nova-cloud-controller's Amulet tests. https://launchpad.net/bugs/1828424 --- zaza/openstack/charm_tests/nova/tests.py | 91 +++++++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 0f27e9f..7a8ff82 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -20,9 +20,10 @@ import logging import unittest import zaza.model -import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.guest +import zaza.openstack.utilities.openstack as openstack_utils class BaseGuestCreateTest(unittest.TestCase): @@ -156,6 +157,94 @@ class NovaCompute(test_utils.OpenStackBaseTest): self.assertFalse(int(run['Code']) == 0) +class NovaCloudController(test_utils.OpenStackBaseTest): + """Run nova-cloud-controller specific tests.""" + + XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') + XENIAL_NEWTON = openstack_utils.get_os_release('xenial_newton') + XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') + BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens') + + @classmethod + def setUpClass(cls): + """Run class setup for running nova-cloud-controller tests.""" + super(NovaCloudController, cls).setUpClass() + cls.current_release = openstack_utils.get_os_release() + + @property + def services(self): + """Return a list of services for the selected OpenStack release.""" + services = ['nova-scheduler', 'nova-conductor'] + if self.current_release <= self.BIONIC_QUEENS: + services.append('nova-api-os-compute') + if self.current_release <= self.XENIAL_MITAKA: + services.append('nova-cert') + if self.current_release >= self.XENIAL_OCATA: + services.append('apache2') + return services + + def test_104_compute_api_functionality(self): + """Verify basic compute API functionality.""" + logging.info('Instantiating nova client...') + keystone_session = openstack_utils.get_overcloud_keystone_session() + nova = openstack_utils.get_nova_session_client(keystone_session) + + logging.info('Checking api functionality...') + + actual_service_names = [service['binary'] for service in + nova.services.list()] + for expected_service_name in ('nova-scheduler', 'nova-conductor', + 'nova-compute'): + assert(expected_service_name in actual_service_names) + + # Thanks to setup.create_flavors we should have a few flavors already: + assert(len(nova.flavors.list()) > 0) + + # Just checking it's not raising and returning an iterable: + assert(len(nova.servers.list()) >= 0) + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change. + + Change debug mode and assert that change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + current_value = zaza.model.get_application_config( + 'nova-cloud-controller')['debug']['value'] + new_value = str(not bool(current_value)).title() + current_value = str(current_value).title() + + set_default = {'debug': current_value} + set_alternate = {'debug': new_value} + default_entry = {'DEFAULT': {'debug': [current_value]}} + alternate_entry = {'DEFAULT': {'debug': [new_value]}} + + # Config file affected by juju set config change + conf_file = '/etc/nova/nova.conf' + + # Make config change, check for service restarts + logging.info( + 'Setting verbose on nova-cloud-controller {}'.format( + set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + self.services) + + def test_901_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + with self.pause_resume(self.services): + logging.info("Testing pause resume") + + class SecurityTests(test_utils.OpenStackBaseTest): """nova-compute and nova-cloud-controller security tests.""" From 0f8396b213556dd1da5061849c0a3a2d6fd50f82 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 17 Apr 2020 10:08:28 +0200 Subject: [PATCH 403/898] Fix test_104_compute_api_functionality --- zaza/openstack/charm_tests/nova/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 7a8ff82..0ac4041 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -191,7 +191,7 @@ class NovaCloudController(test_utils.OpenStackBaseTest): logging.info('Checking api functionality...') - actual_service_names = [service['binary'] for service in + actual_service_names = [service.to_dict()['binary'] for service in nova.services.list()] for expected_service_name in ('nova-scheduler', 'nova-conductor', 'nova-compute'): From 7c1f01e3681e897e2a55b093725309ef2267472b Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 17 Apr 2020 12:04:23 +0200 Subject: [PATCH 404/898] Add test_220_nova_metadata_propagate --- zaza/openstack/charm_tests/nova/tests.py | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 0ac4041..4f94316 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -203,6 +203,42 @@ class NovaCloudController(test_utils.OpenStackBaseTest): # Just checking it's not raising and returning an iterable: assert(len(nova.servers.list()) >= 0) + def test_220_nova_metadata_propagate(self): + """Verify that the vendor-data settings are propagated. + + Change vendor-data-url and assert that change propagates to the correct + file and that services are restarted as a result + """ + if self.current_release < self.XENIAL_NEWTON: + logging.info("Feature didn't exist before Newton. Nothing to test") + return + + # Expected default and alternate values + current_value = zaza.model.get_application_config( + 'nova-cloud-controller')['vendor-data-url']['value'] + new_value = 'http://some-other.url/vdata' + + set_default = {'vendor-data-url': current_value} + set_alternate = {'vendor-data-url': new_value} + default_entry = {'api': { + 'vendordata_dynamic_targets': [current_value]}} + alternate_entry = {'api': {'vendordata_dynamic_targets': [new_value]}} + + # Config file affected by juju set config change + conf_file = '/etc/nova/nova.conf' + + # Make config change, check for service restarts + logging.info( + 'Setting config on nova-cloud-controller to {}'.format( + set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + self.services) + def test_900_restart_on_config_change(self): """Checking restart happens on config change. From 071754e7cf56917ac27ea53e7daa613d2ba3693a Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 17 Apr 2020 12:04:33 +0200 Subject: [PATCH 405/898] Add test_302_api_rate_limiting_is_enabled --- zaza/openstack/charm_tests/nova/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 4f94316..800fce0 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -239,6 +239,14 @@ class NovaCloudController(test_utils.OpenStackBaseTest): alternate_entry, self.services) + def test_302_api_rate_limiting_is_enabled(self): + """Check that API rate limiting is enabled.""" + logging.info('Checking api-paste config file data...') + zaza.model.block_until_oslo_config_entries_match( + 'nova-cloud-controller', '/etc/nova/api-paste.ini', { + 'filter:legacy_ratelimit': { + 'limits': ["( POST, '*', .*, 9999, MINUTE );"]}}) + def test_900_restart_on_config_change(self): """Checking restart happens on config change. From f49259e426320780c51d0c6206c779d3a51a1e5c Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 17 Apr 2020 13:36:01 +0200 Subject: [PATCH 406/898] Set some apt config to accept package defaults Because the OpenStack charms re-render their config whenever things are incorrect, we can accept any new package defaults as a part of the upgrade and resolve incorrect config after the upgrade --- ..._zaza_utilities_parallel_series_upgrade.py | 33 +++++++++++++++++-- .../utilities/parallel_series_upgrade.py | 30 +++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index aefa00b..457ce1b 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -428,6 +428,8 @@ class TestParallelSeriesUpgrade(AioTestCase): ]) mock_post_application_upgrade_functions.assert_called_once_with(None) + @mock.patch.object(upgrade_utils, 'add_confdef_file') + @mock.patch.object(upgrade_utils, 'remove_confdef_file') @mock.patch.object( upgrade_utils.series_upgrade_utils, 'async_complete_series_upgrade') @mock.patch.object(upgrade_utils, 'reboot') @@ -438,7 +440,9 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_async_dist_upgrade, mock_async_do_release_upgrade, mock_reboot, - mock_async_complete_series_upgrade + mock_async_complete_series_upgrade, + mock_remove_confdef_file, + mock_add_confdef_file ): await upgrade_utils.series_upgrade_machine( '1', @@ -450,7 +454,11 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_async_do_release_upgrade.assert_called_once_with('1') mock_reboot.assert_called_once_with('1') mock_async_complete_series_upgrade.assert_called_once_with('1') + mock_remove_confdef_file.assert_called_once_with('1') + mock_add_confdef_file.assert_called_once_with('1') + @mock.patch.object(upgrade_utils, 'add_confdef_file') + @mock.patch.object(upgrade_utils, 'remove_confdef_file') @mock.patch.object(upgrade_utils.os_utils, 'async_set_origin') @mock.patch.object( upgrade_utils.series_upgrade_utils, 'async_complete_series_upgrade') @@ -463,7 +471,9 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_async_do_release_upgrade, mock_reboot, mock_async_complete_series_upgrade, - mock_async_set_origin + mock_async_set_origin, + mock_remove_confdef_file, + mock_add_confdef_file ): await upgrade_utils.series_upgrade_machine( '1', @@ -479,6 +489,8 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_async_complete_series_upgrade.assert_called_once_with('1') mock_async_set_origin.assert_called_once_with( 'app', 'openstack-origin') + mock_remove_confdef_file.assert_called_once_with('1') + mock_add_confdef_file.assert_called_once_with('1') async def test_maybe_pause_things_primary(self): await upgrade_utils.maybe_pause_things( @@ -523,6 +535,23 @@ class TestParallelSeriesUpgrade(AioTestCase): pause_non_leader_primary=False) self.async_run_action.assert_not_called() + async def test_add_confdef_file(self): + await upgrade_utils.add_confdef_file('1') + cmd = ( + """echo """ + """'DPkg::options { "--force-confdef"; "--force-confnew"; }' | """ + """sudo tee /etc/apt/apt.conf.d/local""" + ) + self.async_run_on_machine.assert_called_once_with( + '1', cmd + ) + + async def test_remove_confdef_file(self): + await upgrade_utils.remove_confdef_file('1') + self.async_run_on_machine.assert_called_once_with( + '1', 'sudo rm /etc/apt/apt.conf.d/local' + ) + async def test_async_do_release_upgrade(self): await upgrade_utils.async_do_release_upgrade('1') do_release_upgrade_cmd = ( diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 29e276d..9c49d15 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -381,8 +381,10 @@ async def series_upgrade_machine( logging.info( "About to series-upgrade ({})".format(machine)) await run_pre_upgrade_functions(machine, pre_upgrade_functions) + await add_confdef_file(machine) await async_dist_upgrade(machine) await async_do_release_upgrade(machine) + await remove_confdef_file(machine) await reboot(machine) if origin: await os_utils.async_set_origin(application, origin) @@ -390,6 +392,34 @@ async def series_upgrade_machine( await run_post_upgrade_functions(post_upgrade_functions) +async def add_confdef_file(machine): + """Add the file /etc/apt/apt-conf.d/local setup to accept defaults. + + :param machine: The machine to manage + :type machine: str + :returns: None + :rtype: None + """ + create_file = ( + """echo 'DPkg::options { "--force-confdef"; "--force-confnew"; }' | """ + """sudo tee /etc/apt/apt.conf.d/local""" + ) + await model.async_run_on_machine(machine, create_file) + + +async def remove_confdef_file(machine): + """Remove the file /etc/apt/apt-conf.d/local setup to accept defaults. + + :param machine: The machine to manage + :type machine: str + :returns: None + :rtype: None + """ + await model.async_run_on_machine( + machine, + "sudo rm /etc/apt/apt.conf.d/local") + + async def run_pre_upgrade_functions(machine, pre_upgrade_functions): """Execute list supplied functions. From 82928825fd2db43a0694bf07f34fac828335a630 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 17 Apr 2020 14:20:42 +0200 Subject: [PATCH 407/898] Move NGW QoS test back to the NGW test class Commit 55fc718f981f22cb7a555de993960841d8d5196a moved a QoS data plane test from Neutron Gateway tests to the Neutron API tests. This was probably done because the Neutron API charm gate was doing end to end tests for all deployment topologies. On the back of splitting out the data plane tests from Neutron API gate to the respective data plane charms we need to move this test back where it belongs. Apparantly the Neutron Open vSwitch test class have its own variant of this test already that will be executed by the neutron-openvswitch charm gate. --- zaza/openstack/charm_tests/neutron/tests.py | 68 +++++++++------------ 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index cfbb32b..6d3dfc7 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -115,6 +115,35 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): _APP_NAME = 'neutron-gateway' + def test_401_enable_qos(self): + """Check qos settings set via neutron-api charm.""" + if (self.current_os_release >= + openstack_utils.get_os_release('trusty_mitaka')): + logging.info('running qos check') + + with self.config_change( + {'enable-qos': 'False'}, + {'enable-qos': 'True'}, + application_name="neutron-api"): + + self._validate_openvswitch_agent_qos() + + @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60)) + def _validate_openvswitch_agent_qos(self): + """Validate that the qos extension is enabled in the ovs agent.""" + # obtain the dhcp agent to identify the neutron-gateway host + dhcp_agent = self.neutron_client.list_agents( + binary='neutron-dhcp-agent')['agents'][0] + neutron_gw_host = dhcp_agent['host'] + logging.debug('neutron gw host: {}'.format(neutron_gw_host)) + + # check extensions on the ovs agent to validate qos + ovs_agent = self.neutron_client.list_agents( + binary='neutron-openvswitch-agent', + host=neutron_gw_host)['agents'][0] + + self.assertIn('qos', ovs_agent['configurations']['extensions']) + @unittest.expectedFailure def test_800_ovs_bridges_are_managed_by_us(self): """Checking OVS bridges' external-id. @@ -291,29 +320,6 @@ class NeutronCreateNetworkTest(test_utils.OpenStackBaseTest): class NeutronApiTest(NeutronCreateNetworkTest): """Test basic Neutron API Charm functionality.""" - def test_401_enable_qos(self): - """Check qos settings set via neutron-api charm.""" - if (self.current_os_release >= - openstack_utils.get_os_release('trusty_mitaka')): - logging.info('running qos check') - - dhcp_agents = self.neutron_client.list_agents( - binary='neutron-dhcp-agent')['agents'] - if not dhcp_agents: - ovn_agents = self.neutron_client.list_agents( - binary='ovn-controller')['agents'] - if ovn_agents: - raise unittest.SkipTest( - "QoS tests are currently not supported on OVN " - "deployments") - - with self.config_change( - {'enable-qos': 'False'}, - {'enable-qos': 'True'}, - application_name="neutron-api"): - - self._validate_openvswitch_agent_qos() - def test_900_restart_on_config_change(self): """Checking restart happens on config change. @@ -367,22 +373,6 @@ class NeutronApiTest(NeutronCreateNetworkTest): pgrep_full=pgrep_full): logging.info("Testing pause resume") - @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60)) - def _validate_openvswitch_agent_qos(self): - """Validate that the qos extension is enabled in the ovs agent.""" - # obtain the dhcp agent to identify the neutron-gateway host - dhcp_agent = self.neutron_client.list_agents( - binary='neutron-dhcp-agent')['agents'][0] - neutron_gw_host = dhcp_agent['host'] - logging.debug('neutron gw host: {}'.format(neutron_gw_host)) - - # check extensions on the ovs agent to validate qos - ovs_agent = self.neutron_client.list_agents( - binary='neutron-openvswitch-agent', - host=neutron_gw_host)['agents'][0] - - self.assertIn('qos', ovs_agent['configurations']['extensions']) - class SecurityTest(test_utils.OpenStackBaseTest): """Neutron Security Tests.""" From e93fa0b57145b9bf459e5292562e67213569363f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 17 Apr 2020 15:26:41 +0200 Subject: [PATCH 408/898] neutron/policyd: Use ``create_network`` for test At present the neutron policyd test assumes a network visible to the demo user has already been created at the time of the test being executed. This may not always be the case, so let's manage the resource we use for the test. --- zaza/openstack/charm_tests/policyd/tests.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 360736d..2845781 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -514,7 +514,7 @@ class KeystoneTests(BasePolicydSpecialization): class NeutronApiTests(BasePolicydSpecialization): """Test the policyd override using the neutron client.""" - _rule = {'rule.yaml': "{'get_network': '!'}"} + _rule = {'rule.yaml': "{'create_network': '!'}"} @classmethod def setUpClass(cls, application_name=None): @@ -541,13 +541,16 @@ class NeutronApiTests(BasePolicydSpecialization): neutron_client = openstack_utils.get_neutron_session_client( self.get_keystone_session_demo_user(ip)) try: - # If we are allowed to list networks, this will return something. - # if the policyd override is present, then no error is generated, - # but no networks are returned. - networks = neutron_client.list_networks() - logging.debug("networks: {}".format(networks)) - if len(networks['networks']) == 0: - raise PolicydOperationFailedException() + # If we are allowed to create networks, this will return something. + # if the policyd override is present, an exception will be raised + created_network = neutron_client.create_network( + { + 'network': { + 'name': 'zaza-policyd-test', + }, + }) + logging.debug("networks: {}".format(created_network)) + neutron_client.delete_network(created_network['network']['id']) except Exception: raise PolicydOperationFailedException() From 28e1ce2d3f5d71a2a26cc8e1cad465d0b08e1b18 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 17 Apr 2020 16:42:19 +0200 Subject: [PATCH 409/898] Add check to include subordinates for idleness Depends-On: https://github.com/openstack-charmers/zaza/pull/346 --- zaza/openstack/utilities/parallel_series_upgrade.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index 9c49d15..c366846 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -207,7 +207,8 @@ async def parallel_series_upgrade( await series_upgrade_utils.async_set_series( application, to_series=to_series) app_idle = [ - model.async_wait_for_unit_idle(unit) for unit in status["units"] + model.async_wait_for_unit_idle(unit, include_subordinates=True) + for unit in status["units"] ] await asyncio.gather(*app_idle) prepare_group = [ @@ -308,7 +309,7 @@ async def serial_series_upgrade( await series_upgrade_utils.async_set_series( application, to_series=to_series) if not follower_first and leader_machine not in completed_machines: - await model.async_wait_for_unit_idle(leader) + await model.async_wait_for_unit_idle(leader, include_subordinates=True) await prepare_series_upgrade(leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) @@ -325,7 +326,8 @@ async def serial_series_upgrade( machine = unit['machine'] if machine in completed_machines: continue - await model.async_wait_for_unit_idle(unit_name) + await model.async_wait_for_unit_idle( + unit_name, include_subordinates=True) await prepare_series_upgrade(machine, to_series=to_series) logging.info("About to upgrade follower of {}: {}" .format(application, machine)) @@ -338,7 +340,7 @@ async def serial_series_upgrade( completed_machines.append(machine) if follower_first and leader_machine not in completed_machines: - await model.async_wait_for_unit_idle(leader) + await model.async_wait_for_unit_idle(leader, include_subordinates=True) await prepare_series_upgrade(leader_machine, to_series=to_series) logging.info("About to upgrade leader of {}: {}" .format(application, leader_machine)) From 975248f571e3b3e69d2ef9574fc0126c54d5423a Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 17 Apr 2020 13:13:39 +0200 Subject: [PATCH 410/898] Add test_106_compute_catalog_endpoints --- zaza/openstack/charm_tests/nova/tests.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 800fce0..9fac67b 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -203,6 +203,18 @@ class NovaCloudController(test_utils.OpenStackBaseTest): # Just checking it's not raising and returning an iterable: assert(len(nova.servers.list()) >= 0) + def test_106_compute_catalog_endpoints(self): + """Verify that the compute endpoints are present in the catalog.""" + overcloud_auth = openstack_utils.get_overcloud_auth() + keystone_client = openstack_utils.get_keystone_client( + overcloud_auth) + actual_endpoints = keystone_client.service_catalog.get_endpoints() + logging.info('Checking compute endpoints...') + actual_compute_interfaces = [endpoint['interface'] for endpoint in + actual_endpoints['compute']] + for expected_interface in ('internal', 'admin', 'public'): + assert(expected_interface in actual_compute_interfaces) + def test_220_nova_metadata_propagate(self): """Verify that the vendor-data settings are propagated. From 845f728b4d676c474af34fe20fbe3975cd865d1e Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sun, 19 Apr 2020 16:03:39 +0200 Subject: [PATCH 411/898] n-ovs: Remove redundant unit references --- zaza/openstack/charm_tests/neutron/tests.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index cfbb32b..0cf148e 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -442,10 +442,6 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): """Run class setup for running Neutron Openvswitch tests.""" super(NeutronOpenvSwitchTest, cls).setUpClass(cls) - cls.compute_unit = zaza.model.get_units('nova-compute')[0] - cls.neutron_api_unit = zaza.model.get_units('neutron-api')[0] - cls.n_ovs_unit = zaza.model.get_units('neutron-openvswitch')[0] - # set up client cls.neutron_client = ( openstack_utils.get_neutron_session_client(cls.keystone_session)) From 8b8155347ec09b17b1623d482a75ddac9ec7c773 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 17 Apr 2020 18:04:40 +0200 Subject: [PATCH 412/898] Fix typos --- zaza/openstack/charm_tests/test_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 82fed63..137f096 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -160,7 +160,7 @@ class BaseCharmTest(unittest.TestCase): Workaround: libjuju refuses to accept data with types other than strings - through the zuzu.model.set_application_config + through the zaza.model.set_application_config :param config: Config dictionary with any typed values :type config: Dict[str,Any] @@ -317,7 +317,7 @@ class BaseCharmTest(unittest.TestCase): # If this is not an OSLO config file set default_config={} if default_entry: logging.debug( - 'Waiting for updates to propagate to '.format(config_file)) + 'Waiting for updates to propagate to {}'.format(config_file)) model.block_until_oslo_config_entries_match( self.application_name, config_file, From 1594a09625a80cfa5d04c44f39cb0d43bb57cda7 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Sun, 19 Apr 2020 16:08:11 +0200 Subject: [PATCH 413/898] Fix test_220_nova_metadata_propagate --- zaza/openstack/charm_tests/nova/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 9fac67b..1b32f83 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -161,9 +161,9 @@ class NovaCloudController(test_utils.OpenStackBaseTest): """Run nova-cloud-controller specific tests.""" XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') - XENIAL_NEWTON = openstack_utils.get_os_release('xenial_newton') XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens') + BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky') @classmethod def setUpClass(cls): @@ -221,8 +221,8 @@ class NovaCloudController(test_utils.OpenStackBaseTest): Change vendor-data-url and assert that change propagates to the correct file and that services are restarted as a result """ - if self.current_release < self.XENIAL_NEWTON: - logging.info("Feature didn't exist before Newton. Nothing to test") + if self.current_release < self.BIONIC_ROCKY: + logging.info("Feature didn't exist before Rocky. Nothing to test") return # Expected default and alternate values From 6005ca7517c2c7f31c5221b78efcfe6542c68b50 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 17 Apr 2020 15:15:12 +0200 Subject: [PATCH 414/898] Add test_310_pci_alias_config --- zaza/openstack/charm_tests/nova/tests.py | 66 ++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 1b32f83..6c5c3b1 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -16,6 +16,7 @@ """Encapsulate nova testing.""" +import json import logging import unittest @@ -259,6 +260,71 @@ class NovaCloudController(test_utils.OpenStackBaseTest): 'filter:legacy_ratelimit': { 'limits': ["( POST, '*', .*, 9999, MINUTE );"]}}) + def test_310_pci_alias_config(self): + """Verify that the pci alias data is rendered properly. + + Change pci-alias and assert that change propagates to the correct + file and that services are restarted as a result + """ + logging.info('Checking pci aliases in nova config...') + + # Expected default and alternate values + current_value = zaza.model.get_application_config( + 'nova-cloud-controller')['pci-alias'] + try: + current_value = current_value['value'] + except KeyError: + current_value = None + new_value = '[{}, {}]'.format( + json.dumps({ + 'name': 'IntelNIC', + 'capability_type': 'pci', + 'product_id': '1111', + 'vendor_id': '8086', + 'device_type': 'type-PF' + }, sort_keys=True), + json.dumps({ + 'name': ' Cirrus Logic ', + 'capability_type': 'pci', + 'product_id': '0ff2', + 'vendor_id': '10de', + 'device_type': 'type-PCI' + }, sort_keys=True)) + + set_default = {'pci-alias': current_value} + set_alternate = {'pci-alias': new_value} + + expected_conf_section = 'DEFAULT' + expected_conf_key = 'pci_alias' + if self.current_release >= self.XENIAL_OCATA: + expected_conf_section = 'pci' + expected_conf_key = 'alias' + + default_entry = {expected_conf_section: {}} + alternate_entry = {expected_conf_section: { + expected_conf_key: [ + ('{"capability_type": "pci", "device_type": "type-PF", ' + '"name": "IntelNIC", "product_id": "1111", ' + '"vendor_id": "8086"}'), + ('{"capability_type": "pci", "device_type": "type-PCI", ' + '"name": " Cirrus Logic ", "product_id": "0ff2", ' + '"vendor_id": "10de"}')]}} + + # Config file affected by juju set config change + conf_file = '/etc/nova/nova.conf' + + # Make config change, check for service restarts + logging.info( + 'Setting config on nova-cloud-controller to {}'.format( + set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + self.services) + def test_900_restart_on_config_change(self): """Checking restart happens on config change. From 13d9702cdb594c74cb6f9c8eff1521fe7064b049 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 17 Apr 2020 15:28:28 +0200 Subject: [PATCH 415/898] Add test_902_quota_settings --- zaza/openstack/charm_tests/nova/tests.py | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 6c5c3b1..f366629 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -366,6 +366,49 @@ class NovaCloudController(test_utils.OpenStackBaseTest): with self.pause_resume(self.services): logging.info("Testing pause resume") + def test_902_quota_settings(self): + """Verify that the quota settings are propagated. + + Change quota-instances and assert that change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + current_value = zaza.model.get_application_config( + 'nova-cloud-controller')['quota-instances'] + try: + current_value = current_value['value'] + except KeyError: + current_value = 0 + new_value = '20' + + set_default = {'quota-instances': current_value} + set_alternate = {'quota-instances': new_value} + + expected_conf_section = 'DEFAULT' + expected_conf_key = 'quota_instances' + if self.current_release >= self.XENIAL_OCATA: + expected_conf_section = 'quota' + expected_conf_key = 'instances' + + default_entry = {expected_conf_section: {}} + alternate_entry = {expected_conf_section: { + expected_conf_key: [new_value]}} + + # Config file affected by juju set config change + conf_file = '/etc/nova/nova.conf' + + # Make config change, check for service restarts + logging.info( + 'Setting config on nova-cloud-controller to {}'.format( + set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + self.services) + class SecurityTests(test_utils.OpenStackBaseTest): """nova-compute and nova-cloud-controller security tests.""" From e3fb0fde92e47ac0c6b34639f805d66c1b095ff5 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sun, 19 Apr 2020 16:08:35 +0200 Subject: [PATCH 416/898] n-ovs: Await start of execution before awaiting idle At present we may start interrogating the model for a result of a change made by the functional test before all units of the application have started executing. --- zaza/openstack/charm_tests/neutron/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 0cf148e..f4f3895 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -458,6 +458,7 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): self.application_name, {'enable-sriov': 'True'}) + zaza.model.wait_for_agent_status() zaza.model.wait_for_application_states() self._check_settings_in_config( @@ -481,6 +482,7 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): {'enable-sriov': 'False'}) logging.info('Waiting for config-changes to complete...') + zaza.model.wait_for_agent_status() zaza.model.wait_for_application_states() logging.debug('OK') @@ -543,6 +545,7 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): 'neutron-openvswitch', {'disable-security-groups': 'True'}) + zaza.model.wait_for_agent_status() zaza.model.wait_for_application_states() expected = { @@ -565,6 +568,7 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): 'neutron-api', {'neutron-security-groups': 'False'}) + zaza.model.wait_for_agent_status() zaza.model.wait_for_application_states() def test_401_restart_on_config_change(self): From 9477fd4c525c648a0db6f7e76e5167adcac45f00 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 20 Apr 2020 00:53:46 +0200 Subject: [PATCH 417/898] Fix test_106_compute_catalog_endpoints prior to Rocky --- zaza/openstack/charm_tests/nova/tests.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index f366629..922ba0c 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -210,11 +210,18 @@ class NovaCloudController(test_utils.OpenStackBaseTest): keystone_client = openstack_utils.get_keystone_client( overcloud_auth) actual_endpoints = keystone_client.service_catalog.get_endpoints() + logging.info('Checking compute endpoints...') - actual_compute_interfaces = [endpoint['interface'] for endpoint in - actual_endpoints['compute']] - for expected_interface in ('internal', 'admin', 'public'): - assert(expected_interface in actual_compute_interfaces) + + if self.current_release < self.BIONIC_ROCKY: + actual_compute_endpoints = actual_endpoints['compute'][0] + for expected_url in ('internalURL', 'adminURL', 'publicURL'): + assert(expected_url in actual_compute_endpoints) + else: + actual_compute_interfaces = [endpoint['interface'] for endpoint in + actual_endpoints['compute']] + for expected_interface in ('internal', 'admin', 'public'): + assert(expected_interface in actual_compute_interfaces) def test_220_nova_metadata_propagate(self): """Verify that the vendor-data settings are propagated. From 169dff2d8ec7857e11e45b42687f7538064d37e2 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 20 Apr 2020 08:31:58 +0200 Subject: [PATCH 418/898] n-ovs: Do not treat ``bool`` config as ``str`` When applying configuration to a model the helpers will compare settings already set to what is being attempted set to be able to accurately predict model behaviour. When passing ``bool`` values as ``str`` this does not work and it may lead to unwanted behaviour depending on the model state at the time the test runs. --- zaza/openstack/charm_tests/neutron/tests.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index f4f3895..5e318ad 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -497,7 +497,7 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): expected = { section: { - config_file_key: [vpair[1]], + config_file_key: [str(vpair[1])], }, } @@ -517,7 +517,7 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): 'neutron-api', 'l2-population', 'l2_population', - ['False', 'True'], + [False, True], 'agent', '/etc/neutron/plugins/ml2/openvswitch_agent.ini') @@ -540,10 +540,10 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): zaza.model.set_application_config( 'neutron-api', - {'neutron-security-groups': 'True'}) + {'neutron-security-groups': True}) zaza.model.set_application_config( 'neutron-openvswitch', - {'disable-security-groups': 'True'}) + {'disable-security-groups': True}) zaza.model.wait_for_agent_status() zaza.model.wait_for_application_states() @@ -563,10 +563,10 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): logging.info('Restoring to default configuration...') zaza.model.set_application_config( 'neutron-openvswitch', - {'disable-security-groups': 'False'}) + {'disable-security-groups': False}) zaza.model.set_application_config( 'neutron-api', - {'neutron-security-groups': 'False'}) + {'neutron-security-groups': False}) zaza.model.wait_for_agent_status() zaza.model.wait_for_application_states() @@ -579,8 +579,8 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): """ self.restart_on_changed( '/etc/neutron/neutron.conf', - {'debug': 'false'}, - {'debug': 'true'}, + {'debug': False}, + {'debug': True}, {'DEFAULT': {'debug': ['False']}}, {'DEFAULT': {'debug': ['True']}}, ['neutron-openvswitch-agent'], @@ -592,8 +592,8 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): logging.debug('Skipping test') return - set_default = {'enable-qos': 'false'} - set_alternate = {'enable-qos': 'true'} + set_default = {'enable-qos': False} + set_alternate = {'enable-qos': True} app_name = 'neutron-api' conf_file = '/etc/neutron/plugins/ml2/openvswitch_agent.ini' From 92038654085eb39849c33d75b12b943e8465a794 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 20 Apr 2020 09:29:15 +0200 Subject: [PATCH 419/898] n-ovs: Do not use model.set_application_config directly There are two problems with doing so as part of individual functional tests: 1) If the application already have the value set the test will time out waiting for a change that will never be made. 2) python-libjuju ``set_config`` call requires values to be ``str`` regardless of their actual type. Pairing this fact with the requirement to use the actual type when comparing values before attempting to set them makes this very confusing and error prone. juju/python-libjuju#388 openstack-charmers/zaza#348 Use the ``config_change`` helper instead. --- zaza/openstack/charm_tests/neutron/tests.py | 43 ++++++--------------- 1 file changed, 11 insertions(+), 32 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 5e318ad..c7f1648 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -538,38 +538,17 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): else: conf_file = "/etc/neutron/plugins/ml2/ml2_conf.ini" - zaza.model.set_application_config( - 'neutron-api', - {'neutron-security-groups': True}) - zaza.model.set_application_config( - 'neutron-openvswitch', - {'disable-security-groups': True}) - - zaza.model.wait_for_agent_status() - zaza.model.wait_for_application_states() - - expected = { - 'securitygroup': { - 'enable_security_group': ['False'], - }, - } - - zaza.model.block_until_oslo_config_entries_match( - self.application_name, - conf_file, - expected, - ) - - logging.info('Restoring to default configuration...') - zaza.model.set_application_config( - 'neutron-openvswitch', - {'disable-security-groups': False}) - zaza.model.set_application_config( - 'neutron-api', - {'neutron-security-groups': False}) - - zaza.model.wait_for_agent_status() - zaza.model.wait_for_application_states() + with self.config_change( + {'neutron-security-groups': False}, + {'neutron-security-groups': True}, + application_name='neutron-api'): + with self.config_change( + {'disable-security-groups': False}, + {'disable-security-groups': True}): + zaza.model.block_until_oslo_config_entries_match( + self.application_name, + conf_file, + {'securitygroup': {'enable_security_group': ['False']}}) def test_401_restart_on_config_change(self): """Verify that the specified services are restarted. From 3d11b9c018bf200fb22a8edc5e8e2fa70169f25b Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 20 Apr 2020 10:51:38 +0200 Subject: [PATCH 420/898] Fix test_106_compute_catalog_endpoints on Queens --- zaza/openstack/charm_tests/nova/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 922ba0c..03faaae 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -163,6 +163,7 @@ class NovaCloudController(test_utils.OpenStackBaseTest): XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') + XENIAL_QUEENS = openstack_utils.get_os_release('xenial_queens') BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens') BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky') @@ -213,7 +214,7 @@ class NovaCloudController(test_utils.OpenStackBaseTest): logging.info('Checking compute endpoints...') - if self.current_release < self.BIONIC_ROCKY: + if self.current_release < self.XENIAL_QUEENS: actual_compute_endpoints = actual_endpoints['compute'][0] for expected_url in ('internalURL', 'adminURL', 'publicURL'): assert(expected_url in actual_compute_endpoints) From 9cec2efabe30fb0709bc098c48ec10bcb85cc9d4 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 20 Apr 2020 11:47:12 +0200 Subject: [PATCH 421/898] Actually pass to and from series into upgrade --- zaza/openstack/utilities/parallel_series_upgrade.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index c366846..ba8b205 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -120,6 +120,7 @@ def upgrade_ubuntu_lite(from_series='xenial', to_series='bionic'): parallel_series_upgrade( 'ubuntu-lite', pause_non_leader_primary=False, pause_non_leader_subordinate=False, + from_series=from_series, to_series=to_series, completed_machines=completed_machines, origin=None) ) From 673d1a9d7f90e60f0053dafb06f7d900541a4d7e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 20 Apr 2020 11:18:23 +0200 Subject: [PATCH 422/898] Barbican tests --- requirements.txt | 1 + setup.py | 1 + .../charm_tests/barbican/__init__.py | 15 +++ zaza/openstack/charm_tests/barbican/tests.py | 105 ++++++++++++++++++ 4 files changed, 122 insertions(+) create mode 100644 zaza/openstack/charm_tests/barbican/__init__.py create mode 100644 zaza/openstack/charm_tests/barbican/tests.py diff --git a/requirements.txt b/requirements.txt index ed9c353..fa0e460 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,6 +23,7 @@ python-openstackclient>=3.14.0 aodhclient gnocchiclient>=7.0.5,<8.0.0 pika>=1.1.0,<2.0.0 +python-barbicanclient python-designateclient python-ceilometerclient python-cinderclient diff --git a/setup.py b/setup.py index 0184f72..6d08832 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ install_require = [ 'aodhclient<1.4.0', 'gnocchiclient>=7.0.5,<8.0.0', 'pika>=1.1.0,<2.0.0', + 'python-barbicanclient>=4.0.1,<5.0.0', 'python-designateclient>=1.5,<3.0.0', 'python-heatclient<2.0.0', 'python-glanceclient<3.0.0', diff --git a/zaza/openstack/charm_tests/barbican/__init__.py b/zaza/openstack/charm_tests/barbican/__init__.py new file mode 100644 index 0000000..eecbd79 --- /dev/null +++ b/zaza/openstack/charm_tests/barbican/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing barbican.""" diff --git a/zaza/openstack/charm_tests/barbican/tests.py b/zaza/openstack/charm_tests/barbican/tests.py new file mode 100644 index 0000000..e07042a --- /dev/null +++ b/zaza/openstack/charm_tests/barbican/tests.py @@ -0,0 +1,105 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate barbican testing.""" + +import logging + +import barbicanclient.client as barbican_client +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +class BarbicanTest(test_utils.OpenStackBaseTest): + """Run nova-compute specific tests.""" + + _SERVICES = ['apache2', 'barbican-worker'] + + def test_110_catalog_endpoints(self): + """Verify that the endpoints are present in the catalog.""" + overcloud_auth = openstack_utils.get_overcloud_auth() + keystone_client = openstack_utils.get_keystone_client( + overcloud_auth) + actual_endpoints = keystone_client.service_catalog.get_endpoints() + for service_type in ('key-manager', 'identity'): + actual_interfaces = [endpoint['interface'] for endpoint in + actual_endpoints[service_type]] + for expected_interface in ('internal', 'admin', 'public'): + assert(expected_interface in actual_interfaces) + + def test_400_api_connection(self): + """Simple api calls to check service is up and responding.""" + logging.info('Authenticating with the barbican endpoint') + overcloud_auth = openstack_utils.get_overcloud_auth() + keystone_client = openstack_utils.get_keystone_client( + overcloud_auth) + keystone_session = openstack_utils.get_overcloud_keystone_session() + barbican_endpoint = keystone_client.service_catalog.url_for( + service_type='key-manager', interface='publicURL') + barbican = barbican_client.Client(session=keystone_session, + endpoint=barbican_endpoint) + + logging.info('Creating a secret') + my_secret = barbican.secrets.create() + my_secret.name = u'Random plain text password' + my_secret.payload = u'password' + + logging.info('Storing the secret') + my_secret_ref = my_secret.store() + assert(my_secret_ref is not None) + + logging.info('Deleting the secret') + my_secret.delete() + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change. + + Change debug mode and assert that change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + current_value = zaza.model.get_application_config( + self.application_name)['debug']['value'] + new_value = str(not bool(current_value)).title() + current_value = str(current_value).title() + + set_default = {'debug': current_value} + set_alternate = {'debug': new_value} + default_entry = {'DEFAULT': {'debug': [current_value]}} + alternate_entry = {'DEFAULT': {'debug': [new_value]}} + + # Config file affected by juju set config change + conf_file = '/etc/barbican/barbican.conf' + + # Make config change, check for service restarts + logging.info( + 'Changing settings on {} to {}'.format( + self.application_name, set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + self._SERVICES) + + def test_910_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + with self.pause_resume(self._SERVICES): + logging.info("Testing pause resume") From 329d4fb8c568b2dca89c0a79f64b682b7d898fae Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 20 Apr 2020 12:51:27 +0100 Subject: [PATCH 423/898] Fix hacluster tests for focal with mysql8 The hacluster tests assume that all the subordinates are hacluster. However, with mysql8 there is a mysql-router subordinate as well, and it doesn't support the clean-up action. This change ensures that the tests checks that the subordinate IS hacluter prior to applying the action/test. --- zaza/openstack/charm_tests/hacluster/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index fb9d8e2..ebffb84 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -49,6 +49,10 @@ class HaclusterTest(test_utils.OpenStackBaseTest): if primary_status["units"][leader].get("subordinates"): for subordinate in primary_status["units"][leader]["subordinates"]: + # mysql-router is a subordinate from focal onwards + _app = subordinate.split('/')[0] + if _app != 'hacluster': + continue logging.info("Cleaning {}".format(subordinate)) _action = "cleanup" action_id = zaza.model.run_action(subordinate, "cleanup") @@ -84,6 +88,10 @@ class HaclusterTest(test_utils.OpenStackBaseTest): if primary_status["units"][leader].get("subordinates"): for subordinate in primary_status["units"][leader]["subordinates"]: + # mysql-router is a subordinate from focal onwards + _app = subordinate.split('/')[0] + if _app != 'hacluster': + continue logging.info("Pausing {}".format(subordinate)) zaza.model.run_action(subordinate, "pause") zaza.model.block_until_unit_wl_status( From d652eff6d9666f0a73ad66c9ebe46f71b2745fa3 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 21 Apr 2020 07:16:58 +0000 Subject: [PATCH 424/898] Add a helper for testing restart on changed Most api charms test restart on changed by flipping the value of debug. This has led to the same test being copied around. To reduce the boiler plate code this change adds a new test helper which can be used instead. --- zaza/openstack/charm_tests/test_utils.py | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 137f096..78a2023 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -254,6 +254,41 @@ class BaseCharmTest(unittest.TestCase): # TODO: Optimize with a block on a specific application until idle. model.block_until_all_units_idle() + def restart_on_changed_use_debug(self, config_file, services): + """Check restart happens on config change by flipping debug mode. + + Change debug mode and assert that change propagates to the correct + file and that services are restarted as a result + + :param config_file: Config file to check for settings + :type config_file: str + :param services: Services expected to be restarted when config_file is + changed. + :type services: list + """ + # Expected default and alternate values + current_value = model.get_application_config( + self.application_name)['debug']['value'] + new_value = str(not bool(current_value)).title() + current_value = str(current_value).title() + + set_default = {'debug': current_value} + set_alternate = {'debug': new_value} + default_entry = {'DEFAULT': {'debug': [current_value]}} + alternate_entry = {'DEFAULT': {'debug': [new_value]}} + + # Make config change, check for service restarts + logging.info( + 'Changing settings on {} to {}'.format( + self.application_name, set_alternate)) + self.restart_on_changed( + config_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + services) + def restart_on_changed(self, config_file, default_config, alternate_config, default_entry, alternate_entry, services, pgrep_full=False): From 6694caf281baa2ba47d54a1847cf3962cdb7a404 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 21 Apr 2020 07:45:42 +0000 Subject: [PATCH 425/898] Improve docstring and allo section override --- zaza/openstack/charm_tests/test_utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 78a2023..ae431c1 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -254,13 +254,16 @@ class BaseCharmTest(unittest.TestCase): # TODO: Optimize with a block on a specific application until idle. model.block_until_all_units_idle() - def restart_on_changed_use_debug(self, config_file, services): + def restart_on_changed_debug_oslo_config_file(self, config_file, services, + config_section='DEFAULT'): """Check restart happens on config change by flipping debug mode. Change debug mode and assert that change propagates to the correct - file and that services are restarted as a result + file and that services are restarted as a result. config_file must be + an oslo config file and debug option must be set in the + `config_section` section. - :param config_file: Config file to check for settings + :param config_file: OSLO Config file to check for settings :type config_file: str :param services: Services expected to be restarted when config_file is changed. @@ -274,8 +277,8 @@ class BaseCharmTest(unittest.TestCase): set_default = {'debug': current_value} set_alternate = {'debug': new_value} - default_entry = {'DEFAULT': {'debug': [current_value]}} - alternate_entry = {'DEFAULT': {'debug': [new_value]}} + default_entry = {config_section: {'debug': [current_value]}} + alternate_entry = {config_section: {'debug': [new_value]}} # Make config change, check for service restarts logging.info( From 347a39d11e5526f176970ce9e8231452092492a9 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 21 Apr 2020 14:18:27 +0200 Subject: [PATCH 426/898] Use restart_on_changed_debug_oslo_config_file() --- zaza/openstack/charm_tests/barbican/tests.py | 28 ++------------------ 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/zaza/openstack/charm_tests/barbican/tests.py b/zaza/openstack/charm_tests/barbican/tests.py index e07042a..3d54917 100644 --- a/zaza/openstack/charm_tests/barbican/tests.py +++ b/zaza/openstack/charm_tests/barbican/tests.py @@ -17,7 +17,6 @@ import logging import barbicanclient.client as barbican_client -import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -69,31 +68,8 @@ class BarbicanTest(test_utils.OpenStackBaseTest): Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ - # Expected default and alternate values - current_value = zaza.model.get_application_config( - self.application_name)['debug']['value'] - new_value = str(not bool(current_value)).title() - current_value = str(current_value).title() - - set_default = {'debug': current_value} - set_alternate = {'debug': new_value} - default_entry = {'DEFAULT': {'debug': [current_value]}} - alternate_entry = {'DEFAULT': {'debug': [new_value]}} - - # Config file affected by juju set config change - conf_file = '/etc/barbican/barbican.conf' - - # Make config change, check for service restarts - logging.info( - 'Changing settings on {} to {}'.format( - self.application_name, set_alternate)) - self.restart_on_changed( - conf_file, - set_default, - set_alternate, - default_entry, - alternate_entry, - self._SERVICES) + self.restart_on_changed_debug_oslo_config_file( + '/etc/barbican/barbican.conf', self._SERVICES) def test_910_pause_resume(self): """Run pause and resume tests. From 72b420b56acb40622ea79f3be9555ccbbf876f59 Mon Sep 17 00:00:00 2001 From: Dmitrii Shcherbakov Date: Tue, 21 Apr 2020 16:50:02 +0300 Subject: [PATCH 427/898] Use the new svc restart checker where possible restart_on_changed_debug_oslo_config_file was added to avoid code duplication. --- zaza/openstack/charm_tests/aodh/tests.py | 10 +---- zaza/openstack/charm_tests/cinder/tests.py | 10 +---- zaza/openstack/charm_tests/designate/tests.py | 13 ++---- zaza/openstack/charm_tests/glance/tests.py | 26 ++---------- zaza/openstack/charm_tests/nova/tests.py | 41 ++----------------- 5 files changed, 12 insertions(+), 88 deletions(-) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index a8c7ee8..c11e054 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -114,20 +114,12 @@ class AodhTest(test_utils.OpenStackBaseTest): Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ - # Expected default and alternate values - set_default = {'debug': 'False'} - set_alternate = {'debug': 'True'} - # Config file affected by juju set config change conf_file = '/etc/aodh/aodh.conf' # Make config change, check for service restarts - self.restart_on_changed( + self.restart_on_changed_debug_oslo_config_file( conf_file, - set_default, - set_alternate, - {'DEFAULT': {'debug': ['False']}}, - {'DEFAULT': {'debug': ['True']}}, self.services) self.query_aodh_api() diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index cdccce6..d645eed 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -168,21 +168,13 @@ class CinderTests(test_utils.OpenStackBaseTest): Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ - # Expected default and alternate values - set_default = {'debug': 'False'} - set_alternate = {'debug': 'True'} - # Config file affected by juju set config change conf_file = '/etc/cinder/cinder.conf' # Make config change, check for service restarts logging.debug('Setting debug mode...') - self.restart_on_changed( + self.restart_on_changed_debug_oslo_config_file( conf_file, - set_default, - set_alternate, - {'DEFAULT': {'debug': ['False']}}, - {'DEFAULT': {'debug': ['True']}}, self.services) def test_901_pause_resume(self): diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 5066dd4..f8a32db 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -98,22 +98,15 @@ class DesignateTests(BaseDesignateTest): Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ - # Expected default and alternate values - set_default = {'debug': 'False'} - set_alternate = {'debug': 'True'} - # Services which are expected to restart upon config change, # and corresponding config files affected by the change conf_file = '/etc/designate/designate.conf' # Make config change, check for service restarts - self.restart_on_changed( + self.restart_on_changed_debug_oslo_config_file( conf_file, - set_default, - set_alternate, - {'DEFAULT': {'debug': ['False']}}, - {'DEFAULT': {'debug': ['True']}}, - self.designate_svcs) + self.designate_svcs, + ) def test_910_pause_and_resume(self): """Run pause and resume tests. diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index 033ec28..ba1051f 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -69,22 +69,6 @@ class GlanceTest(test_utils.OpenStackBaseTest): def test_900_restart_on_config_change(self): """Checking restart happens on config change.""" - # Expected default and alternate values - current_value = openstack_utils.get_application_config_option( - 'glance', 'debug') - # this is bool, not str - assert type(current_value) == bool - new_value = not current_value - - # convert bool to str - current_value = str(current_value) - new_value = str(new_value) - - set_default = {'debug': current_value} - set_alternate = {'debug': new_value} - default_entry = {'DEFAULT': {'debug': [current_value]}} - alternate_entry = {'DEFAULT': {'debug': [new_value]}} - # Config file affected by juju set config change conf_file = '/etc/glance/glance-api.conf' @@ -96,14 +80,10 @@ class GlanceTest(test_utils.OpenStackBaseTest): services.update({'glance-registry': conf_file}) # Make config change, check for service restarts - logging.info('changing config: {}'.format(set_alternate)) - self.restart_on_changed( + logging.info('changing debug config') + self.restart_on_changed_debug_oslo_config_file( conf_file, - set_default, - set_alternate, - default_entry, - alternate_entry, - services) + self.services) def test_901_pause_resume(self): """Run pause and resume tests. diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 03faaae..eefdab5 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -80,29 +80,13 @@ class NovaCompute(test_utils.OpenStackBaseTest): Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ - # Expected default and alternate values - current_value = zaza.model.get_application_config( - 'nova-compute')['debug']['value'] - new_value = str(not bool(current_value)).title() - current_value = str(current_value).title() - - set_default = {'debug': current_value} - set_alternate = {'debug': new_value} - default_entry = {'DEFAULT': {'debug': [current_value]}} - alternate_entry = {'DEFAULT': {'debug': [new_value]}} - # Config file affected by juju set config change conf_file = '/etc/nova/nova.conf' # Make config change, check for service restarts - logging.info( - 'Setting verbose on nova-compute {}'.format(set_alternate)) - self.restart_on_changed( + logging.info('Changing the debug config on nova-compute') + self.restart_on_changed_debug_oslo_config_file( conf_file, - set_default, - set_alternate, - default_entry, - alternate_entry, ['nova-compute']) def test_920_change_aa_profile(self): @@ -339,30 +323,13 @@ class NovaCloudController(test_utils.OpenStackBaseTest): Change debug mode and assert that change propagates to the correct file and that services are restarted as a result """ - # Expected default and alternate values - current_value = zaza.model.get_application_config( - 'nova-cloud-controller')['debug']['value'] - new_value = str(not bool(current_value)).title() - current_value = str(current_value).title() - - set_default = {'debug': current_value} - set_alternate = {'debug': new_value} - default_entry = {'DEFAULT': {'debug': [current_value]}} - alternate_entry = {'DEFAULT': {'debug': [new_value]}} - # Config file affected by juju set config change conf_file = '/etc/nova/nova.conf' # Make config change, check for service restarts - logging.info( - 'Setting verbose on nova-cloud-controller {}'.format( - set_alternate)) - self.restart_on_changed( + logging.info('Changing debug config on nova-cloud-controller') + self.restart_on_changed_debug_oslo_config_file( conf_file, - set_default, - set_alternate, - default_entry, - alternate_entry, self.services) def test_901_pause_resume(self): From 16e82b6e8665104fee1c75c9b0470b15cc75d8ae Mon Sep 17 00:00:00 2001 From: Dmitrii Shcherbakov Date: Tue, 21 Apr 2020 17:39:49 +0300 Subject: [PATCH 428/898] Fix test_900_restart_on_config_change Recently broken by PR 245: 72b420b56acb40622ea79f3be9555ccbbf876f59 --- zaza/openstack/charm_tests/glance/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index ba1051f..363032d 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -83,7 +83,7 @@ class GlanceTest(test_utils.OpenStackBaseTest): logging.info('changing debug config') self.restart_on_changed_debug_oslo_config_file( conf_file, - self.services) + services) def test_901_pause_resume(self): """Run pause and resume tests. From 5d20e2564e6424d4006dba9299c3faed9674cada Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 22 Apr 2020 18:08:06 +0100 Subject: [PATCH 429/898] Disable the pristine disk check on focal The test won't pass on serverstack due to the snapd taking a copy of the /mnt/vdb info at boot time and thus not being able to umount it is root without also running through all of the procs and checking if they also have it mounted. Thus, this just disables the test on focal and newer. --- zaza/openstack/charm_tests/ceph/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 8b1c947..7b903ce 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -371,6 +371,13 @@ class CephTest(test_utils.OpenStackBaseTest): As the ephemeral device will have data on it we can use it to validate that these checks work as intended. """ + current_release = zaza_openstack.get_os_release() + focal_ussuri = zaza_openstack.get_os_release('focal_ussuri') + if current_release >= focal_ussuri: + # NOTE(ajkavanagh) - focal (on ServerStack) is broken for /dev/vdb + # and so this test can't pass + logging.warn("Skipping pristine disk test for focal and higher") + return logging.info('Checking behaviour when non-pristine disks appear...') logging.info('Configuring ephemeral-unmount...') alternate_conf = { From 14a6f0658665c5f2df443872ca9797fd789d4b2d Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 22 Apr 2020 17:32:33 +0000 Subject: [PATCH 430/898] Fix glance simplestreams sync test races We expect at least three images. Avoid various race conditions by retrying using tenacity. --- .../glance_simplestreams_sync/tests.py | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py index 6ddb1b3..648172a 100644 --- a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py @@ -24,7 +24,7 @@ import zaza.openstack.utilities.openstack as openstack_utils @tenacity.retry( - retry=tenacity.retry_if_result(lambda images: len(images) == 0), + retry=tenacity.retry_if_result(lambda images: len(images) < 3), wait=tenacity.wait_fixed(6), # interval between retries stop=tenacity.stop_after_attempt(100)) # retry times def retry_image_sync(glance_client): @@ -33,6 +33,19 @@ def retry_image_sync(glance_client): return list(glance_client.images.list()) +@tenacity.retry( + retry=tenacity.retry_if_exception_type(json.decoder.JSONDecodeError), + wait=tenacity.wait_fixed(10), reraise=True, + stop=tenacity.stop_after_attempt(10)) +def get_product_streams(url): + """Get product streams json data with retry.""" + # There is a race between the images being available in glance and any + # metadata being written. Use tenacity to avoid this race. + client = requests.session() + json_data = client.get(url).text + return json.loads(json_data) + + class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): """Glance Simple Streams Sync Test.""" @@ -48,7 +61,7 @@ class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): cls.keystone_session) def test_010_wait_for_image_sync(self): - """Wait for images to be synced. Expect at least one.""" + """Wait for images to be synced. Expect at least three.""" self.assertTrue(retry_image_sync(self.glance_client)) def test_050_gss_permissions_regression_check_lp1611987(self): @@ -91,12 +104,21 @@ class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): catalog = self.keystone_client.service_catalog.get_endpoints() ps_interface = catalog["product-streams"][0][key] url = "{}/{}".format(ps_interface, uri) - client = requests.session() - json_data = client.get(url).text - product_streams = json.loads(json_data) - images = product_streams["products"] - for image in expected_images: - self.assertIn(image, images) + # There is a race between the images being available in glance and the + # metadata being written for each image. Use tenacity to avoid this + # race and make the test idempotent. + @tenacity.retry( + retry=tenacity.retry_if_exception_type(AssertionError), + wait=tenacity.wait_fixed(10), reraise=True, + stop=tenacity.stop_after_attempt(10)) + def _check_local_product_streams(url, expected_images): + product_streams = get_product_streams(url) + images = product_streams["products"] + + for image in expected_images: + self.assertIn(image, images) + + _check_local_product_streams(url, expected_images) logging.debug("Local product stream successful") From 9df7157989625beb9691092072422f55a05b0312 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 24 Apr 2020 12:47:46 +0000 Subject: [PATCH 431/898] Masakari testing fixes * Add method for deleteing pacemaker nodes. * Due to LP #1874719 use above to delete node1 * During test cleanup the tests re-enable hosts from a masakari pov, if that host still has outstanding notifications it will fail so retry the enable. * pacemakerd process name has changed on focal so account for that when killing it. --- zaza/openstack/charm_tests/masakari/tests.py | 4 ++ zaza/openstack/configure/hacluster.py | 17 ++++++++ zaza/openstack/configure/masakari.py | 43 ++++++++++++++++---- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/masakari/tests.py b/zaza/openstack/charm_tests/masakari/tests.py index 52d9c01..80390f5 100644 --- a/zaza/openstack/charm_tests/masakari/tests.py +++ b/zaza/openstack/charm_tests/masakari/tests.py @@ -135,6 +135,10 @@ class MasakariTest(test_utils.OpenStackBaseTest): def test_instance_failover(self): """Test masakari managed guest migration.""" + # Workaround for Bug #1874719 + zaza.openstack.configure.hacluster.remove_node( + 'masakari', + 'node1') # Launch guest self.assertTrue( zaza.openstack.configure.hacluster.check_all_nodes_online( diff --git a/zaza/openstack/configure/hacluster.py b/zaza/openstack/configure/hacluster.py index 9e687b8..eec2a73 100644 --- a/zaza/openstack/configure/hacluster.py +++ b/zaza/openstack/configure/hacluster.py @@ -66,6 +66,23 @@ def get_nodes_status(service_name, model_name=None): return status +def remove_node(service_name, node_name, model_name=None): + """Remove given node from pacemaker. + + :param service_name: Name of Juju application to run query against. + :type service_name: str + :param node_name: Name of node to delete. + :type node_name: str + :param model_name: Name of model unit_name resides in. + :type model_name: str + """ + remove_cmd = 'crm_node --force -R {}'.format(node_name) + zaza.model.run_on_leader( + service_name, + remove_cmd, + model_name=model_name) + + def check_all_nodes_online(service_name, model_name=None): """Return whether all the crm nodes are online. diff --git a/zaza/openstack/configure/masakari.py b/zaza/openstack/configure/masakari.py index 9b8e9bf..a529bbe 100644 --- a/zaza/openstack/configure/masakari.py +++ b/zaza/openstack/configure/masakari.py @@ -19,6 +19,8 @@ and recovery. """ import logging +import openstack.exceptions as ostack_except +import tenacity import zaza.model import zaza.openstack.utilities.openstack as openstack_utils @@ -81,6 +83,28 @@ def create_segments(segment_number=1, host_assignment_method=None): masakari_client) +@tenacity.retry( + wait=tenacity.wait_exponential(multiplier=2, max=60), + reraise=True, stop=tenacity.stop_after_attempt(5), + retry=tenacity.retry_if_exception_type(ostack_except.ConflictException)) +def enable_host(masakari_client, host, segment): + """Enable hypervisor within masakari. + + :param masakari_client: Authenticated masakari client + :type masakari_client: openstack.instance_ha.v1._proxy.Proxy + :param host: Uuid of host to enable + :type host: str + :param segment: Uuid of segment host is associated with. + :type segment: str + """ + logging.info("Removing maintenance mode from masakari " + "host {}".format(host)) + masakari_client.update_host( + host, + segment_id=segment, + **{'on_maintenance': False}) + + def enable_hosts(masakari_client=None): """Enable all hypervisors within masakari. @@ -98,12 +122,7 @@ def enable_hosts(masakari_client=None): for segment in masakari_client.segments(): for host in masakari_client.hosts(segment_id=segment.uuid): if host.on_maintenance: - logging.info("Removing maintenance mode from masakari " - "host {}".format(host.uuid)) - masakari_client.update_host( - host.uuid, - segment_id=segment.uuid, - **{'on_maintenance': False}) + enable_host(masakari_client, host.uuid, segment.uuid) def _svc_control(unit_name, action, services, model_name): @@ -179,10 +198,18 @@ def simulate_compute_host_failure(unit_name, model_name): 'stop', ['corosync', 'nova-compute'], model_name) - logging.info('Sending pacemaker_remoted a SIGTERM') + compute_app = unit_name.split('/')[0] + release_pair = openstack_utils.get_current_os_release_pair( + application=compute_app) + if (openstack_utils.get_os_release(release_pair=release_pair) >= + openstack_utils.get_os_release('focal_ussuri')): + pacemaker_proc = '/usr/sbin/pacemaker-remoted' + else: + pacemaker_proc = '/usr/sbin/pacemaker_remoted' + logging.info('Sending {} a SIGTERM'.format(pacemaker_proc)) zaza.model.run_on_unit( unit_name, - 'pkill -9 -f /usr/sbin/pacemaker_remoted', + 'pkill -9 -f {}'.format(pacemaker_proc), model_name=model_name) From a2e38eecd36b456495610247eb8fa91922fb6f1e Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Sat, 25 Apr 2020 08:00:22 +0530 Subject: [PATCH 432/898] Add test cases for nova shelving accounting in quotas Add functional test case to verify new config param quota-count-usage-from-placement in nova-cloud-controller charm. Charm changes - https://review.opendev.org/#/c/715182/ Related-Bug: #1864859 --- zaza/openstack/charm_tests/nova/tests.py | 61 ++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index eefdab5..1b12240 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -150,6 +150,7 @@ class NovaCloudController(test_utils.OpenStackBaseTest): XENIAL_QUEENS = openstack_utils.get_os_release('xenial_queens') BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens') BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky') + BIONIC_TRAIN = openstack_utils.get_os_release('bionic_train') @classmethod def setUpClass(cls): @@ -384,6 +385,66 @@ class NovaCloudController(test_utils.OpenStackBaseTest): alternate_entry, self.services) + def test_903_enable_quota_count_usage_from_placement(self): + """Verify that quota-count-usage-from-placement is propagated. + + Change quota-count-usage-from-placement and assert that nova + configuration file is updated and the services are restarted. + This parameter is not supported for releases Date: Wed, 29 Apr 2020 18:14:35 +0100 Subject: [PATCH 433/898] Add LP bug number and a bit of explanation --- zaza/openstack/charm_tests/ceph/tests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 7b903ce..ff79820 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -375,7 +375,13 @@ class CephTest(test_utils.OpenStackBaseTest): focal_ussuri = zaza_openstack.get_os_release('focal_ussuri') if current_release >= focal_ussuri: # NOTE(ajkavanagh) - focal (on ServerStack) is broken for /dev/vdb - # and so this test can't pass + # and so this test can't pass: LP#1842751 discusses the issue, but + # basically the snapd daemon along with lxcfs results in /dev/vdb + # being mounted in the lxcfs process namespace. If the charm + # 'tries' to umount it, it can (as root), but the mount is still + # 'held' by lxcfs and thus nothing else can be done with it. This + # is only a problem in serverstack with images with a default + # /dev/vdb ephemeral logging.warn("Skipping pristine disk test for focal and higher") return logging.info('Checking behaviour when non-pristine disks appear...') From eb0cba9efc0ba5c665d034986423ddee13503aec Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 20 Apr 2020 18:10:03 +0100 Subject: [PATCH 434/898] Add zaza tests for TrilioVault Add setup and tests for trilio-{data-mover,dm-api,wlm} charms. Add attach_volume utility to attach cinder volumes to nova servers. --- zaza/openstack/charm_tests/trilio/__init__.py | 15 + zaza/openstack/charm_tests/trilio/setup.py | 83 ++++ zaza/openstack/charm_tests/trilio/tests.py | 379 ++++++++++++++++++ zaza/openstack/utilities/openstack.py | 40 ++ 4 files changed, 517 insertions(+) create mode 100644 zaza/openstack/charm_tests/trilio/__init__.py create mode 100644 zaza/openstack/charm_tests/trilio/setup.py create mode 100644 zaza/openstack/charm_tests/trilio/tests.py diff --git a/zaza/openstack/charm_tests/trilio/__init__.py b/zaza/openstack/charm_tests/trilio/__init__.py new file mode 100644 index 0000000..d22e570 --- /dev/null +++ b/zaza/openstack/charm_tests/trilio/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing TrilioVault.""" diff --git a/zaza/openstack/charm_tests/trilio/setup.py b/zaza/openstack/charm_tests/trilio/setup.py new file mode 100644 index 0000000..a7ba7b3 --- /dev/null +++ b/zaza/openstack/charm_tests/trilio/setup.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for configuring Trilio.""" + +import logging +import os + +import zaza.model as zaza_model +import zaza.openstack.utilities.juju as juju_utils +import zaza.openstack.utilities.generic as generic_utils + + +def basic_setup(): + """Run setup for testing Trilio. + + Setup for testing Trilio is currently part of functional + tests. + """ + logging.info("Configuring NFS Server") + nfs_server_ip = zaza_model.get_app_ips("nfs-server-test-fixture")[0] + trilio_wlm_unit = zaza_model.get_first_unit_name("trilio-wlm") + + nfs_shares_conf = {"nfs-shares": "{}:/srv/testing".format(nfs_server_ip)} + _trilio_services = ["trilio-wlm", "trilio-data-mover"] + + conf_changed = False + for juju_service in _trilio_services: + app_config = zaza_model.get_application_config(juju_service) + if app_config["nfs-shares"] != nfs_shares_conf["nfs-shares"]: + zaza_model.set_application_config(juju_service, nfs_shares_conf) + conf_changed = True + + if conf_changed: + zaza_model.wait_for_agent_status() + # NOTE(jamespage): wlm-api service must be running in order + # to execute the setup actions + zaza_model.block_until_service_status( + unit_name=trilio_wlm_unit, + services=["wlm-api"], + target_status="active", + ) + + logging.info("Executing create-cloud-admin-trust") + password = juju_utils.leader_get("keystone", "admin_passwd") + + generic_utils.assertActionRanOK( + zaza_model.run_action_on_leader( + "trilio-wlm", + "create-cloud-admin-trust", + raise_on_failure=True, + action_params={"password": password}, + ) + ) + + logging.info("Executing create-license") + test_license = os.environ.get("TEST_TRILIO_LICENSE") + if test_license and os.path.exists(test_license): + zaza_model.attach_resource("trilio-wlm", + resource_name='license', + resource_path=test_license) + generic_utils.assertActionRanOK( + zaza_model.run_action_on_leader( + "trilio-wlm", "create-license", + raise_on_failure=True + ) + ) + + else: + logging.error("Unable to find Trilio License file") diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py new file mode 100644 index 0000000..e7595a5 --- /dev/null +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of tests for vault.""" + +import logging +import tenacity + +import zaza.model as zaza_model + +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.juju as juju_utils +import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.configure.guest as guest_utils + + +def _resource_reaches_status( + unit, auth_args, command, resource_id, target_status +): + """Wait for a workload resource to reach a status. + + :param unit: unit to run cli commands on + :type unit: zaza_model.Unit + :param auth_args: authentication arguments for command + :type auth_args: str + :param command: command to execute + :type command: str + :param resource_id: resource ID to monitor + :type resource_id: str + :param target_status: status to monitor for + :type target_status: str + """ + resource_status = ( + juju_utils.remote_run( + unit, + remote_cmd=command.format( + auth_args=auth_args, resource_id=resource_id + ), + timeout=180, + fatal=True, + ) + .strip() + .split("\n")[-1] + ) + logging.info( + "Checking resource ({}) status: {}".format( + resource_id, resource_status + ) + ) + if resource_status == target_status: + return + raise Exception("Resource not ready: {}".format(resource_status)) + + +class WorkloadmgrCLIHelper(object): + """Helper for working with workloadmgrcli.""" + + WORKLOAD_CREATE_CMD = ( + "openstack {auth_args} workload create " + "--instance instance-id={instance_id} " + "-f value -c ID" + ) + + WORKLOAD_STATUS_CMD = ( + "openstack {auth_args} workload show " + "-f value -c status " + " {resource_id} " + ) + + SNAPSHOT_CMD = ( + "openstack {auth_args} workload snapshot --full {workload_id}" + ) + + SNAPSHOT_ID_CMD = ( + "openstack {auth_args} workload snapshot list " + "--workload_id {workload_id} " + "-f value -c ID" + ) + + SNAPSHOT_STATUS_CMD = ( + "openstack {auth_args} workload snapshot show " + "-f value -c status " + "{resource_id} " + ) + + ONECLICK_RESTORE_CMD = ( + "openstack {auth_args} workload snapshot oneclick-restore " + "{snapshot_id} " + ) + + def __init__(self, keystone_client): + """Initialise helper. + + :param keystone_client: keystone client + :type keystone_client: keystoneclient.v3 + """ + self.trilio_wlm_unit = zaza_model.get_first_unit_name("trilio-wlm") + self.auth_args = self._auth_arguments(keystone_client) + + @classmethod + def _auth_arguments(cls, keystone_client): + """Generate workloadmgrcli arguments for cloud authentication. + + :returns: string of required cli arguments for authentication + :rtype: str + """ + overcloud_auth = openstack_utils.get_overcloud_auth() + overcloud_auth.update( + { + "OS_DOMAIN_ID": openstack_utils.get_domain_id( + keystone_client, domain_name="admin_domain" + ), + "OS_TENANT_ID": openstack_utils.get_project_id( + keystone_client, + project_name="admin", + domain_name="admin_domain", + ), + "OS_TENANT_NAME": "admin", + } + ) + + _required_keys = [ + "OS_AUTH_URL", + "OS_USERNAME", + "OS_PASSWORD", + "OS_REGION_NAME", + "OS_DOMAIN_ID", + "OS_TENANT_ID", + "OS_TENANT_NAME", + ] + + params = [] + for os_key in _required_keys: + params.append( + "--{}={}".format( + os_key.lower().replace("_", "-"), overcloud_auth[os_key] + ) + ) + return " ".join(params) + + def create_workload(self, instance_id): + """Create a new workload. + + :param instance_id: instance ID to create workload from + :type instance_id: str + :returns: workload ID + :rtype: str + """ + workload_id = juju_utils.remote_run( + self.trilio_wlm_unit, + remote_cmd=self.WORKLOAD_CREATE_CMD.format( + auth_args=self.auth_args, instance_id=instance_id + ), + timeout=180, + fatal=True, + ).strip() + + retryer = tenacity.Retrying( + wait=tenacity.wait_exponential(multiplier=1, max=60), + stop=tenacity.stop_after_delay(180), + reraise=True, + ) + retryer( + _resource_reaches_status, + self.trilio_wlm_unit, + self.auth_args, + self.WORKLOAD_STATUS_CMD, + workload_id, + "available", + ) + + return workload_id + + def create_snapshot(self, workload_id): + """Create a new snapshot. + + :param workload_id: workload ID to create snapshot from + :type workload_id: str + :returns: snapshot ID + :rtype: str + """ + juju_utils.remote_run( + self.trilio_wlm_unit, + remote_cmd=self.SNAPSHOT_CMD.format( + auth_args=self.auth_args, workload_id=workload_id + ), + timeout=180, + fatal=True, + ) + snapshot_id = juju_utils.remote_run( + self.trilio_wlm_unit, + remote_cmd=self.SNAPSHOT_ID_CMD.format( + auth_args=self.auth_args, workload_id=workload_id + ), + timeout=180, + fatal=True, + ).strip() + + retryer = tenacity.Retrying( + wait=tenacity.wait_exponential(multiplier=1, max=60), + stop=tenacity.stop_after_delay(720), + reraise=True, + ) + + retryer( + _resource_reaches_status, + self.trilio_wlm_unit, + self.auth_args, + self.SNAPSHOT_STATUS_CMD, + snapshot_id, + "available", + ) + + return snapshot_id + + def oneclick_restore(self, snapshot_id): + """Restore a workload from a snapshot. + + :param snapshot_id: snapshot ID to restore + :type snapshot_id: str + """ + juju_utils.remote_run( + self.trilio_wlm_unit, + remote_cmd=self.ONECLICK_RESTORE_CMD.format( + auth_args=self.auth_args, snapshot_id=snapshot_id + ), + timeout=180, + fatal=True, + ) + + # TODO validate restore but currently failing with 4.0 + # pre-release + + +class TrilioBaseTest(test_utils.OpenStackBaseTest): + """Base test class for charms.""" + + RESOURCE_PREFIX = "zaza-triliovault-tests" + conf_file = None + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super().setUpClass() + cls.cinder_client = openstack_utils.get_cinder_session_client( + cls.keystone_session + ) + cls.nova_client = openstack_utils.get_nova_session_client( + cls.keystone_session + ) + cls.keystone_client = openstack_utils.get_keystone_session_client( + cls.keystone_session + ) + + def test_restart_on_config_change(self): + """Check restart happens on config change. + + Change debug mode and assert that change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + set_default = {"debug": False} + set_alternate = {"debug": True} + + # Make config change, check for service restarts + self.restart_on_changed( + self.conf_file, + set_default, + set_alternate, + {"DEFAULT": {"debug": ["False"]}}, + {"DEFAULT": {"debug": ["True"]}}, + self.services, + ) + + def test_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + with self.pause_resume(self.services, pgrep_full=False): + logging.info("Testing pause resume") + + def test_snapshot_workload(self): + """Ensure that a workload can be created and snapshot'ed.""" + # Setup volume and instance and attach one to the other + volume = openstack_utils.create_volume( + self.cinder_client, + size="1", + name="{}-100-vol".format(self.RESOURCE_PREFIX), + ) + + instance = guest_utils.launch_instance( + glance_setup.CIRROS_IMAGE_NAME, + vm_name="{}-server".format(self.RESOURCE_PREFIX), + ) + + # Trilio need direct access to ceph - OMG + openstack_utils.attach_volume(self.nova_client, volume.id, instance.id) + + workloadmgrcli = WorkloadmgrCLIHelper(self.keystone_client) + + # Create workload using instance + logging.info("Creating workload configuration") + workload_id = workloadmgrcli.create_workload(instance.id) + logging.info("Created workload: {}".format(workload_id)) + + logging.info("Initiating snapshot") + snapshot_id = workloadmgrcli.create_snapshot(workload_id) + logging.info( + "Snapshot of workload {} created: {}".format( + workload_id, snapshot_id + ) + ) + + logging.info("Deleting server and volume ready for restore") + openstack_utils.delete_resource( + self.nova_client.servers, instance.id, "deleting instance" + ) + # NOTE: Trilio leaves a snapshot in place - + # drop before volume deletion. + for volume_snapshot in self.cinder_client.volume_snapshots.list(): + openstack_utils.delete_resource( + self.cinder_client.volume_snapshots, + volume_snapshot.id, + "deleting snapshot", + ) + openstack_utils.delete_resource( + self.cinder_client.volumes, volume.id, "deleting volume" + ) + + logging.info("Initiating restore") + workloadmgrcli.oneclick_restore(snapshot_id) + + +class TrilioWLMTest(TrilioBaseTest): + """Tests for Trilio Workload Manager charm.""" + + conf_file = "/etc/workloadmgr/workloadmgr.conf" + application_name = "trilio-wlm" + + services = [ + "workloadmgr-api", + "workloadmgr-scheduler", + "workloadmgr-workloads", + "workloadmgr-cron", + ] + + +class TrilioDMAPITest(TrilioBaseTest): + """Tests for Trilio Data Mover API charm.""" + + conf_file = "/etc/dmapi/dmapi.conf" + application_name = "trilio-dm-api" + + services = ["dmapi-api"] + + +class TrilioDataMoverTest(TrilioBaseTest): + """Tests for Trilio Data Mover charm.""" + + conf_file = "/etc/tvault-contego/tvault-contego.conf" + application_name = "trilio-data-mover" + + services = ["tvault-contego"] diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 875071b..3e72faf 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -487,6 +487,24 @@ def get_project_id(ks_client, project_name, api_version=2, domain_name=None): return None +def get_domain_id(ks_client, domain_name, api_version=2): + """Return domain ID. + + :param ks_client: Authenticated keystoneclient + :type ks_client: keystoneclient.v3.Client object + :param domain_name: Name of the domain + :type domain_name: string + :param api_version: API version number + :type api_version: int + :returns: Domain ID + :rtype: string or None + """ + all_domains = ks_client.domains.list(name=domain_name) + if all_domains: + return all_domains[0].id + return None + + # Neutron Helpers def get_gateway_uuids(): """Return machine uuids for neutron-gateway(s). @@ -2069,6 +2087,28 @@ def create_volume(cinder, size, name=None, image=None): return volume +def attach_volume(nova, volume_id, instance_id): + """Attach a cinder volume to a nova instance. + + :param nova: Authenticated nova client + :type nova: novaclient.v2.client.Client + :param volume_id: the id of the volume to attach + :type volume_id: str + :param instance_id: the id of the instance to attach the volume to + :type instance_id: str + :returns: nova volume pointer + :rtype: novaclient.v2.volumes.Volume + """ + logging.info( + 'Attaching volume {} to instance {}'.format( + volume_id, instance_id + ) + ) + return nova.volumes.create_server_volume(server_id=instance_id, + volume_id=volume_id, + device='/dev/vdx') + + def create_volume_backup(cinder, volume_id, name=None): """Create cinder volume backup. From 135e2dd8060bcda5cb1a2e44082192ff7aa50db0 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 4 May 2020 16:42:43 +0200 Subject: [PATCH 435/898] Re-enable neutron/test_800_ovs_bridges_are_managed_by_us --- zaza/openstack/charm_tests/neutron/tests.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index d94b47f..cb90b74 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -144,16 +144,12 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): self.assertIn('qos', ovs_agent['configurations']['extensions']) - @unittest.expectedFailure def test_800_ovs_bridges_are_managed_by_us(self): """Checking OVS bridges' external-id. OVS bridges created by us should be marked as managed by us in their external-id. See http://docs.openvswitch.org/en/latest/topics/integration/ - - NOTE(lourot): this test is expected to fail as long as this feature - hasn't landed yet: https://review.opendev.org/717074 """ for unit in zaza.model.get_units(self._APP_NAME, model_name=self.model_name): From 740bc2060f4e1d267aaa5c265b68b3e46221ba38 Mon Sep 17 00:00:00 2001 From: Dmitrii Shcherbakov Date: Mon, 4 May 2020 20:45:55 +0300 Subject: [PATCH 436/898] Pass arguments BaseCharmTest.setUpClass Not doing so triggers an incorrect behavior leading to functional test failures as the application name is not set correctly. https://github.com/openstack-charmers/zaza-openstack-tests/issues/256 https://review.opendev.org/#/c/712980/1 --- unit_tests/charm_tests/test_utils.py | 32 ++++++++++++++++++++++++ zaza/openstack/charm_tests/test_utils.py | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 unit_tests/charm_tests/test_utils.py diff --git a/unit_tests/charm_tests/test_utils.py b/unit_tests/charm_tests/test_utils.py new file mode 100644 index 0000000..280e2e2 --- /dev/null +++ b/unit_tests/charm_tests/test_utils.py @@ -0,0 +1,32 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import zaza.openstack.charm_tests.test_utils as test_utils + +from unittest.mock import patch + + +class TestOpenStackBaseTest(unittest.TestCase): + + @patch.object(test_utils.openstack_utils, 'get_cacert') + @patch.object(test_utils.openstack_utils, 'get_overcloud_keystone_session') + @patch.object(test_utils.BaseCharmTest, 'setUpClass') + def test_setUpClass(self, _setUpClass, _get_ovcks, _get_cacert): + + class MyTestClass(test_utils.OpenStackBaseTest): + model_name = 'deadbeef' + + MyTestClass.setUpClass('foo', 'bar') + _setUpClass.assert_called_with('foo', 'bar') diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index ae431c1..bcb5aff 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -427,7 +427,7 @@ class OpenStackBaseTest(BaseCharmTest): @classmethod def setUpClass(cls, application_name=None, model_alias=None): """Run setup for test class to create common resources.""" - super(OpenStackBaseTest, cls).setUpClass() + super(OpenStackBaseTest, cls).setUpClass(application_name, model_alias) cls.keystone_session = openstack_utils.get_overcloud_keystone_session( model_name=cls.model_name) cls.cacert = openstack_utils.get_cacert() From db0fff648040f4f08ba6a76f039f968c29302e73 Mon Sep 17 00:00:00 2001 From: James Page Date: Tue, 5 May 2020 12:28:09 +0100 Subject: [PATCH 437/898] triliovault: Check restore completes Ensure that the oneclick restore process completes as part of the snapshot test case. Drop use of api_version in get_domain_id utility. --- zaza/openstack/charm_tests/trilio/tests.py | 56 ++++++++++++++++++---- zaza/openstack/utilities/openstack.py | 4 +- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index e7595a5..99ef3e0 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -102,13 +102,26 @@ class WorkloadmgrCLIHelper(object): "{snapshot_id} " ) + RESTORE_LIST_CMD = ( + "openstack {auth_args} workloadmgr restore list " + "--snapshot_id {snapshot_id} " + "-f value -c ID" + ) + + RESTORE_STATUS_CMD = ( + "openstack {auth_args} workloadmgr restore show " + "-f value -c status {resource_id}" + ) + def __init__(self, keystone_client): """Initialise helper. :param keystone_client: keystone client :type keystone_client: keystoneclient.v3 """ - self.trilio_wlm_unit = zaza_model.get_first_unit_name("trilio-wlm") + self.trilio_wlm_unit = zaza_model.get_first_unit_name( + "trilio-wlm" + ) self.auth_args = self._auth_arguments(keystone_client) @classmethod @@ -147,7 +160,8 @@ class WorkloadmgrCLIHelper(object): for os_key in _required_keys: params.append( "--{}={}".format( - os_key.lower().replace("_", "-"), overcloud_auth[os_key] + os_key.lower().replace("_", "-"), + overcloud_auth[os_key], ) ) return " ".join(params) @@ -170,7 +184,7 @@ class WorkloadmgrCLIHelper(object): ).strip() retryer = tenacity.Retrying( - wait=tenacity.wait_exponential(multiplier=1, max=60), + wait=tenacity.wait_exponential(multiplier=1, max=30), stop=tenacity.stop_after_delay(180), reraise=True, ) @@ -211,7 +225,7 @@ class WorkloadmgrCLIHelper(object): ).strip() retryer = tenacity.Retrying( - wait=tenacity.wait_exponential(multiplier=1, max=60), + wait=tenacity.wait_exponential(multiplier=1, max=30), stop=tenacity.stop_after_delay(720), reraise=True, ) @@ -241,9 +255,31 @@ class WorkloadmgrCLIHelper(object): timeout=180, fatal=True, ) + restore_id = juju_utils.remote_run( + self.trilio_wlm_unit, + remote_cmd=self.RESTORE_LIST_CMD.format( + auth_args=self.auth_args, snapshot_id=snapshot_id + ), + timeout=180, + fatal=True, + ).strip() - # TODO validate restore but currently failing with 4.0 - # pre-release + retryer = tenacity.Retrying( + wait=tenacity.wait_exponential(multiplier=1, max=30), + stop=tenacity.stop_after_delay(720), + reraise=True, + ) + + retryer( + _resource_reaches_status, + self.trilio_wlm_unit, + self.auth_args, + self.RESTORE_STATUS_CMD, + restore_id, + "available", + ) + + return restore_id class TrilioBaseTest(test_utils.OpenStackBaseTest): @@ -310,7 +346,9 @@ class TrilioBaseTest(test_utils.OpenStackBaseTest): ) # Trilio need direct access to ceph - OMG - openstack_utils.attach_volume(self.nova_client, volume.id, instance.id) + openstack_utils.attach_volume( + self.nova_client, volume.id, instance.id + ) workloadmgrcli = WorkloadmgrCLIHelper(self.keystone_client) @@ -333,7 +371,9 @@ class TrilioBaseTest(test_utils.OpenStackBaseTest): ) # NOTE: Trilio leaves a snapshot in place - # drop before volume deletion. - for volume_snapshot in self.cinder_client.volume_snapshots.list(): + for ( + volume_snapshot + ) in self.cinder_client.volume_snapshots.list(): openstack_utils.delete_resource( self.cinder_client.volume_snapshots, volume_snapshot.id, diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 3e72faf..55f8d1b 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -487,15 +487,13 @@ def get_project_id(ks_client, project_name, api_version=2, domain_name=None): return None -def get_domain_id(ks_client, domain_name, api_version=2): +def get_domain_id(ks_client, domain_name): """Return domain ID. :param ks_client: Authenticated keystoneclient :type ks_client: keystoneclient.v3.Client object :param domain_name: Name of the domain :type domain_name: string - :param api_version: API version number - :type api_version: int :returns: Domain ID :rtype: string or None """ From c35b4e0fcf185339b17c103031eb54d660e31a64 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 5 May 2020 16:22:52 +0200 Subject: [PATCH 438/898] Disable test_101_neutron_sriov_config on Trusty https://bugs.launchpad.net/charm-neutron-openvswitch/+bug/1876888 --- zaza/openstack/charm_tests/neutron/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index d94b47f..e40831c 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -438,10 +438,10 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): def test_101_neutron_sriov_config(self): """Verify data in the sriov agent config file.""" - trusty_kilo = openstack_utils.get_os_release('trusty_kilo') - if self.current_os_release < trusty_kilo: + xenial_mitaka = openstack_utils.get_os_release('xenial_mitaka') + if self.current_os_release < xenial_mitaka: logging.debug('Skipping test, sriov agent not supported on < ' - 'trusty/kilo') + 'xenial/mitaka') return zaza.model.set_application_config( From c04e632ae092dd91dcbb2f316b84c3c95ad75df9 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 6 May 2020 14:59:32 +0100 Subject: [PATCH 439/898] The octavia tests require multiple LTS images This patch ensures that bionic and focal images are available for the LTS octavia tests. --- zaza/openstack/charm_tests/octavia/setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/octavia/setup.py b/zaza/openstack/charm_tests/octavia/setup.py index 4f93097..55f8903 100644 --- a/zaza/openstack/charm_tests/octavia/setup.py +++ b/zaza/openstack/charm_tests/octavia/setup.py @@ -26,6 +26,12 @@ import zaza.openstack.utilities.openstack as openstack import zaza.openstack.configure.guest +def ensure_lts_images(): + """Ensure that bionic and focal images are available for the tests.""" + glance_setup.add_lts_image(image_name='bionic', release='bionic') + glance_setup.add_lts_image(image_name='focal', release='focal') + + def add_amphora_image(image_url=None): """Add Octavia ``amphora`` test image to glance. From 59ec82f542f68d4ffc4b02742dc865a4fcf99cf1 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 7 May 2020 12:53:19 +0200 Subject: [PATCH 440/898] Make ssh_test() more robust --- .../test_zaza_utilities_openstack.py | 3 ++- zaza/openstack/charm_tests/barbican/tests.py | 2 +- zaza/openstack/charm_tests/neutron/tests.py | 1 - zaza/openstack/utilities/openstack.py | 19 ++++++++++++++++--- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 46efb0d..5c344bf 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -738,7 +738,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'bob', '10.0.0.10', 'myvm', - password='reallyhardpassord') + password='reallyhardpassord', + retry=False) paramiko_mock.connect.assert_called_once_with( '10.0.0.10', password='reallyhardpassord', diff --git a/zaza/openstack/charm_tests/barbican/tests.py b/zaza/openstack/charm_tests/barbican/tests.py index 3d54917..a11ff1f 100644 --- a/zaza/openstack/charm_tests/barbican/tests.py +++ b/zaza/openstack/charm_tests/barbican/tests.py @@ -22,7 +22,7 @@ import zaza.openstack.utilities.openstack as openstack_utils class BarbicanTest(test_utils.OpenStackBaseTest): - """Run nova-compute specific tests.""" + """Run barbican specific tests.""" _SERVICES = ['apache2', 'barbican-worker'] diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index e40831c..1f07ca9 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -712,7 +712,6 @@ class NeutronNetworkingTest(unittest.TestCase): openstack_utils.ssh_command( username, address, 'instance', 'ping -c 1 192.168.0.1', password=password, privkey=privkey, verify=verify) - pass def floating_ips_from_instance(instance): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 55f8d1b..98bb838 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2300,7 +2300,7 @@ def ping_response(ip): check=True) -def ssh_test(username, ip, vm_name, password=None, privkey=None): +def ssh_test(username, ip, vm_name, password=None, privkey=None, retry=True): """SSH to given ip using supplied credentials. :param username: Username to connect with @@ -2315,6 +2315,9 @@ def ssh_test(username, ip, vm_name, password=None, privkey=None): :param privkey: Private key to authenticate with. If a password is supplied it is used rather than the private key. :type privkey: str + :param retry: If True, retry a few times if an exception is raised in the + process, e.g. on connection failure. + :type retry: boolean :raises: exceptions.SSHFailed """ def verify(stdin, stdout, stderr): @@ -2328,8 +2331,18 @@ def ssh_test(username, ip, vm_name, password=None, privkey=None): vm_name)) raise exceptions.SSHFailed() - ssh_command(username, ip, vm_name, 'uname -n', - password=password, privkey=privkey, verify=verify) + # NOTE(lourot): paramiko.SSHClient().connect() calls read_all() which can + # raise an EOFError, see + # * https://docs.paramiko.org/en/stable/api/packet.html + # * https://github.com/paramiko/paramiko/issues/925 + # So retrying a few times makes sense. + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3 if retry else 1), + wait=tenacity.wait_exponential(multiplier=1, min=2, max=10), + reraise=True): + with attempt: + ssh_command(username, ip, vm_name, 'uname -n', + password=password, privkey=privkey, verify=verify) def ssh_command(username, From c8ea324ccb3ff57e84ada7c07d054aaa4df8918b Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 7 May 2020 20:09:02 +0100 Subject: [PATCH 441/898] Add focal template to image list --- zaza/openstack/utilities/openstack.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 875071b..dc50a68 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -72,7 +72,9 @@ CIRROS_RELEASE_URL = 'http://download.cirros-cloud.net/version/released' CIRROS_IMAGE_URL = 'http://download.cirros-cloud.net' UBUNTU_IMAGE_URLS = { 'bionic': ('http://cloud-images.ubuntu.com/{release}/current/' - '{release}-server-cloudimg-{arch}.img') + '{release}-server-cloudimg-{arch}.img'), + 'focal': ('http://cloud-images.ubuntu.com/{release}/current/' + '{release}-server-cloudimg-{arch}.img'), } CHARM_TYPES = { From 7783e38f668758101374d3151a762cfcbd8e3cb8 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 7 May 2020 22:17:31 +0000 Subject: [PATCH 442/898] Handle race in policyd tests --- zaza/openstack/charm_tests/policyd/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 2845781..88d6699 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -430,6 +430,8 @@ class BasePolicydSpecialization(PolicydTest, # now do the policyd override. logging.info("Doing policyd override with: {}".format(self._rule)) self._set_policy_with(self._rule) + zaza_model.block_until_wl_status_info_starts_with( + self.application_name, "PO:") zaza_model.block_until_all_units_idle() # now make sure the operation fails From 8edff98f94ea39b901caf02982a705f32cc7ab36 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 8 May 2020 08:28:24 +0200 Subject: [PATCH 443/898] Add OVN charm tests --- zaza/openstack/charm_tests/ovn/__init__.py | 15 +++++ zaza/openstack/charm_tests/ovn/tests.py | 73 ++++++++++++++++++++++ zaza/openstack/utilities/openstack.py | 5 ++ zaza/openstack/utilities/os_versions.py | 4 ++ 4 files changed, 97 insertions(+) create mode 100644 zaza/openstack/charm_tests/ovn/__init__.py create mode 100644 zaza/openstack/charm_tests/ovn/tests.py diff --git a/zaza/openstack/charm_tests/ovn/__init__.py b/zaza/openstack/charm_tests/ovn/__init__.py new file mode 100644 index 0000000..bd5900c --- /dev/null +++ b/zaza/openstack/charm_tests/ovn/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing OVN.""" diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py new file mode 100644 index 0000000..0708c67 --- /dev/null +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -0,0 +1,73 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate OVN testing.""" + +import logging + +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +class BaseCharmOperationTest(test_utils.BaseCharmTest): + """Base OVN Charm operation tests.""" + + # override if not possible to determine release pair from charm under test + release_application = None + + @classmethod + def setUpClass(cls): + """Run class setup for OVN charm operation tests.""" + super(BaseCharmOperationTest, cls).setUpClass() + cls.services = ['NotImplemented'] # This must be overridden + cls.current_release = openstack_utils.get_os_release( + openstack_utils.get_current_os_release_pair( + cls.release_application or cls.application_name)) + + def test_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped, then resume and check + they are started. + """ + with self.pause_resume(self.services): + logging.info('Testing pause resume (services="{}")' + .format(self.services)) + + +class CentralCharmOperationTest(BaseCharmOperationTest): + """OVN Central Charm operation tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for OVN Central charm operation tests.""" + super(CentralCharmOperationTest, cls).setUpClass() + cls.services = [ + 'ovn-northd', + 'ovsdb-server', + ] + + +class ChassisCharmOperationTest(BaseCharmOperationTest): + """OVN Chassis Charm operation tests.""" + + release_application = 'ovn-central' + + @classmethod + def setUpClass(cls): + """Run class setup for OVN Chassis charm operation tests.""" + super(ChassisCharmOperationTest, cls).setUpClass() + cls.services = [ + 'ovn-controller', + ] diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 98bb838..5c7d46a 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -108,6 +108,10 @@ CHARM_TYPES = { 'pkg': 'designate-common', 'origin_setting': 'openstack-origin' }, + 'ovn-central': { + 'pkg': 'ovn-common', + 'origin_setting': 'source' + }, } # Older tests use the order the services appear in the list to imply @@ -126,6 +130,7 @@ UPGRADE_SERVICES = [ {'name': 'nova-compute', 'type': CHARM_TYPES['nova']}, {'name': 'openstack-dashboard', 'type': CHARM_TYPES['openstack-dashboard']}, + {'name': 'ovn-central', 'type': CHARM_TYPES['ovn-central']}, ] diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 4ffd445..b2e430e 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -230,4 +230,8 @@ PACKAGE_CODENAMES = { ('9', 'train'), ('10', 'ussuri'), ]), + 'ovn-common': OrderedDict([ + ('2', 'train'), + ('20', 'ussuri'), + ]), } From 37297aff8f2eda9f496eb687f28ce20f7fd8bc76 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 8 May 2020 21:53:01 +0000 Subject: [PATCH 444/898] Reduce the number of permutations of msg checks Reduce the number of checks by 30%. Remove duplicate test 410. Use logging.info to get output. --- .../charm_tests/rabbitmq_server/tests.py | 94 ++++++++----------- 1 file changed, 37 insertions(+), 57 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index b114314..43e9bc8 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -87,6 +87,9 @@ class RmqTests(test_utils.OpenStackBaseTest): for check_unit in units: check_unit_name = check_unit.entity_id + if dest_unit_name == check_unit_name: + logging.info("Skipping check for this unit to itself.") + continue check_unit_host = check_unit.public_address check_unit_host_name = host_names[check_unit_name] @@ -95,20 +98,20 @@ class RmqTests(test_utils.OpenStackBaseTest): dest_unit_host, amqp_msg_stamp)).upper() # Publish amqp message - logging.debug('Publish message to: {} ' - '({} {})'.format(dest_unit_host, - dest_unit_name, - dest_unit_host_name)) + logging.info('Publish message to: {} ' + '({} {})'.format(dest_unit_host, + dest_unit_name, + dest_unit_host_name)) rmq_utils.publish_amqp_message_by_unit(dest_unit, amqp_msg, ssl=ssl, port=port) # Get amqp message - logging.debug('Get message from: {} ' - '({} {})'.format(check_unit_host, - check_unit_name, - check_unit_host_name)) + logging.info('Get message from: {} ' + '({} {})'.format(check_unit_host, + check_unit_name, + check_unit_host_name)) amqp_msg_rcvd = self._retry_get_amqp_message(check_unit, ssl=ssl, @@ -116,8 +119,8 @@ class RmqTests(test_utils.OpenStackBaseTest): # Validate amqp message content if amqp_msg == amqp_msg_rcvd: - logging.debug('Message {} received ' - 'OK.'.format(amqp_msg_counter)) + logging.info('Message {} received ' + 'OK.'.format(amqp_msg_counter)) else: logging.error('Expected: {}'.format(amqp_msg)) logging.error('Actual: {}'.format(amqp_msg_rcvd)) @@ -131,8 +134,8 @@ class RmqTests(test_utils.OpenStackBaseTest): def test_400_rmq_cluster_running_nodes(self): """Verify cluster status shows every cluster node as running member.""" - logging.debug('Checking that all units are in cluster_status ' - 'running nodes...') + logging.info('Checking that all units are in cluster_status ' + 'running nodes...') units = zaza.model.get_units(self.application_name) @@ -148,8 +151,8 @@ class RmqTests(test_utils.OpenStackBaseTest): unit for messages. Uses Standard amqp tcp port, no ssl. """ - logging.debug('Checking amqp message publish/get on all units ' - '(ssl off)...') + logging.info('Checking amqp message publish/get on all units ' + '(ssl off)...') units = zaza.model.get_units(self.application_name) self._test_rmq_amqp_messages_all_units(units, ssl=False) @@ -170,36 +173,13 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('Skipping SSL tests due to client' ' compatibility issues') return - logging.debug('Checking amqp message publish/get on all units ' - '(ssl on)...') + logging.info('Checking amqp message publish/get on all units ' + '(ssl on)...') self._test_rmq_amqp_messages_all_units(units, ssl=True, port=5671) logging.info('OK') - def test_410_rmq_amqp_messages_all_units_ssl_alt_port(self): - """Send (and check) amqp messages to every rmq unit (alt ssl port). - - Send amqp messages with ssl on, to every rmq unit and check - every rmq unit for messages. Custom ssl tcp port. - - """ - units = zaza.model.get_units(self.application_name) - - # http://pad.lv/1625044 - if CompareHostReleases(get_series(units[0])) <= 'trusty': - logging.info('SKIP') - logging.info('Skipping SSL tests due to client' - ' compatibility issues') - return - logging.debug('Checking amqp message publish/get on all units ' - '(ssl on)...') - - units = zaza.model.get_units(self.application_name) - self._test_rmq_amqp_messages_all_units(units, - ssl=True, port=5999) - logging.info('OK') - @tenacity.retry( retry=tenacity.retry_if_result(lambda ret: ret is not None), wait=tenacity.wait_fixed(30), @@ -211,14 +191,14 @@ class RmqTests(test_utils.OpenStackBaseTest): def test_412_rmq_management_plugin(self): """Enable and check management plugin.""" - logging.debug('Checking tcp socket connect to management plugin ' - 'port on all rmq units...') + logging.info('Checking tcp socket connect to management plugin ' + 'port on all rmq units...') units = zaza.model.get_units(self.application_name) mgmt_port = 15672 # Enable management plugin - logging.debug('Enabling management_plugin charm config option...') + logging.info('Enabling management_plugin charm config option...') config = {'management_plugin': 'True'} zaza.model.set_application_config('rabbitmq-server', config) rmq_utils.wait_for_cluster() @@ -227,10 +207,10 @@ class RmqTests(test_utils.OpenStackBaseTest): ret = self._retry_port_knock_units(units, mgmt_port) self.assertIsNone(ret, msg=ret) - logging.debug('Connect to all units (OK)') + logging.info('Connect to all units (OK)') # Disable management plugin - logging.debug('Disabling management_plugin charm config option...') + logging.info('Disabling management_plugin charm config option...') config = {'management_plugin': 'False'} zaza.model.set_application_config('rabbitmq-server', config) rmq_utils.wait_for_cluster() @@ -259,21 +239,21 @@ class RmqTests(test_utils.OpenStackBaseTest): host_names = generic_utils.get_unit_hostnames(units) # check_rabbitmq monitor - logging.debug('Checking nrpe check_rabbitmq on units...') + logging.info('Checking nrpe check_rabbitmq on units...') cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' 'check_rabbitmq.cfg'] ret = self._retry_check_commands_on_units(cmds, units) self.assertIsNone(ret, msg=ret) # check_rabbitmq_queue monitor - logging.debug('Checking nrpe check_rabbitmq_queue on units...') + logging.info('Checking nrpe check_rabbitmq_queue on units...') cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' 'check_rabbitmq_queue.cfg'] ret = self._retry_check_commands_on_units(cmds, units) self.assertIsNone(ret, msg=ret) # check dat file existence - logging.debug('Checking nrpe dat file existence on units...') + logging.info('Checking nrpe dat file existence on units...') for u in units: unit_host_name = host_names[u.entity_id] @@ -291,7 +271,7 @@ class RmqTests(test_utils.OpenStackBaseTest): def test_910_pause_and_resume(self): """The services can be paused and resumed.""" - logging.debug('Checking pause and resume actions...') + logging.info('Checking pause and resume actions...') unit = zaza.model.get_units(self.application_name)[0] assert unit.workload_status == "active" @@ -307,21 +287,21 @@ class RmqTests(test_utils.OpenStackBaseTest): assert unit.workload_status == "active" rmq_utils.wait_for_cluster() - logging.debug('OK') + logging.info('OK') def test_911_cluster_status(self): """Test rabbitmqctl cluster_status action can be returned.""" - logging.debug('Checking cluster status action...') + logging.info('Checking cluster status action...') unit = zaza.model.get_units(self.application_name)[0] action = zaza.model.run_action(unit.entity_id, "cluster-status") self.assertIsInstance(action, juju.action.Action) - logging.debug('OK') + logging.info('OK') def test_912_check_queues(self): """Test rabbitmqctl check_queues action can be returned.""" - logging.debug('Checking cluster status action...') + logging.info('Checking cluster status action...') unit = zaza.model.get_units(self.application_name)[0] action = zaza.model.run_action(unit.entity_id, "check-queues") @@ -329,7 +309,7 @@ class RmqTests(test_utils.OpenStackBaseTest): def test_913_list_unconsumed_queues(self): """Test rabbitmqctl list-unconsumed-queues action can be returned.""" - logging.debug('Checking list-unconsumed-queues action...') + logging.info('Checking list-unconsumed-queues action...') unit = zaza.model.get_units(self.application_name)[0] self._test_rmq_amqp_messages_all_units([unit]) @@ -354,7 +334,7 @@ class RmqTests(test_utils.OpenStackBaseTest): # should have already been consumed. assert queue_data['messages'] == 0, 'Found unexpected message count.' - logging.debug('OK') + logging.info('OK') @tenacity.retry( retry=tenacity.retry_if_result(lambda errors: bool(errors)), @@ -370,8 +350,8 @@ class RmqTests(test_utils.OpenStackBaseTest): RabbitMQ cluster on removal """ - logging.debug('Checking that units correctly clean up after ' - 'themselves on unit removal...') + logging.info('Checking that units correctly clean up after ' + 'themselves on unit removal...') config = {'min-cluster-size': '2'} zaza.model.set_application_config('rabbitmq-server', config) rmq_utils.wait_for_cluster() @@ -397,4 +377,4 @@ class RmqTests(test_utils.OpenStackBaseTest): errors.append(e) self.assertFalse(errors, msg=errors) - logging.debug('OK') + logging.info('OK') From a8a25b213476b89df9449aca7ba4fcbe4f85b876 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 11 May 2020 10:02:42 +0200 Subject: [PATCH 445/898] Handle leader prepare step before non-leaders --- zaza/openstack/utilities/parallel_series_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index ba8b205..aa6ab0e 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -212,11 +212,11 @@ async def parallel_series_upgrade( for unit in status["units"] ] await asyncio.gather(*app_idle) + await prepare_series_upgrade(leader_machine, to_series=to_series) prepare_group = [ prepare_series_upgrade(machine, to_series=to_series) for machine in machines] await asyncio.gather(*prepare_group) - await prepare_series_upgrade(leader_machine, to_series=to_series) if leader_machine not in completed_machines: machines.append(leader_machine) upgrade_group = [ From 9ae0d464b3f6e6d73e8214efc171912e41e57b5e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 11 May 2020 14:54:27 +0200 Subject: [PATCH 446/898] Fix "'NeutronGatewayTest' object has no attribute 'neutron_client'" in test_401_enable_qos --- zaza/openstack/charm_tests/neutron/tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 1f07ca9..6e2759b 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -113,6 +113,10 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): super(NeutronGatewayTest, cls).setUpClass(cls) cls.services = cls._get_services() + # set up clients + cls.neutron_client = ( + openstack_utils.get_neutron_session_client(cls.keystone_session)) + _APP_NAME = 'neutron-gateway' def test_401_enable_qos(self): @@ -128,7 +132,8 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): self._validate_openvswitch_agent_qos() - @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60)) + @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60), + reraise=True, stop=tenacity.stop_after_attempt(8)) def _validate_openvswitch_agent_qos(self): """Validate that the qos extension is enabled in the ovs agent.""" # obtain the dhcp agent to identify the neutron-gateway host From 695fbd85dacdf1bcb2fcb7d58dedb7295ddab521 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 12 May 2020 14:12:35 +0200 Subject: [PATCH 447/898] Ensure the secondary-first function takes the same arguments --- zaza/openstack/utilities/series_upgrade.py | 24 ++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 98626b7..bff8afc 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -112,9 +112,11 @@ def series_upgrade_non_leaders_first( to_series="xenial", origin='openstack-origin', completed_machines=[], - post_upgrade_functions=None, pause_non_leader_primary=False, - pause_non_leader_subordinate=False + pause_non_leader_subordinate=False, + files=None, + workaround_script=None, + post_upgrade_functions=None ): """Series upgrade non leaders first. @@ -141,6 +143,10 @@ def series_upgrade_non_leaders_first( paused :type pause_non_leader_subordinate: bool :param from_series: The series from which to upgrade + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str :returns: None :rtype: None """ @@ -194,6 +200,8 @@ def series_upgrade_non_leaders_first( series_upgrade(leader, machine, from_series=from_series, to_series=to_series, origin=origin, + workaround_script=workaround_script, + files=files, post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: @@ -208,9 +216,11 @@ async def async_series_upgrade_non_leaders_first( to_series="xenial", origin='openstack-origin', completed_machines=[], - post_upgrade_functions=None, pause_non_leader_primary=False, - pause_non_leader_subordinate=False + pause_non_leader_subordinate=False, + files=None, + workaround_script=None, + post_upgrade_functions=None ): """Series upgrade non leaders first. @@ -237,6 +247,10 @@ async def async_series_upgrade_non_leaders_first( paused :type pause_non_leader_subordinate: bool :param from_series: The series from which to upgrade + :param files: Workaround files to scp to unit under upgrade + :type files: list + :param workaround_script: Workaround script to run during series upgrade + :type workaround_script: str :returns: None :rtype: None """ @@ -292,6 +306,8 @@ async def async_series_upgrade_non_leaders_first( leader, machine, from_series=from_series, to_series=to_series, origin=origin, + workaround_script=workaround_script, + files=files, post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) else: From 19fd66a17829f679fb95e85ad02c62e606d77fb0 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 12 May 2020 14:22:16 +0200 Subject: [PATCH 448/898] Fix new lint errors --- zaza/openstack/charm_tests/manila_ganesha/tests.py | 3 +-- zaza/openstack/utilities/series_upgrade.py | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 44ed2cd..e24a9f3 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -125,6 +125,5 @@ packages: openstack_utils.ssh_command( username, fip_2, 'instance-2', - 'sudo cat /mnt/ceph/test'.format( - mount_path), + 'sudo cat /mnt/ceph/test', password=password, privkey=privkey, verify=verify) diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index bff8afc..f12f77b 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -190,7 +190,7 @@ def series_upgrade_non_leaders_first( completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded. " - .format(unit, machine, application)) + .format(unit, machine)) model.block_until_all_units_idle() # Series upgrade the leader @@ -206,7 +206,7 @@ def series_upgrade_non_leaders_first( completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded." - .format(unit, machine, application)) + .format(unit, machine)) model.block_until_all_units_idle() @@ -295,7 +295,7 @@ async def async_series_upgrade_non_leaders_first( completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded. " - .format(unit, machine, application)) + .format(unit, machine)) await model.async_block_until_all_units_idle() # Series upgrade the leader @@ -312,7 +312,7 @@ async def async_series_upgrade_non_leaders_first( completed_machines.append(machine) else: logging.info("Skipping unit: {}. Machine: {} already upgraded." - .format(unit, machine, application)) + .format(unit, machine)) await model.async_block_until_all_units_idle() From 593254f7905f6329bde1bcf41003043b1617167f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 13 May 2020 11:52:48 +0100 Subject: [PATCH 449/898] libjuju-2.8 breaks with juju 2.7.6 and 2.8.0 - pinning This is a temporary pin until a fix is provided. --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index fa0e460..1f685cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiounittest async_generator boto3 -juju +juju<2.8.0 juju_wait PyYAML<=4.2,>=3.0 flake8>=2.2.4 diff --git a/setup.py b/setup.py index 6d08832..3e781da 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ install_require = [ 'cryptography', 'hvac<0.7.0', 'jinja2', - 'juju', + 'juju<2.8.0', 'juju-wait', 'lxml', 'PyYAML', From fd3f0bb09355d73574cb83700a7d8381e981260f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 21 Apr 2020 15:39:48 +0100 Subject: [PATCH 450/898] Migrate 499 cinder-ceph test to zaza This takes the 499 test from basic_deployment.py (Amulet) test and ports it over to zaza. --- .../charm_tests/ceph/mon/__init__.py | 15 ++++ zaza/openstack/charm_tests/ceph/mon/tests.py | 69 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 zaza/openstack/charm_tests/ceph/mon/__init__.py create mode 100644 zaza/openstack/charm_tests/ceph/mon/tests.py diff --git a/zaza/openstack/charm_tests/ceph/mon/__init__.py b/zaza/openstack/charm_tests/ceph/mon/__init__.py new file mode 100644 index 0000000..867c3af --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/mon/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ceph-mon for cinder-ceph.""" diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py new file mode 100644 index 0000000..50063c7 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -0,0 +1,69 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Ceph-mon Testing for cinder-ceph.""" + +import logging +import unittest + +import zaza.model + +from zaza.openstack.utilities import ( + generic as generic_utils, + openstack as openstack_utils, +) + + +class CinderCephMonTest(unittest.TestCase): + """Verify that the ceph mon units are healthy.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph security tests.""" + super().setUpClass() + + # ported from the cinder-ceph Amulet test + def test_499_ceph_cmds_exit_zero(self): + """Verify expected state with security-checklist.""" + logging.info("Checking exit values are 0 on ceph commands.") + + units = zaza.model.get_units("ceph-mon", model_name=self.model_name) + current_release = openstack_utils.get_os_release() + bionic_train = openstack_utils.get_os_release('bionic_train') + if current_release < bionic_train: + units.extend(zaza.model.get_utils("cinder-ceph", + model_name=self.model_name)) + + commands = [ + 'sudo ceph health', + 'sudo ceph mds stat', + 'sudo ceph pg stat', + 'sudo ceph osd stat', + 'sudo ceph mon stat', + ] + + for unit in units: + run_commands(unit.name, commands) + + +def run_commands(self, unit_name, commands): + """Run commands on unit. + + Apply context to commands until all variables have been replaced, then + run the command on the given unit. + """ + for cmd in commands: + generic_utils.assertRemoteRunOK(zaza.model.run_on_unit( + unit_name, + cmd)) From c6f047268139b418c2e6f24b054daab57b9e0f18 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 4 May 2020 18:23:07 +0100 Subject: [PATCH 451/898] Fix the broken bits in the tests copied from amulet --- zaza/openstack/charm_tests/ceph/mon/tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index 50063c7..276ed9f 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -23,14 +23,15 @@ from zaza.openstack.utilities import ( generic as generic_utils, openstack as openstack_utils, ) +import zaza.openstack.charm_tests.test_utils as test_utils -class CinderCephMonTest(unittest.TestCase): +class CinderCephMonTest(test_utils.OpenStackBaseTest): """Verify that the ceph mon units are healthy.""" @classmethod def setUpClass(cls): - """Run class setup for running ceph security tests.""" + """Run class setup for running ceph mon tests with cinder.""" super().setUpClass() # ported from the cinder-ceph Amulet test @@ -57,7 +58,7 @@ class CinderCephMonTest(unittest.TestCase): run_commands(unit.name, commands) -def run_commands(self, unit_name, commands): +def run_commands(unit_name, commands): """Run commands on unit. Apply context to commands until all variables have been replaced, then From 294fd83e498cfaeefe8c92913626271ed6fcb162 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 4 May 2020 18:23:34 +0100 Subject: [PATCH 452/898] Fix up cinder tests to work when cinder isn't the application under test The cinder tests were written with the assumption that cinder was the charm that was under test. This modifies the test so that the cinder tests work in a model with cinder where cinder isn't the application that is being explicitly tested. --- zaza/openstack/charm_tests/cinder/tests.py | 68 ++++++++++++++++++---- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index d645eed..460ff00 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -32,7 +32,10 @@ class CinderTests(test_utils.OpenStackBaseTest): @classmethod def setUpClass(cls): """Run class setup for running tests.""" - super(CinderTests, cls).setUpClass() + super(CinderTests, cls).setUpClass(application_name='cinder') + cls.application_name = 'cinder' + cls.lead_unit = zaza.model.get_lead_unit_name( + "cinder", model_name=cls.model_name) cls.cinder_client = openstack_utils.get_cinder_session_client( cls.keystone_session) cls.nova_client = openstack_utils.get_nova_session_client( @@ -42,18 +45,61 @@ class CinderTests(test_utils.OpenStackBaseTest): def tearDown(cls): """Remove test resources.""" logging.info('Running teardown') - for snapshot in cls.cinder_client.volume_snapshots.list(): + volumes = list(cls.cinder_client.volumes.list()) + snapped_volumes = [v for v in volumes if v.name.endswith("-from-snap")] + if snapped_volumes: + logging.info("Removing volumes from snapshot") + cls._remove_volumes(snapped_volumes) + volumes = list(cls.cinder_client.volumes.list()) + + snapshots = list(cls.cinder_client.volume_snapshots.list()) + if snapshots: + logging.info("tearDown - snapshots: {}".format( + ", ".join(s.name for s in snapshots))) + cls._remove_snapshots(snapshots) + + if volumes: + logging.info("tearDown - volumes: {}".format( + ", ".join(v.name for v in volumes))) + cls._remove_volumes(volumes) + + @classmethod + def _remove_snapshots(cls, snapshots): + """Remove snapshots passed as param. + + :param volumes: the snapshots to delete + :type volumes: List[snapshot objects] + """ + for snapshot in snapshots: if snapshot.name.startswith(cls.RESOURCE_PREFIX): - openstack_utils.delete_resource( - cls.cinder_client.volume_snapshots, - snapshot.id, - msg="snapshot") - for volume in cls.cinder_client.volumes.list(): + logging.info("removing snapshot: {}".format(snapshot.name)) + try: + openstack_utils.delete_resource( + cls.cinder_client.volume_snapshots, + snapshot.id, + msg="snapshot") + except Exception as e: + logging.error("error removing snapshot: {}".format(str(e))) + raise + + @classmethod + def _remove_volumes(cls, volumes): + """Remove volumes passed as param. + + :param volumes: the volumes to delete + :type volumes: List[volume objects] + """ + for volume in volumes: if volume.name.startswith(cls.RESOURCE_PREFIX): - openstack_utils.delete_resource( - cls.cinder_client.volumes, - volume.id, - msg="volume") + logging.info("removing volume: {}".format(volume.name)) + try: + openstack_utils.delete_resource( + cls.cinder_client.volumes, + volume.id, + msg="volume") + except Exception as e: + logging.error("error removing volume: {}".format(str(e))) + raise def test_100_volume_create_extend_delete(self): """Test creating, extending a volume.""" From 3de62513161efbb0b1ea831628bff15764086a59 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 5 May 2020 15:07:44 +0100 Subject: [PATCH 453/898] Fix broken code and collect errors on ceph commands --- zaza/openstack/charm_tests/ceph/mon/tests.py | 16 ++++++++++++---- zaza/openstack/utilities/exceptions.py | 6 ++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index 276ed9f..0fea084 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -22,6 +22,7 @@ import zaza.model from zaza.openstack.utilities import ( generic as generic_utils, openstack as openstack_utils, + exceptions as zaza_exceptions ) import zaza.openstack.charm_tests.test_utils as test_utils @@ -43,7 +44,7 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): current_release = openstack_utils.get_os_release() bionic_train = openstack_utils.get_os_release('bionic_train') if current_release < bionic_train: - units.extend(zaza.model.get_utils("cinder-ceph", + units.extend(zaza.model.get_units("cinder-ceph", model_name=self.model_name)) commands = [ @@ -64,7 +65,14 @@ def run_commands(unit_name, commands): Apply context to commands until all variables have been replaced, then run the command on the given unit. """ + errors = [] for cmd in commands: - generic_utils.assertRemoteRunOK(zaza.model.run_on_unit( - unit_name, - cmd)) + try: + generic_utils.assertRemoteRunOK(zaza.model.run_on_unit( + unit_name, + cmd)) + except Exception as e: + errors.append("unit: {}, command: {}, error: {}" + .format(unit_name, cmd, str(e))) + if errors: + raise zaza_exceptions.CephGenericError("\n".join(errors)) diff --git a/zaza/openstack/utilities/exceptions.py b/zaza/openstack/utilities/exceptions.py index e9e5d23..f8673a1 100644 --- a/zaza/openstack/utilities/exceptions.py +++ b/zaza/openstack/utilities/exceptions.py @@ -168,6 +168,12 @@ class CephPoolNotConfigured(Exception): pass +class CephGenericError(Exception): + """A generic/other Ceph error occurred.""" + + pass + + class NovaGuestMigrationFailed(Exception): """Nova guest migration failed.""" From 935af95dd23c38a4d8d570482498c6f72f649be6 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 6 May 2020 12:09:22 +0100 Subject: [PATCH 454/898] Add in missing 500 test from amulet test This is the missing 500 test from the previous amulet tests that were missed on the first pass. --- zaza/openstack/charm_tests/ceph/mon/tests.py | 66 +++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index 0fea084..39cf4a2 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -15,7 +15,6 @@ """Ceph-mon Testing for cinder-ceph.""" import logging -import unittest import zaza.model @@ -58,6 +57,57 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): for unit in units: run_commands(unit.name, commands) + # ported from the cinder-ceph Amulet test + def test_500_ceph_alternatives_cleanup(self): + """Check ceph alternatives removed when ceph-mon relation is broken.""" + # Skip this test if release is less than xenial_ocata as in that case + # cinder HAS a relation with ceph directly and this test would fail + current_release = openstack_utils.get_os_release() + xenial_ocata = openstack_utils.get_os_release('xenial_ocata') + if current_release < xenial_ocata: + logging.info("Skipping test as release < xenial-ocata") + return + + units = zaza.model.get_units("cinder-ceph", + model_name=self.model_name) + + # check each unit prior to breaking relation + for unit in units: + dir_list = directory_listing(unit.name, "/etc/ceph") + if 'ceph.conf' in dir_list: + logging.debug( + "/etc/ceph/ceph.conf exists BEFORE relation-broken") + else: + raise zaza_exceptions.CephGenericError( + "unit: {} - /etc/ceph/ceph.conf does not exist " + "BEFORE relation-broken".format(unit.name)) + + # remove the relation so that /etc/ceph/ceph.conf is removed + logging.info("Removing ceph-mon:client <-> cinder-ceph:ceph relation") + zaza.model.remove_relation( + "ceph-mon", "ceph-mon:client" "cinder-ceph:ceph") + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # check each unit after to breaking relation + for unit in units: + dir_list = directory_listing(unit.name, "/etc/ceph") + if 'ceph.conf' not in dir_list: + logging.debug( + "/etc/ceph/ceph.conf removed AFTER relation-broken") + else: + raise zaza_exceptions.CephGenericError( + "unit: {} - /etc/ceph/ceph.conf still exists " + "AFTER relation-broken".format(unit.name)) + + # Restore cinder-ceph and ceph-mon relation to keep tests idempotent + logging.info("Restoring ceph-mon:client <-> cinder-ceph:ceph relation") + zaza.model.add_relation( + "ceph-mon", "ceph-mon:client" "cinder-ceph:ceph") + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + logging.info("... Done.") + def run_commands(unit_name, commands): """Run commands on unit. @@ -76,3 +126,17 @@ def run_commands(unit_name, commands): .format(unit_name, cmd, str(e))) if errors: raise zaza_exceptions.CephGenericError("\n".join(errors)) + + +def directory_listing(unit_name, directory): + """Return a list of files/directories from a directory on a unit. + + :param unit_name: the unit to fetch the directory listing from + :type unit_name: str + :param directory: the directory to fetch the listing from + :type directory: str + :returns: A listing using "ls -1" on the unit + :rtype: List[str] + """ + result = zaza.model.run_on_unit(unit_name, "ls -1 {}".format(directory)) + return result['Stdout'].splitlines() From a7df335cc84cc5fb175cb4c8df00faf7c38d838c Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 6 May 2020 15:31:53 +0100 Subject: [PATCH 455/898] Fix missing comma --- zaza/openstack/charm_tests/ceph/mon/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index 39cf4a2..dd6d7aa 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -85,7 +85,7 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): # remove the relation so that /etc/ceph/ceph.conf is removed logging.info("Removing ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.remove_relation( - "ceph-mon", "ceph-mon:client" "cinder-ceph:ceph") + "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() @@ -103,7 +103,7 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): # Restore cinder-ceph and ceph-mon relation to keep tests idempotent logging.info("Restoring ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.add_relation( - "ceph-mon", "ceph-mon:client" "cinder-ceph:ceph") + "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() logging.info("... Done.") From 8e826f6f8bc277b75b962b2516238ae1a80a3a83 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 13 May 2020 07:57:24 +0100 Subject: [PATCH 456/898] Add debug to 105 test --- zaza/openstack/charm_tests/cinder/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 460ff00..ec6caf0 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -126,12 +126,18 @@ class CinderTests(test_utils.OpenStackBaseTest): def test_105_volume_create_from_img(self): """Test creating a volume from an image.""" + logging.debug("finding image {} ..." + .format(glance_setup.LTS_IMAGE_NAME) image = self.nova_client.glance.find_image( glance_setup.LTS_IMAGE_NAME) + logging.debug("using cinder_client to create volume from image {}" + .format(image.id)) vol_img = self.cinder_client.volumes.create( name='{}-105-vol-from-img'.format(self.RESOURCE_PREFIX), size=3, imageRef=image.id) + logging.debug("now waiting for volume {} to reach available" + .format(vol_img.id)) openstack_utils.resource_reaches_status( self.cinder_client.volumes, vol_img.id, From 4c2f723332e2bfa5e67e80741c606e845050895c Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 13 May 2020 14:34:13 +0100 Subject: [PATCH 457/898] Fix missing brace --- zaza/openstack/charm_tests/cinder/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index ec6caf0..4a851ce 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -127,7 +127,7 @@ class CinderTests(test_utils.OpenStackBaseTest): def test_105_volume_create_from_img(self): """Test creating a volume from an image.""" logging.debug("finding image {} ..." - .format(glance_setup.LTS_IMAGE_NAME) + .format(glance_setup.LTS_IMAGE_NAME)) image = self.nova_client.glance.find_image( glance_setup.LTS_IMAGE_NAME) logging.debug("using cinder_client to create volume from image {}" From a56fa1fd74668f3ed3f5dcba3bdd3f43c59e7695 Mon Sep 17 00:00:00 2001 From: Marco Silva Date: Thu, 14 May 2020 09:34:44 +0100 Subject: [PATCH 458/898] Add mysqldump test to MySqlCommonTests Copy function from MySQLInnoDBClusterTests to test mysqldump as standard on mysql applications. DocImpact Closes-Bug: #0000000 Implements: MySQL dump test --- zaza/openstack/charm_tests/mysql/tests.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 77bc966..fb1ac61 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -75,6 +75,23 @@ class MySQLBaseTest(test_utils.OpenStackBaseTest): class MySQLCommonTests(MySQLBaseTest): """Common mysql charm tests.""" + def test_110_mysqldump(self): + """Backup mysql. + + Run the mysqldump action. + """ + _db = "keystone" + _file_key = "mysqldump-file" + logging.info("Execute mysqldump action") + action = zaza.model.run_action_on_leader( + self.application, + "mysqldump", + action_params={"databases": _db}) + _results = action.data["results"] + assert _db in _results[_file_key], ( + "Mysqldump action failed: {}".format(action.data)) + logging.info("Passed mysqldump action test.") + def test_910_restart_on_config_change(self): """Checking restart happens on config change. From 5d05eb70ccc3d71613b5326a6a28407175498d8d Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 14 May 2020 19:37:45 +0100 Subject: [PATCH 459/898] Ensure model starts executing before the wait There was a race before a block_until_all_units_idle() where it could blast through the check before it started removing the relation. This ensures that it waits until something happens and then waits for it to finish. --- zaza/openstack/charm_tests/ceph/mon/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index dd6d7aa..63c4033 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -86,6 +86,7 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): logging.info("Removing ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.remove_relation( "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") + zaza.model.wait_for_agent_status() logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() @@ -104,6 +105,7 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): logging.info("Restoring ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.add_relation( "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") + zaza.model.wait_for_agent_status() logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() logging.info("... Done.") From 52ee23927c27bbf02bf0a47fa5090e78d01a094a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 15 May 2020 13:54:01 +0000 Subject: [PATCH 460/898] Make masakari wait longer for hosts to online A recent functional test of masakari monitors failed due to a tenacity retry missing the host coming online by 10s, so wait longer. FWIW that this affects the post-test cleanup rather than the test itself. *1 https://openstack-ci-reports.ubuntu.com/artifacts/test_charm_pipeline_func_full/openstack/charm-masakari-monitors/726808/1/5655/consoleText.test_charm_func_full_8812.txt ` --- zaza/openstack/configure/masakari.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/configure/masakari.py b/zaza/openstack/configure/masakari.py index a529bbe..707416f 100644 --- a/zaza/openstack/configure/masakari.py +++ b/zaza/openstack/configure/masakari.py @@ -85,7 +85,7 @@ def create_segments(segment_number=1, host_assignment_method=None): @tenacity.retry( wait=tenacity.wait_exponential(multiplier=2, max=60), - reraise=True, stop=tenacity.stop_after_attempt(5), + reraise=True, stop=tenacity.stop_after_attempt(10), retry=tenacity.retry_if_exception_type(ostack_except.ConflictException)) def enable_host(masakari_client, host, segment): """Enable hypervisor within masakari. From 1a0210b0a53871b1c59b8f5110d56a5cfedf3c5b Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sat, 16 May 2020 08:29:12 +0000 Subject: [PATCH 461/898] Remove node1 Due to LP #1874719 delete node1 as its not used. --- zaza/openstack/charm_tests/pacemaker_remote/tests.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zaza/openstack/charm_tests/pacemaker_remote/tests.py b/zaza/openstack/charm_tests/pacemaker_remote/tests.py index f0d0b4d..328d5cf 100644 --- a/zaza/openstack/charm_tests/pacemaker_remote/tests.py +++ b/zaza/openstack/charm_tests/pacemaker_remote/tests.py @@ -26,5 +26,8 @@ class PacemakerRemoteTest(unittest.TestCase): def test_check_nodes_online(self): """Test that all nodes are online.""" + zaza.openstack.configure.hacluster.remove_node( + 'api', + 'node1') self.assertTrue( zaza.openstack.configure.hacluster.check_all_nodes_online('api')) From 21655a3741f80afad43c138910e1ff3d5979535c Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Sat, 16 May 2020 18:52:35 +0100 Subject: [PATCH 462/898] Fix broken dictionary assignment for domain --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 47cabf3..3148940 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1600,7 +1600,7 @@ def get_undercloud_auth(): 'API_VERSION': 3, } if domain: - auth_settings['OS_DOMAIN_NAME': 'admin_domain'] = domain + auth_settings['OS_DOMAIN_NAME'] = domain else: auth_settings['OS_USER_DOMAIN_NAME'] = ( os.environ.get('OS_USER_DOMAIN_NAME')) From 215e5accef726ce74b4f92f78b65635a993b1a75 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Sun, 17 May 2020 21:19:14 +0100 Subject: [PATCH 463/898] Add retries to instance pinging It's a bit too optimistic to expect an instance to respond to the first ping. This patch gives the instance up to 8 retries with increasingly lengthened waits to respond to a ping. This should help with Juju storage backed nova instances. Fixes: #265 --- zaza/openstack/configure/guest.py | 28 ++++++++++++++++++++------ zaza/openstack/utilities/exceptions.py | 6 ++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/configure/guest.py b/zaza/openstack/configure/guest.py index bd70bca..c2e20de 100644 --- a/zaza/openstack/configure/guest.py +++ b/zaza/openstack/configure/guest.py @@ -22,6 +22,14 @@ import time import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.nova.utils as nova_utils +import zaza.openstack.utilities.exceptions as openstack_exceptions + +from tenacity import ( + RetryError, + Retrying, + stop_after_attempt, + wait_exponential, +) boot_tests = { 'cirros': { @@ -134,12 +142,20 @@ def launch_instance(instance_key, use_boot_volume=False, vm_name=None, port=port)['floating_ip_address'] logging.info('Assigned floating IP {} to {}'.format(ip, vm_name)) try: - openstack_utils.ping_response(ip) - except subprocess.CalledProcessError as e: - logging.error('Pinging {} failed with {}'.format(ip, e.returncode)) - logging.error('stdout: {}'.format(e.stdout)) - logging.error('stderr: {}'.format(e.stderr)) - raise + for attempt in Retrying( + stop=stop_after_attempt(8), + wait=wait_exponential(multiplier=1, min=2, max=60)): + with attempt: + try: + openstack_utils.ping_response(ip) + except subprocess.CalledProcessError as e: + logging.error('Pinging {} failed with {}' + .format(ip, e.returncode)) + logging.error('stdout: {}'.format(e.stdout)) + logging.error('stderr: {}'.format(e.stderr)) + raise + except RetryError: + raise openstack_exceptions.NovaGuestNoPingResponse() # Check ssh'ing to instance. logging.info('Testing ssh access.') diff --git a/zaza/openstack/utilities/exceptions.py b/zaza/openstack/utilities/exceptions.py index e9e5d23..ab4e258 100644 --- a/zaza/openstack/utilities/exceptions.py +++ b/zaza/openstack/utilities/exceptions.py @@ -180,6 +180,12 @@ class NovaGuestRestartFailed(Exception): pass +class NovaGuestNoPingResponse(Exception): + """Nova guest failed to respond to pings.""" + + pass + + class PolicydError(Exception): """Policyd override failed.""" From 5e5c8e488bda1ae2d50b1c9f01ed3fe42f6d02cd Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 18 May 2020 12:52:19 +0100 Subject: [PATCH 464/898] Add retries to teardown as it seems to be sensitive to network glitches. --- zaza/openstack/charm_tests/cinder/tests.py | 41 ++++++++++++++-------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 4a851ce..2648538 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -23,6 +23,13 @@ import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.glance.setup as glance_setup +from tenacity import ( + RetryError, + Retrying, + stop_after_attempt, + wait_exponential, +) + class CinderTests(test_utils.OpenStackBaseTest): """Encapsulate Cinder tests.""" @@ -45,23 +52,27 @@ class CinderTests(test_utils.OpenStackBaseTest): def tearDown(cls): """Remove test resources.""" logging.info('Running teardown') - volumes = list(cls.cinder_client.volumes.list()) - snapped_volumes = [v for v in volumes if v.name.endswith("-from-snap")] - if snapped_volumes: - logging.info("Removing volumes from snapshot") - cls._remove_volumes(snapped_volumes) - volumes = list(cls.cinder_client.volumes.list()) + for attempt in Retrying( + stop=stop_after_attempt(8), + wait=wait_exponential(multiplier=1, min=2, max=60)): + with attempt: + volumes = list(cls.cinder_client.volumes.list()) + snapped_volumes = [v for v in volumes if v.name.endswith("-from-snap")] + if snapped_volumes: + logging.info("Removing volumes from snapshot") + cls._remove_volumes(snapped_volumes) + volumes = list(cls.cinder_client.volumes.list()) - snapshots = list(cls.cinder_client.volume_snapshots.list()) - if snapshots: - logging.info("tearDown - snapshots: {}".format( - ", ".join(s.name for s in snapshots))) - cls._remove_snapshots(snapshots) + snapshots = list(cls.cinder_client.volume_snapshots.list()) + if snapshots: + logging.info("tearDown - snapshots: {}".format( + ", ".join(s.name for s in snapshots))) + cls._remove_snapshots(snapshots) - if volumes: - logging.info("tearDown - volumes: {}".format( - ", ".join(v.name for v in volumes))) - cls._remove_volumes(volumes) + if volumes: + logging.info("tearDown - volumes: {}".format( + ", ".join(v.name for v in volumes))) + cls._remove_volumes(volumes) @classmethod def _remove_snapshots(cls, snapshots): From 7834eda00ea5c303ad499b51913142fd0eb460c2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 18 May 2020 12:48:05 +0000 Subject: [PATCH 465/898] Switch to cirros image for ceph mirror tests The image used for the tests is just treated as a blob, no guests are booted using it so to speed up the tests when using slow storage by switching to using cirros image. The cirros image is ~20 times smaller than the bionic one. --- zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py index 16ebfc0..2e238ae 100644 --- a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py @@ -23,7 +23,9 @@ import zaza.model import zaza.openstack.utilities.ceph import zaza.openstack.utilities.openstack as openstack -from zaza.openstack.charm_tests.glance.setup import LTS_IMAGE_NAME +from zaza.openstack.charm_tests.glance.setup import ( + LTS_IMAGE_NAME, + CIRROS_IMAGE_NAME) class CephRBDMirrorBase(test_utils.OpenStackBaseTest): @@ -197,7 +199,12 @@ class CephRBDMirrorTest(CephRBDMirrorBase): glance = openstack.get_glance_session_client(session) cinder = openstack.get_cinder_session_client(session) - image = next(glance.images.list(name=LTS_IMAGE_NAME)) + image = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) + if not image: + logging.info("Failed to find {} image, falling back to {}".format( + CIRROS_IMAGE_NAME, + LTS_IMAGE_NAME)) + image = openstack.get_images_by_name(glance, LTS_IMAGE_NAME) # NOTE(fnordahl): for some reason create volume from image often fails # when run just after deployment is finished. We should figure out From 0ec2e43041ea52d26f0ba5f4e49e29997af22251 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 18 May 2020 18:25:09 +0000 Subject: [PATCH 466/898] Fix ceph rbd mirror test I recently broke the ceph rbd mirror test due to get_images_by_name returning a list of images not a single image. --- zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py index 2e238ae..6c2fa1b 100644 --- a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py @@ -199,12 +199,14 @@ class CephRBDMirrorTest(CephRBDMirrorBase): glance = openstack.get_glance_session_client(session) cinder = openstack.get_cinder_session_client(session) - image = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) - if not image: + images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) + if images: + image = images[0] + else: logging.info("Failed to find {} image, falling back to {}".format( CIRROS_IMAGE_NAME, LTS_IMAGE_NAME)) - image = openstack.get_images_by_name(glance, LTS_IMAGE_NAME) + image = openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] # NOTE(fnordahl): for some reason create volume from image often fails # when run just after deployment is finished. We should figure out From dfc62353620d0229fec14b8f4a05fcfd6332c34c Mon Sep 17 00:00:00 2001 From: Dmitrii Shcherbakov Date: Mon, 18 May 2020 19:58:16 +0300 Subject: [PATCH 467/898] Add cleanup to the unit removal case For some reason test cases are sometimes executed out of order and so test_921_remove_unit is sometimes run before the pause_and_resume test case which results in an error. While the root cause for it must be found it would also be good to avoid side-effects in individual test cases and return the environment back to its original state. There is no 'start' hook implementation for charm-rabbitmq-server, however, changes that close to the 20.05 release are discouraged so this change uses an upgrade-charm event simulation to re-trigger the addition of a unit (which was previously removed) to the cluster. NOTE: after an execution of hooks/upgrade-charm finishes, the charm will stay in the waiting state with the following status until the next update-status event: 'Unit has peers, but RabbitMQ not clustered' Related-Bug: #1730709 --- .../charm_tests/rabbitmq_server/tests.py | 58 ++++++++++++++----- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 43e9bc8..5220b3e 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -343,11 +343,14 @@ class RmqTests(test_utils.OpenStackBaseTest): def _retry_check_unit_cluster_nodes(self, u, unit_node_names): return rmq_utils.check_unit_cluster_nodes(u, unit_node_names) - def test_921_remove_unit(self): + def test_921_remove_and_add_unit(self): """Test if unit cleans up when removed from Rmq cluster. Test if a unit correctly cleans up by removing itself from the - RabbitMQ cluster on removal + RabbitMQ cluster on removal. + + Add the unit back to the cluster at the end of the test case to + avoid side-effects. """ logging.info('Checking that units correctly clean up after ' @@ -356,25 +359,48 @@ class RmqTests(test_utils.OpenStackBaseTest): zaza.model.set_application_config('rabbitmq-server', config) rmq_utils.wait_for_cluster() - units = zaza.model.get_units(self.application_name) - removed_unit = units[-1] - left_units = units[:-1] + all_units = zaza.model.get_units(self.application_name) + removed_unit = all_units[-1] + left_units = all_units[:-1] + logging.info('Simulating unit {} removal'.format(removed_unit)) zaza.model.run_on_unit(removed_unit.entity_id, 'hooks/stop') + logging.info('Waiting until unit {} reaches "waiting" state' + ''.format(removed_unit)) zaza.model.block_until_unit_wl_status(removed_unit.entity_id, "waiting") - unit_host_names = generic_utils.get_unit_hostnames(left_units) - unit_node_names = [] - for unit in unit_host_names: - unit_node_names.append('rabbit@{}'.format(unit_host_names[unit])) - errors = [] + def check_units(units): + unit_host_names = generic_utils.get_unit_hostnames(units) + unit_node_names = [] + for unit in unit_host_names: + unit_node_names.append('rabbit@{}'.format( + unit_host_names[unit])) + errors = [] - for u in left_units: - e = self._retry_check_unit_cluster_nodes(u, - unit_node_names) - if e: - errors.append(e) + for u in units: + e = self._retry_check_unit_cluster_nodes(u, + unit_node_names) + if e: + errors.append(e) + + self.assertFalse(errors, msg=errors) + + logging.info('Checking that all units except for {} are present' + 'in the cluster'.format(removed_unit)) + check_units(left_units) + + logging.info('Re-adding the removed unit {} back to the cluster' + 'by simulating the upgrade-charm event' + ''.format(removed_unit)) + # TODO(dmitriis): Fix the rabbitmq charm to add a proper way to add a + # unit back to the cluster and replace this. + zaza.model.run_on_unit(removed_unit.entity_id, 'hooks/upgrade-charm') + logging.info('Waiting until unit {} reaches "active" state' + ''.format(removed_unit)) + zaza.model.block_until_unit_wl_status(removed_unit.entity_id, + "active") + logging.info('Checking that all units are present in the cluster') + check_units(all_units) - self.assertFalse(errors, msg=errors) logging.info('OK') From 0dad4aca1480fdb85992a82def909cfe739b1a11 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 15 May 2020 15:29:51 +0200 Subject: [PATCH 468/898] Minor improvements to RmqTests.test_913_list_unconsumed_queues --- .../charm_tests/rabbitmq_server/tests.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 5220b3e..d0d2306 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -76,7 +76,9 @@ class RmqTests(test_utils.OpenStackBaseTest): rmq_utils.configure_ssl_off(units) # Publish and get amqp messages in all possible unit combinations. - # Qty of checks == (qty of units) ^ 2 + # Qty of checks == qty_of_units * (qty_of_units - 1) + assert len(units) >= 2, 'Test is useful only with 2 units or more.' + amqp_msg_counter = 1 host_names = generic_utils.get_unit_hostnames(units) @@ -311,8 +313,9 @@ class RmqTests(test_utils.OpenStackBaseTest): """Test rabbitmqctl list-unconsumed-queues action can be returned.""" logging.info('Checking list-unconsumed-queues action...') - unit = zaza.model.get_units(self.application_name)[0] - self._test_rmq_amqp_messages_all_units([unit]) + units = zaza.model.get_units(self.application_name) + self._test_rmq_amqp_messages_all_units(units) + unit = units[0] action = zaza.model.run_action(unit.entity_id, 'list-unconsumed-queues') self.assertIsInstance(action, juju.action.Action) @@ -332,7 +335,15 @@ class RmqTests(test_utils.OpenStackBaseTest): # Since we just reused _test_rmq_amqp_messages_all_units, we should # have created the queue if it didn't already exist, but all messages # should have already been consumed. - assert queue_data['messages'] == 0, 'Found unexpected message count.' + if queue_data['messages'] != 0: + logging.error( + '{} has {} remaining messages in {} instead of 0.'.format( + unit.entity_id, queue_data['messages'], + queue_data['name'])) + if queue_data['messages'] >= 1: + logging.error('One message is: {}'.format( + self._retry_get_amqp_message(unit))) + assert False, 'Found unexpected message count.' logging.info('OK') From c06cb8d62513bb5962a878e3d304066143c3b959 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 19 May 2020 07:47:55 -0700 Subject: [PATCH 469/898] MySQL 8 Scale-in and Scale-out tests (#275) * MySQL 8 Scale-in and Scale-out tests Depends-On: https://github.com/openstack-charmers/zaza/pull/353 * Review requests * Typo --- zaza/openstack/charm_tests/mysql/tests.py | 166 ++++++++++++++++++++-- 1 file changed, 155 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 77bc966..5088010 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -71,6 +71,45 @@ class MySQLBaseTest(test_utils.OpenStackBaseTest): self.non_leaders.append(unit) return self.leader, self.non_leaders + def get_cluster_status(self): + """Get cluster status. + + Return cluster status dict from the cluster-status action or raise + assertion error. + + :returns: Dictionary of cluster status + :rtype: dict + """ + logging.info("Running cluster-status action") + action = zaza.model.run_action_on_leader( + self.application, + "cluster-status", + action_params={}) + assert action.data.get("results") is not None, ( + "Cluster status action failed: No results: {}" + .format(action.data)) + assert action.data["results"].get("cluster-status") is not None, ( + "Cluster status action failed: No cluster-status: {}" + .format(action.data)) + return json.loads(action.data["results"]["cluster-status"]) + + def get_rw_primary_node(self): + """Get RW primary node. + + Return RW primary node unit. + + :returns: Unit object of primary node + :rtype: Union[Unit, None] + """ + _status = self.get_cluster_status() + _primary_ip = _status['groupInformationSourceMember'] + if ":" in _primary_ip: + _primary_ip = _primary_ip.split(':')[0] + units = zaza.model.get_units(self.application_name) + for unit in units: + if _primary_ip in unit.public_address: + return unit + class MySQLCommonTests(MySQLBaseTest): """Common mysql charm tests.""" @@ -412,14 +451,10 @@ class MySQLInnoDBClusterTests(MySQLCommonTests): Run the cluster-status action. """ logging.info("Execute cluster-status action") - action = zaza.model.run_action_on_leader( - self.application, - "cluster-status", - action_params={}) - cluster_status = json.loads(action.data["results"]["cluster-status"]) + cluster_status = self.get_cluster_status() assert "OK" in cluster_status["defaultReplicaSet"]["status"], ( - "Cluster status action failed: {}" - .format(action.data)) + "Cluster status is not OK: {}" + .format(cluster_status)) logging.info("Passed cluster-status action test.") def test_110_mysqldump(self): @@ -538,10 +573,15 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): logging.info("Execute reboot-cluster-from-complete-outage " "action after cold boot ...") - action = zaza.model.run_action_on_leader( - self.application, - "reboot-cluster-from-complete-outage", - action_params={}) + # We do not know which unit has the most up to date data + # run reboot-cluster-from-complete-outage until we get a success. + for unit in zaza.model.get_units(self.application): + action = zaza.model.run_action( + unit.entity_id, + "reboot-cluster-from-complete-outage", + action_params={}) + if action.data["results"].get("outcome"): + break assert "Success" in action.data["results"]["outcome"], ( "Reboot cluster from complete outage action failed: {}" .format(action.data)) @@ -670,3 +710,107 @@ class MySQL8MigrationTests(MySQLBaseTest): test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) + + +class MySQLInnoDBClusterScaleTest(MySQLBaseTest): + """Percona Cluster cold start tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running mysql-innodb-cluster scale tests.""" + super().setUpClass() + cls.application = "mysql-innodb-cluster" + cls.test_config = lifecycle_utils.get_charm_config(fatal=False) + cls.states = cls.test_config.get("target_deploy_status", {}) + + def test_800_remove_leader(self): + """Remove leader node. + + We start with a three node cluster, remove one, down to two. + The cluster will be in waiting state. + """ + logging.info("Scale in test: remove leader") + leader, nons = self.get_leaders_and_non_leaders() + leader_unit = zaza.model.get_unit_from_name(leader) + zaza.model.destroy_unit(self.application_name, leader) + + logging.info("Wait units are waiting ...") + zaza.model.block_until_unit_wl_status(nons[0], "waiting") + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info( + "Removing old unit from cluster: {} " + .format(leader_unit.public_address)) + action = zaza.model.run_action( + nons[0], + "remove-instance", + action_params={ + "address": leader_unit.public_address, + "force": True}) + assert action.data.get("results") is not None, ( + "Remove instance action failed: No results: {}" + .format(action.data)) + + def test_801_add_unit(self): + """Add mysql-innodb-cluster node. + + We start with two node cluster in waiting, add one, back to a full + cluster of three. + """ + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info("Adding unit after removed unit ...") + zaza.model.add_unit(self.application_name) + + logging.info("Wait for application states ...") + zaza.model.wait_for_application_states(states=self.states) + + def test_802_add_unit(self): + """Add another mysql-innodb-cluster node. + + We start with a three node full cluster, add another, up to a four node + cluster. + """ + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info("Adding unit after full cluster ...") + zaza.model.add_unit(self.application_name) + + logging.info("Wait for application states ...") + zaza.model.wait_for_application_states(states=self.states) + + def test_803_remove_fourth(self): + """Remove mysql-innodb-cluster node. + + We start with a four node full cluster, remove one, down to a three + node full cluster. + """ + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + leader, nons = self.get_leaders_and_non_leaders() + non_leader_unit = zaza.model.get_unit_from_name(nons[0]) + zaza.model.destroy_unit(self.application_name, nons[0]) + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info("Scale in test: back down to three") + zaza.model.wait_for_application_states(states=self.states) + + logging.info( + "Removing old unit from cluster: {} " + .format(non_leader_unit.public_address)) + action = zaza.model.run_action( + leader, + "remove-instance", + action_params={ + "address": non_leader_unit.public_address, + "force": True}) + assert action.data.get("results") is not None, ( + "Remove instance action failed: No results: {}" + .format(action.data)) From ec40306f7b4c94409d52ce9c6d8104ede0ee7d23 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 19 May 2020 20:08:33 +0100 Subject: [PATCH 470/898] Improve detection of removing and adding relation --- zaza/openstack/charm_tests/ceph/mon/tests.py | 66 ++++++++++++++++++-- zaza/openstack/charm_tests/cinder/tests.py | 4 +- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index 63c4033..4258a36 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -86,11 +86,19 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): logging.info("Removing ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.remove_relation( "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") - zaza.model.wait_for_agent_status() - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() + # zaza.model.wait_for_agent_status() + logging.info("Wait till relation is removed...") + ceph_mon_units = zaza.model.get_units("ceph-mon", + model_name=self.model_name) + conditions = [ + invert_condition( + does_relation_exist( + u.name, "ceph-mon", "cinder-ceph", "ceph", + self.model_name)) + for u in ceph_mon_units] + zaza.model.block_until(*conditions) - # check each unit after to breaking relation + logging.info("Checking each unit after breaking relation...") for unit in units: dir_list = directory_listing(unit.name, "/etc/ceph") if 'ceph.conf' not in dir_list: @@ -105,12 +113,60 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): logging.info("Restoring ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.add_relation( "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") - zaza.model.wait_for_agent_status() + conditions = [ + does_relation_exist( + u.name, "ceph-mon", "cinder-ceph", "ceph", self.model_name) + for u in ceph_mon_units] logging.info("Wait till model is idle ...") + zaza.model.block_until(*conditions) zaza.model.block_until_all_units_idle() logging.info("... Done.") +def does_relation_exist(unit_name, + application_name, + remote_application_name, + remote_interface_name, + model_name): + """For use in async blocking function, return True if it exists. + + :param unit_name: the unit (by name) that to check on. + :type unit_name: str + :param application_name: Name of application on this side of relation + :type application_name: str + :param remote_application_name: the relation name at that unit to check for + :type relation_application_name: str + :param remote_interface_name: the interface name at that unit to check for + :type relation_interface_name: str + :param model_name: the model to check on + :type model_name: str + :returns: Corouting that returns True if the relation was found + :rtype: Coroutine[[], boolean] + """ + async def _async_does_relation_exist_closure(): + async with zaza.model.run_in_model(model_name) as model: + spec = "{}:{}".format( + remote_application_name, remote_interface_name) + for rel in model.applications[application_name].relations: + if rel.matches(spec): + return True + return False + return _async_does_relation_exist_closure + + +def invert_condition(async_condition): + """Invert the condition provided so it can be provided to the blocking fn. + + :param async_condition: the async callable that is the test + :type async_condition: Callable[] + :returns: Corouting that returns not of the result of a the callable + :rtype: Coroutine[[], bool] + """ + async def _async_invert_condition_closure(): + return not(await async_condition()) + return _async_invert_condition_closure + + def run_commands(unit_name, commands): """Run commands on unit. diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 2648538..4cbf7c0 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -24,7 +24,6 @@ import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.glance.setup as glance_setup from tenacity import ( - RetryError, Retrying, stop_after_attempt, wait_exponential, @@ -57,7 +56,8 @@ class CinderTests(test_utils.OpenStackBaseTest): wait=wait_exponential(multiplier=1, min=2, max=60)): with attempt: volumes = list(cls.cinder_client.volumes.list()) - snapped_volumes = [v for v in volumes if v.name.endswith("-from-snap")] + snapped_volumes = [v for v in volumes + if v.name.endswith("-from-snap")] if snapped_volumes: logging.info("Removing volumes from snapshot") cls._remove_volumes(snapped_volumes) From 51bce56f9edcfe62671e6ba723624ad12496063b Mon Sep 17 00:00:00 2001 From: Dmitrii Shcherbakov Date: Wed, 20 May 2020 00:35:30 +0300 Subject: [PATCH 471/898] Skip test_921_remove_and_add_unit Due to test stability issues we need to skip it and work on it after release. --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index d0d2306..bbb6c3b 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -18,6 +18,7 @@ import json import logging import time import uuid +import unittest import juju import tenacity @@ -354,6 +355,10 @@ class RmqTests(test_utils.OpenStackBaseTest): def _retry_check_unit_cluster_nodes(self, u, unit_node_names): return rmq_utils.check_unit_cluster_nodes(u, unit_node_names) + @unittest.skip( + "Skipping as a significant rework is required, see" + "https://github.com/openstack-charmers/zaza-openstack-tests/issues/290" + ) def test_921_remove_and_add_unit(self): """Test if unit cleans up when removed from Rmq cluster. From 28924d29090c0ac2247fea6fb1a79a73227d8145 Mon Sep 17 00:00:00 2001 From: Marco Silva Date: Wed, 20 May 2020 13:05:43 +0100 Subject: [PATCH 472/898] Move mysqldump from MySQLInnoDBClusterTests to MySQLCommonTests Moved mysqldump function. Added check for percona-cluster application on the function, to avoid breaking checks on non-percona applications. --- zaza/openstack/charm_tests/mysql/tests.py | 30 +++++++---------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index fb1ac61..61c37e0 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -83,6 +83,13 @@ class MySQLCommonTests(MySQLBaseTest): _db = "keystone" _file_key = "mysqldump-file" logging.info("Execute mysqldump action") + # Need to change strict mode to be able to dump database + if self.application_name == "percona-cluster": + action = zaza.model.run_action_on_leader( + self.application_name, + "set-pxc-strict-mode", + action_params={"mode": "MASTER"}) + action = zaza.model.run_action_on_leader( self.application, "mysqldump", @@ -283,10 +290,9 @@ class PerconaClusterColdStartTest(PerconaClusterBaseTest): After bootstrapping a non-leader node, notify bootstrapped on the leader node. """ - _machines = list( + _machines = sorted( juju_utils.get_machine_uuids_for_application(self.application)) # Stop Nodes - _machines.sort() # Avoid hitting an update-status hook logging.debug("Wait till model is idle ...") zaza.model.block_until_all_units_idle() @@ -439,23 +445,6 @@ class MySQLInnoDBClusterTests(MySQLCommonTests): .format(action.data)) logging.info("Passed cluster-status action test.") - def test_110_mysqldump(self): - """Backup mysql. - - Run the mysqldump action. - """ - _db = "keystone" - _file_key = "mysqldump-file" - logging.info("Execute mysqldump action") - action = zaza.model.run_action_on_leader( - self.application, - "mysqldump", - action_params={"databases": _db}) - _results = action.data["results"] - assert _db in _results[_file_key], ( - "Mysqldump action failed: {}".format(action.data)) - logging.info("Passed mysqldump action test.") - def test_120_set_cluster_option(self): """Set cluster option. @@ -504,10 +493,9 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): After a cold start, reboot cluster from complete outage. """ - _machines = list( + _machines = sorted( juju_utils.get_machine_uuids_for_application(self.application)) # Stop Nodes - _machines.sort() # Avoid hitting an update-status hook logging.debug("Wait till model is idle ...") zaza.model.block_until_all_units_idle() From e347f0cb7a7e88eb959a90c21ddf9f891185126c Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 19 May 2020 09:19:44 -0700 Subject: [PATCH 473/898] Handle reboot_from_complete_outage for mysql Test updates to re-enable reboot_from_complete_outage testing on mysql-innodb-cluster. --- zaza/openstack/charm_tests/mysql/tests.py | 24 ++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 5088010..4ed3cd2 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -558,17 +558,24 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): self.resolve_update_status_errors() zaza.model.block_until_all_units_idle() - logging.debug("Wait for application states ...") + logging.debug("Clear error hooks after reboot ...") for unit in zaza.model.get_units(self.application): try: zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") except zaza.model.UnitError: self.resolve_update_status_errors() zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") - states = {self.application: { - "workload-status": "blocked", - "workload-status-message": - "MySQL InnoDB Cluster not healthy: None"}} + logging.debug("Wait for application states blocked ...") + states = { + self.application: { + "workload-status": "blocked", + "workload-status-message": + "MySQL InnoDB Cluster not healthy: None"}, + "mysql-router": { + "workload-status": "blocked", + "workload-status-message": + "Failed to connect to MySQL"}} + zaza.model.wait_for_application_states(states=states) logging.info("Execute reboot-cluster-from-complete-outage " @@ -580,8 +587,11 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): unit.entity_id, "reboot-cluster-from-complete-outage", action_params={}) - if action.data["results"].get("outcome"): + if "Success" in action.data["results"].get("outcome"): break + else: + loggging.info(action.data["results"].get("output")) + assert "Success" in action.data["results"]["outcome"], ( "Reboot cluster from complete outage action failed: {}" .format(action.data)) @@ -734,7 +744,7 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): leader_unit = zaza.model.get_unit_from_name(leader) zaza.model.destroy_unit(self.application_name, leader) - logging.info("Wait units are waiting ...") + logging.info("Wait until unit is in waiting state ...") zaza.model.block_until_unit_wl_status(nons[0], "waiting") logging.info("Wait till model is idle ...") From 4802e632bdf5c8ca7562bddc045e1c199a5b1e2d Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 20 May 2020 15:05:37 -0700 Subject: [PATCH 474/898] lint fix --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 4ed3cd2..44fc98f 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -590,7 +590,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): if "Success" in action.data["results"].get("outcome"): break else: - loggging.info(action.data["results"].get("output")) + logging.info(action.data["results"].get("output")) assert "Success" in action.data["results"]["outcome"], ( "Reboot cluster from complete outage action failed: {}" From c34f2b2a545ab79cb61ed5e9b43ff02e47f76163 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 26 May 2020 07:45:54 +0000 Subject: [PATCH 475/898] Enable guest restart test on ussuri As documented in the 20.05 release note the guest restart function of masakari is currently supported on ussuri+ so enable the test for bionic-ussuri+ --- zaza/openstack/charm_tests/masakari/tests.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/masakari/tests.py b/zaza/openstack/charm_tests/masakari/tests.py index 80390f5..c2a3d85 100644 --- a/zaza/openstack/charm_tests/masakari/tests.py +++ b/zaza/openstack/charm_tests/masakari/tests.py @@ -39,6 +39,7 @@ class MasakariTest(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run class setup for running tests.""" super(MasakariTest, cls).setUpClass() + cls.current_release = openstack_utils.get_os_release() cls.keystone_session = openstack_utils.get_overcloud_keystone_session() cls.model_name = zaza.model.get_juju_model() cls.nova_client = openstack_utils.get_nova_session_client( @@ -169,8 +170,12 @@ class MasakariTest(test_utils.OpenStackBaseTest): zaza.openstack.configure.masakari.enable_hosts() def test_instance_restart_on_fail(self): - """Test singlee guest crash and recovery.""" - raise unittest.SkipTest("Bug #1866638") + """Test single guest crash and recovery.""" + if self.current_release < openstack_utils.get_os_release( + 'bionic_ussuri'): + raise unittest.SkipTest( + "Not supported on {}. Bug #1866638".format( + self.current_release)) vm_name = 'zaza-test-instance-failover' vm = self.ensure_guest(vm_name) _, unit_name = self.get_guests_compute_info(vm_name) @@ -198,6 +203,6 @@ class MasakariTest(test_utils.OpenStackBaseTest): unit_name, vm.id, model_name=self.model_name) - logging.info('{} pid is now {}'.format(vm_name, guest_pid)) + logging.info('{} pid is now {}'.format(vm_name, new_guest_pid)) assert new_guest_pid and new_guest_pid != guest_pid, ( "Restart failed or never happened") From 924b95b47b219bfb4ab8d640a371b79e17287d8d Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 26 May 2020 14:23:16 +0100 Subject: [PATCH 476/898] Unpin juju (libjuju) from < 2.8 python-libjuju was broken at 2.8 and thus zaza and zaza-openstack-tests needed to be pinned to < 2.8. This patch releases that so that the latest versions of libjuju are used in testing. This may get pinned again before the next release window. --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1f685cf..fa0e460 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiounittest async_generator boto3 -juju<2.8.0 +juju juju_wait PyYAML<=4.2,>=3.0 flake8>=2.2.4 diff --git a/setup.py b/setup.py index 3e781da..6d08832 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ install_require = [ 'cryptography', 'hvac<0.7.0', 'jinja2', - 'juju<2.8.0', + 'juju', 'juju-wait', 'lxml', 'PyYAML', From 8434827f5c3849a5740bf1d90e6176d40e9c3318 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 27 May 2020 10:25:49 +0000 Subject: [PATCH 477/898] Add methods for creating pre-deploy certs --- zaza/openstack/configure/pre_deploy_certs.py | 76 ++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 zaza/openstack/configure/pre_deploy_certs.py diff --git a/zaza/openstack/configure/pre_deploy_certs.py b/zaza/openstack/configure/pre_deploy_certs.py new file mode 100644 index 0000000..11ec709 --- /dev/null +++ b/zaza/openstack/configure/pre_deploy_certs.py @@ -0,0 +1,76 @@ +"""Module to setup pre-deploy TLS certs.""" + +import ipaddress +import itertools +import base64 +import os + +import zaza.openstack.utilities.cert + +ISSUER_NAME = 'OSCI' + + +def set_cidr_certs(): + """Create certs and keys for deploy using IP SANS from CIDR. + + Create a certificate authority certificate and key. The CA cert and key + are then base 64 encoded and assigned to the OS_TEST_CAKEY and + OS_TEST_CACERT environment variables. + + Using the CA key a second certificate and key are generated. The new + certificate has a SAN entry for the first 2^11 IPs in the CIDR. + The cert and key are then base 64 encoded and assigned to the OS_TEST_KEY + and OS_TEST_CERT environment variables. + """ + (cakey, cacert) = zaza.openstack.utilities.cert.generate_cert( + ISSUER_NAME, + generate_ca=True) + os.environ['OS_TEST_CAKEY'] = base64.b64encode(cakey).decode() + os.environ['OS_TEST_CACERT'] = base64.b64encode(cacert).decode() + # We need to restrain the number of SubjectAlternativeNames we attempt to + # put # in the certificate. There is a hard limit for what length the sum + # of all extensions in the certificate can have. + # + # - 2^11 ought to be enough for anybody + alt_names = [] + for addr in itertools.islice( + ipaddress.IPv4Network(os.environ.get('OS_CIDR_EXT')), 2**11): + alt_names.append(str(addr)) + (key, cert) = zaza.openstack.utilities.cert.generate_cert( + '*.serverstack', + alternative_names=alt_names, + issuer_name=ISSUER_NAME, + signing_key=cakey) + os.environ['OS_TEST_KEY'] = base64.b64encode(key).decode() + os.environ['OS_TEST_CERT'] = base64.b64encode(cert).decode() + + +def set_certs_per_vips(): + """Create certs and keys for deploy using VIPS. + + Create a certificate authority certificate and key. The CA cert and key + are then base 64 encoded and assigned to the OS_TEST_CAKEY and + OS_TEST_CACERT environment variables. + + Using the CA key a certificate and key is generated for each VIP specified + via environment variables. eg if OS_VIP06=172.20.0.107 is set in the + environment then a cert with a SAN entry for 172.20.0.107 is generated. + The cert and key are then base 64 encoded and assigned to the OS_VIP06_KEY + and OS_VIP06_CERT environment variables. + """ + (cakey, cacert) = zaza.openstack.utilities.cert.generate_cert( + ISSUER_NAME, + generate_ca=True) + os.environ['OS_TEST_CAKEY'] = base64.b64encode(cakey).decode() + os.environ['OS_TEST_CACERT'] = base64.b64encode(cacert).decode() + for vip_name, vip_ip in os.environ.items(): + if vip_name.startswith('OS_VIP'): + (key, cert) = zaza.openstack.utilities.cert.generate_cert( + '*.serverstack', + alternative_names=[vip_ip], + issuer_name=ISSUER_NAME, + signing_key=cakey) + os.environ[ + '{}_KEY'.format(vip_name)] = base64.b64encode(key).decode() + os.environ[ + '{}_CERT'.format(vip_name)] = base64.b64encode(cert).decode() From 362be92006259f6dfaeb1c29b57c015772935446 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 28 May 2020 09:19:12 +0000 Subject: [PATCH 478/898] Retrieve ssl-ca from vault when using vault api If the ssl-{key,chain,ca} charm config option have been set than retrieve the ssl-ca from the vault charm and use it when making called to the vault api. --- zaza/openstack/charm_tests/vault/setup.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 21c3793..18f617d 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -14,6 +14,7 @@ """Run configuration phase.""" +import base64 import functools import requests import tempfile @@ -27,6 +28,22 @@ import zaza.openstack.utilities.generic import zaza.utilities.juju as juju_utils +def get_cacert_file(): + """Retrieve CA cert used for vault EP and write to file. + + :returns: Path to file with CA cert used for Vault EPs + :rtype: str + """ + cacert_file = None + vault_config = zaza.model.get_application_config('vault') + cacert_b64 = vault_config['ssl-ca']['value'] + if cacert_b64: + with tempfile.NamedTemporaryFile(mode='wb', delete=False) as fp: + fp.write(base64.b64decode(cacert_b64)) + cacert_file = fp.name + return cacert_file + + def basic_setup(cacert=None, unseal_and_authorize=False): """Run basic setup for vault tests. @@ -35,6 +52,7 @@ def basic_setup(cacert=None, unseal_and_authorize=False): :param unseal_and_authorize: Whether to unseal and authorize vault. :type unseal_and_authorize: bool """ + cacert = cacert or get_cacert_file() vault_svc = vault_utils.VaultFacade(cacert=cacert) if unseal_and_authorize: vault_svc.unseal() @@ -47,6 +65,7 @@ def basic_setup_and_unseal(cacert=None): :param cacert: Path to CA cert used for vaults api cert. :type cacert: str """ + cacert = cacert or get_cacert_file() vault_svc = vault_utils.VaultFacade(cacert=cacert) vault_svc.unseal() for unit in zaza.model.get_units('vault'): From 2f6d6be3ef2b360c1d0fd1f5d5bc9d2a6a12f89e Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 28 May 2020 09:35:25 +0000 Subject: [PATCH 479/898] Update doc strings --- zaza/openstack/charm_tests/vault/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 18f617d..0709162 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -29,9 +29,9 @@ import zaza.utilities.juju as juju_utils def get_cacert_file(): - """Retrieve CA cert used for vault EP and write to file. + """Retrieve CA cert used for vault endpoints and write to file. - :returns: Path to file with CA cert used for Vault EPs + :returns: Path to file with CA cert. :rtype: str """ cacert_file = None From 24306b8880a0009b5f9bc6ac89584e6973fb3aea Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 28 May 2020 10:32:07 +0000 Subject: [PATCH 480/898] Fix get_machines_for_applications for subordinates In the case where we have a subordinate charm, libjuju juju status on the unit returns an empty dictionary, therefore the existing None check would fail and the subordinate status would never get set. The result being subordinates not included in the returned Iterator. --- zaza/openstack/utilities/juju.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index 80dd69b..b3e54bc 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -96,7 +96,7 @@ def get_machines_for_application(application, model_name=None): # libjuju juju status no longer has units for subordinate charms # Use the application it is subordinate-to to find machines - if status.get("units") is None and status.get("subordinate-to"): + if not status.get("units") and status.get("subordinate-to"): status = get_application_status(status.get("subordinate-to")[0], model_name=model_name) From 7b9b81696720497284ec800296717f359a2fe85a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sat, 22 Feb 2020 19:16:03 +0000 Subject: [PATCH 481/898] Redumentary tempest support --- zaza/openstack/charm_tests/glance/setup.py | 16 +++ zaza/openstack/charm_tests/tempest/setup.py | 134 ++++++++++++++++++ .../tempest/templates/tempest_v3.py | 95 +++++++++++++ 3 files changed, 245 insertions(+) create mode 100644 zaza/openstack/charm_tests/tempest/setup.py create mode 100644 zaza/openstack/charm_tests/tempest/templates/tempest_v3.py diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index c992917..0a920c2 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -18,6 +18,7 @@ import logging import zaza.openstack.utilities.openstack as openstack_utils CIRROS_IMAGE_NAME = "cirros" +CIRROS_ALT_IMAGE_NAME = "cirros_alt" LTS_RELEASE = "bionic" LTS_IMAGE_NAME = "bionic" @@ -77,6 +78,21 @@ def add_cirros_image(glance_client=None, image_name=None): image_name=image_name) +def add_cirros_alt_image(glance_client=None, image_name=None): + """Add a cirros image to the current deployment. + + :param glance: Authenticated glanceclient + :type glance: glanceclient.Client + :param image_name: Label for the image in glance + :type image_name: str + """ + image_name = image_name or CIRROS_ALT_IMAGE_NAME + image_url = openstack_utils.find_cirros_image(arch='x86_64') + add_image(image_url, + glance_client=glance_client, + image_name=image_name) + + def add_lts_image(glance_client=None, image_name=None, release=None): """Add an Ubuntu LTS image to the current deployment. diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py new file mode 100644 index 0000000..c09e61f --- /dev/null +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -0,0 +1,134 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for configuring tempest.""" +import urllib.parse +import os + +import zaza.model +import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.charm_tests.tempest.templates.tempest_v3 as tempest_v3 + +SETUP_ENV_VARS = [ + 'OS_TEST_GATEWAY', + 'OS_TEST_CIDR_EXT', + 'OS_TEST_FIP_RANGE', + 'OS_TEST_NAMESERVER', + 'OS_TEST_CIDR_PRIV', + 'OS_TEST_SWIFT_IP', +] + + +def get_app_access_ip(application_name): + try: + app_config = zaza.model.get_application_config(application_name) + except KeyError: + return '' + vip = app_config.get("vip").get("value") + if vip: + ip = vip + else: + unit = zaza.model.get_units(application_name)[0] + ip = unit.public_address + return ip + + +def add_application_ips(ctxt): + ctxt['keystone'] = get_app_access_ip('keystone') + ctxt['dashboard'] = get_app_access_ip('openstack-dashboard') + ctxt['ncc'] = get_app_access_ip('nova-cloud-controller') + + +def add_neutron_config(ctxt, keystone_session): + neutron_client = openstack_utils.get_neutron_session_client( + keystone_session) + for net in neutron_client.list_networks()['networks']: + if net['name'] == 'ext_net': + ctxt['ext_net'] = net['id'] + break + for router in neutron_client.list_routers()['routers']: + if router['name'] == 'provider-router': + ctxt['provider_router_id'] = router['id'] + break + + +def add_glance_config(ctxt, keystone_session): + glance_client = openstack_utils.get_glance_session_client( + keystone_session) + image = openstack_utils.get_images_by_name( + glance_client, glance_setup.CIRROS_IMAGE_NAME) + image_alt = openstack_utils.get_images_by_name( + glance_client, glance_setup.CIRROS_ALT_IMAGE_NAME) + if image: + ctxt['image_id'] = image[0].id + if image_alt: + ctxt['image_alt_id'] = image_alt[0].id + + +def add_keystone_config(ctxt, keystone_session): + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + for domain in keystone_client.domains.list(): + if domain.name == 'admin_domain': + ctxt['default_domain_id'] = domain.id + break + + +def add_environment_var_config(ctxt): + for var in SETUP_ENV_VARS: + value = os.environ.get(var) + if value: + ctxt[var.lower()] = value + else: + raise ValueError( + ("Environment variables {} must all be set to run this" + " test").format(', '.join(SETUP_ENV_VARS))) + + +def add_access_protocol(ctxt): + overcloud_auth = openstack_utils.get_overcloud_auth() + ctxt['proto'] = urllib.parse.urlparse(overcloud_auth['OS_AUTH_URL']).scheme + + +def get_tempest_context(): + keystone_session = openstack_utils.get_overcloud_keystone_session() + ctxt = {} + add_application_ips(ctxt) + add_neutron_config(ctxt, keystone_session) + add_glance_config(ctxt, keystone_session) + add_keystone_config(ctxt, keystone_session) + add_environment_var_config(ctxt) + add_access_protocol(ctxt) + return ctxt + + +def render_tempest_config(target_file, ctxt, tempest_template): + with open(target_file, 'w') as f: + f.write(tempest_template.file_contents.format(**ctxt)) + + +def setup_tempest(tempest_template): + try: + os.mkdir('tempest') + except FileExistsError: + pass + render_tempest_config( + 'tempest/tempest.conf', + get_tempest_context(), + tempest_template) + + +def tempest_keystone_v3(): + setup_tempest(tempest_v3) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py new file mode 100644 index 0000000..9e4727a --- /dev/null +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -0,0 +1,95 @@ +file_contents = """ +[DEFAULT] +debug = true +use_stderr = false +log_file = tempest.log + +[auth] +test_accounts_file = accounts.yaml +default_credentials_domain_name = admin_domain +admin_username = admin +admin_project_name = admin +admin_password = openstack +admin_domain_name = admin_domain + +[compute] +image_ref = {image_id} +image_ref_alt = {image_alt_id} +flavor_ref = 7 +flavor_ref_alt = 8 +min_compute_nodes = 3 + +# TODO: review this as its release specific +# min_microversion = 2.2 +# max_microversion = latest + +[compute-feature-enabled] +console_output = true +resize = true +live_migration = true +block_migration_for_live_migration = true +attach_encrypted_volume = false + +[identity] +uri = {proto}://{keystone}:5000/v2.0 +uri_v3 = {proto}://{keystone}:5000/v3 +auth_version = v3 +admin_role = Admin +region = RegionOne +default_domain_id = {default_domain_id} +admin_domain_scope = true + +[identity-feature-enabled] +api_v2 = false +api_v3 = true + +[image] +http_image = http://{os_test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz + +[network] +project_network_cidr = {os_test_cidr_priv} +public_network_id = {ext_net} +dns_servers = {os_test_nameserver} +project_networks_reachable = false + +[network-feature-enabled] +ipv6 = false + +[orchestration] +stack_owner_role = Admin +instance_type = m1.small +keypair_name = testkey + +[oslo_concurrency] +lock_path = /tmp + +[scenario] +img_dir = /home/ubuntu/images +img_file = cirros-0.3.4-x86_64-disk.img +img_container_format = bare +img_disk_format = qcow2 + +[validation] +run_validation = true +image_ssh_user = cirros + +[service_available] +ceilometer = true +cinder = true +glance = true +heat = true +horizon = true +ironic = false +neutron = true +nova = true +sahara = false +swift = true +trove = false +zaqar = false + +[volume] +backend_names = cinder-ceph +storage_protocol = ceph + +[volume-feature-enabled] +backup = false""" From 1724e482a1b67a4bbf8b032d3a54782d6959b658 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sat, 22 Feb 2020 19:49:45 +0000 Subject: [PATCH 482/898] Tempest setup --- zaza/openstack/charm_tests/tempest/setup.py | 24 +++++++++++++++---- .../charm_tests/tempest/templates/accounts.py | 8 +++++++ 2 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 zaza/openstack/charm_tests/tempest/templates/accounts.py diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index c09e61f..4bd8c4f 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -15,11 +15,13 @@ """Code for configuring tempest.""" import urllib.parse import os +import subprocess import zaza.model import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.tempest.templates.tempest_v3 as tempest_v3 +import zaza.openstack.charm_tests.tempest.templates.accounts as accounts SETUP_ENV_VARS = [ 'OS_TEST_GATEWAY', @@ -119,16 +121,30 @@ def render_tempest_config(target_file, ctxt, tempest_template): f.write(tempest_template.file_contents.format(**ctxt)) -def setup_tempest(tempest_template): +def setup_tempest(tempest_template, accounts_template): try: - os.mkdir('tempest') + os.makedirs('tempest/etc/') except FileExistsError: pass render_tempest_config( - 'tempest/tempest.conf', + 'tempest/etc/tempest.conf', get_tempest_context(), tempest_template) + render_tempest_config( + 'tempest/etc/accounts.yaml', + get_tempest_context(), + accounts_template) def tempest_keystone_v3(): - setup_tempest(tempest_v3) + setup_tempest(tempest_v3, accounts) + + +def clone_tempest(): + if not os.path.isdir('tempest'): + subprocess.check_call( + [ + 'git', + 'clone', + 'https://opendev.org/openstack/tempest', + 'tempest']) diff --git a/zaza/openstack/charm_tests/tempest/templates/accounts.py b/zaza/openstack/charm_tests/tempest/templates/accounts.py new file mode 100644 index 0000000..3e5329d --- /dev/null +++ b/zaza/openstack/charm_tests/tempest/templates/accounts.py @@ -0,0 +1,8 @@ +file_contents = """ +- username: 'demo' + tenant_name: 'demo' + password: 'pass' +- username: 'alt_demo' + tenant_name: 'alt_demo' + password: 'secret' +""" From 6347bb707f17fa7ec542f53dc39213b300e05951 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 24 Feb 2020 10:34:48 +0000 Subject: [PATCH 483/898] More updates --- zaza/openstack/charm_tests/glance/setup.py | 16 ---- .../openstack/charm_tests/tempest/__init__.py | 15 ++++ zaza/openstack/charm_tests/tempest/setup.py | 81 ++++++++++++++++++- .../charm_tests/tempest/templates/__init__.py | 15 ++++ .../tempest/templates/tempest_v3.py | 14 ++-- zaza/openstack/charm_tests/tempest/tests.py | 14 ++++ 6 files changed, 128 insertions(+), 27 deletions(-) create mode 100644 zaza/openstack/charm_tests/tempest/__init__.py create mode 100644 zaza/openstack/charm_tests/tempest/templates/__init__.py create mode 100644 zaza/openstack/charm_tests/tempest/tests.py diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index 0a920c2..c992917 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -18,7 +18,6 @@ import logging import zaza.openstack.utilities.openstack as openstack_utils CIRROS_IMAGE_NAME = "cirros" -CIRROS_ALT_IMAGE_NAME = "cirros_alt" LTS_RELEASE = "bionic" LTS_IMAGE_NAME = "bionic" @@ -78,21 +77,6 @@ def add_cirros_image(glance_client=None, image_name=None): image_name=image_name) -def add_cirros_alt_image(glance_client=None, image_name=None): - """Add a cirros image to the current deployment. - - :param glance: Authenticated glanceclient - :type glance: glanceclient.Client - :param image_name: Label for the image in glance - :type image_name: str - """ - image_name = image_name or CIRROS_ALT_IMAGE_NAME - image_url = openstack_utils.find_cirros_image(arch='x86_64') - add_image(image_url, - glance_client=glance_client, - image_name=image_name) - - def add_lts_image(glance_client=None, image_name=None, release=None): """Add an Ubuntu LTS image to the current deployment. diff --git a/zaza/openstack/charm_tests/tempest/__init__.py b/zaza/openstack/charm_tests/tempest/__init__.py new file mode 100644 index 0000000..ed3be2f --- /dev/null +++ b/zaza/openstack/charm_tests/tempest/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and using tempest.""" diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 4bd8c4f..3f0a066 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -18,11 +18,15 @@ import os import subprocess import zaza.model +import zaza.utilities.deployment_env as deployment_env import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.tempest.templates.tempest_v3 as tempest_v3 import zaza.openstack.charm_tests.tempest.templates.accounts as accounts +import keystoneauth1 +import novaclient + SETUP_ENV_VARS = [ 'OS_TEST_GATEWAY', 'OS_TEST_CIDR_EXT', @@ -31,6 +35,9 @@ SETUP_ENV_VARS = [ 'OS_TEST_CIDR_PRIV', 'OS_TEST_SWIFT_IP', ] +TEMPEST_FLAVOR_NAME = 'm1.tempest' +TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest' +TEMPEST_CIRROS_ALT_IMAGE_NAME = 'cirros_alt' def get_app_access_ip(application_name): @@ -53,6 +60,16 @@ def add_application_ips(ctxt): ctxt['ncc'] = get_app_access_ip('nova-cloud-controller') +def add_nova_config(ctxt, keystone_session): + nova_client = openstack_utils.get_nova_session_client( + keystone_session) + for flavor in nova_client.flavors.list(): + if flavor.name == TEMPEST_FLAVOR_NAME: + ctxt['flavor_ref'] = flavor.id + if flavor.name == TEMPEST_ALT_FLAVOR_NAME: + ctxt['flavor_ref_alt'] = flavor.id + + def add_neutron_config(ctxt, keystone_session): neutron_client = openstack_utils.get_neutron_session_client( keystone_session) @@ -72,7 +89,7 @@ def add_glance_config(ctxt, keystone_session): image = openstack_utils.get_images_by_name( glance_client, glance_setup.CIRROS_IMAGE_NAME) image_alt = openstack_utils.get_images_by_name( - glance_client, glance_setup.CIRROS_ALT_IMAGE_NAME) + glance_client, TEMPEST_CIRROS_ALT_IMAGE_NAME) if image: ctxt['image_id'] = image[0].id if image_alt: @@ -89,8 +106,9 @@ def add_keystone_config(ctxt, keystone_session): def add_environment_var_config(ctxt): + deploy_env = deployment_env.get_deployment_context() for var in SETUP_ENV_VARS: - value = os.environ.get(var) + value = deploy_env.get(var) if value: ctxt[var.lower()] = value else: @@ -102,12 +120,19 @@ def add_environment_var_config(ctxt): def add_access_protocol(ctxt): overcloud_auth = openstack_utils.get_overcloud_auth() ctxt['proto'] = urllib.parse.urlparse(overcloud_auth['OS_AUTH_URL']).scheme + ctxt['admin_username'] = overcloud_auth['OS_USERNAME'] + ctxt['admin_password'] = overcloud_auth['OS_PASSWORD'] + ctxt['admin_project_name'] = overcloud_auth['OS_PROJECT_NAME'] + ctxt['admin_domain_name'] = overcloud_auth['OS_DOMAIN_NAME'] + ctxt['default_credentials_domain_name'] = overcloud_auth[ + 'OS_PROJECT_DOMAIN_NAME'] def get_tempest_context(): keystone_session = openstack_utils.get_overcloud_keystone_session() ctxt = {} add_application_ips(ctxt) + add_nova_config(ctxt, keystone_session) add_neutron_config(ctxt, keystone_session) add_glance_config(ctxt, keystone_session) add_keystone_config(ctxt, keystone_session) @@ -136,7 +161,7 @@ def setup_tempest(tempest_template, accounts_template): accounts_template) -def tempest_keystone_v3(): +def render_tempest_config_keystone_v3(): setup_tempest(tempest_v3, accounts) @@ -145,6 +170,54 @@ def clone_tempest(): subprocess.check_call( [ 'git', - 'clone', + 'clone', 'https://opendev.org/openstack/tempest', 'tempest']) + + +def add_cirros_alt_image(): + """Add a cirros image to the current deployment. + + :param glance: Authenticated glanceclient + :type glance: glanceclient.Client + :param image_name: Label for the image in glance + :type image_name: str + """ + image_url = openstack_utils.find_cirros_image(arch='x86_64') + glance_setup.add_image( + image_url, + glance_client=None, + image_name=TEMPEST_CIRROS_ALT_IMAGE_NAME) + + +def add_tempest_flavors(): + keystone_session = openstack_utils.get_overcloud_keystone_session() + nova_client = openstack_utils.get_nova_session_client( + keystone_session) + try: + nova_client.flavors.create( + name=TEMPEST_FLAVOR_NAME, + ram=256, + vcpus=1, + disk=1) + except novaclient.exceptions.Conflict: + pass + try: + nova_client.flavors.create( + name=TEMPEST_ALT_FLAVOR_NAME, + ram=512, + vcpus=1, + disk=1) + except novaclient.exceptions.Conflict: + pass + + +def add_tempest_roles(): + keystone_session = openstack_utils.get_overcloud_keystone_session() + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + for role_name in ['Member', 'ResellerAdmin']: + try: + keystone_client.roles.create('Member') + except keystoneauth1.exceptions.http.Conflict: + pass diff --git a/zaza/openstack/charm_tests/tempest/templates/__init__.py b/zaza/openstack/charm_tests/tempest/templates/__init__.py new file mode 100644 index 0000000..1400887 --- /dev/null +++ b/zaza/openstack/charm_tests/tempest/templates/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of templates for tempest.""" diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index 9e4727a..ea2d88d 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -6,17 +6,17 @@ log_file = tempest.log [auth] test_accounts_file = accounts.yaml -default_credentials_domain_name = admin_domain -admin_username = admin -admin_project_name = admin -admin_password = openstack -admin_domain_name = admin_domain +default_credentials_domain_name = {default_credentials_domain_name} +admin_username = {admin_username} +admin_project_name = {admin_project_name} +admin_password = {admin_password} +admin_domain_name = {admin_domain_name} [compute] image_ref = {image_id} image_ref_alt = {image_alt_id} -flavor_ref = 7 -flavor_ref_alt = 8 +flavor_ref = {flavor_ref} +flavor_ref_alt = {flavor_ref_alt} min_compute_nodes = 3 # TODO: review this as its release specific diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py new file mode 100644 index 0000000..86b4f5b --- /dev/null +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -0,0 +1,14 @@ +import zaza.charm_lifecycle.test +import tempest.cmd.main + + +class TempestTest(): + + test_runner = zaza.charm_lifecycle.test.DIRECT + + def run(self): + the_app = tempest.cmd.main.Main() + return the_app.run([ + 'run', + '--smoke', + '--config', 'tempest/etc/tempest.conf']) From e8624d3297e8f03b682dcb3403454dda5c298132 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 24 Feb 2020 15:23:16 +0000 Subject: [PATCH 484/898] Fix missing ResellerAdmin --- zaza/openstack/charm_tests/tempest/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 3f0a066..f1c773e 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -218,6 +218,6 @@ def add_tempest_roles(): keystone_session) for role_name in ['Member', 'ResellerAdmin']: try: - keystone_client.roles.create('Member') + keystone_client.roles.create(role_name) except keystoneauth1.exceptions.http.Conflict: pass From be3659c4a2ca13d3c0135cef954510285cf69964 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 3 Mar 2020 12:49:41 +0000 Subject: [PATCH 485/898] Add support for blacklist/whitelist etc --- zaza/openstack/charm_tests/tempest/tests.py | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 86b4f5b..8880756 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -1,11 +1,52 @@ +import os + +import zaza +import zaza.charm_lifecycle.utils import zaza.charm_lifecycle.test import tempest.cmd.main +import tempfile class TempestTest(): test_runner = zaza.charm_lifecycle.test.DIRECT + def run(self): + charm_config = zaza.charm_lifecycle.utils.get_charm_config() + tempest_options = ['run', '--config', 'tempest/etc/tempest.conf'] + for model_alias in zaza.model.get_juju_model_aliases().keys(): + tempest_test_key = model_alias + if model_alias == zaza.charm_lifecycle.utils.DEFAULT_MODEL_ALIAS: + tempest_test_key = 'default' + config = charm_config['tests_options']['tempest'][tempest_test_key] + if config.get('regex'): + tempest_options.extend(['--regex', config.get('regex')]) + if config.get('black-regex'): + tempest_options.extend(['--black-regex', config.get('black-regex')]) + with tempfile.TemporaryDirectory() as tmpdirname: + if config.get('whitelist'): + white_file = os.path.join(tmpdirname, 'white.cfg') + with open(white_file, 'w') as f: + f.write('\n'.join(config.get('whitelist'))) + f.write('\n') + tempest_options.extend(['--whitelist-file', white_file]) + if config.get('blacklist'): + black_file = os.path.join(tmpdirname, 'black.cfg') + with open(black_file, 'w') as f: + f.write('\n'.join(config.get('blacklist'))) + f.write('\n') + tempest_options.extend(['--blacklist-file', black_file]) + print(tempest_options) + the_app = tempest.cmd.main.Main() + _exec_tempest = the_app.run(tempest_options) + if not _exec_tempest: + return False + return True + +class TempestSmokeTest(): + + test_runner = zaza.charm_lifecycle.test.DIRECT + def run(self): the_app = tempest.cmd.main.Main() return the_app.run([ From 182cda90138b8f46396055285462fa6ce9e8d3bd Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 4 Mar 2020 02:08:01 +0000 Subject: [PATCH 486/898] Add smoke to TempestTest as blacklist/whitelist/black-regex will be useful with it. --- zaza/openstack/charm_tests/tempest/tests.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 8880756..67ba09b 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -19,6 +19,8 @@ class TempestTest(): if model_alias == zaza.charm_lifecycle.utils.DEFAULT_MODEL_ALIAS: tempest_test_key = 'default' config = charm_config['tests_options']['tempest'][tempest_test_key] + if config.get('smoke'): + tempest_options.extend(['--smoke']) if config.get('regex'): tempest_options.extend(['--regex', config.get('regex')]) if config.get('black-regex'): @@ -42,14 +44,3 @@ class TempestTest(): if not _exec_tempest: return False return True - -class TempestSmokeTest(): - - test_runner = zaza.charm_lifecycle.test.DIRECT - - def run(self): - the_app = tempest.cmd.main.Main() - return the_app.run([ - 'run', - '--smoke', - '--config', 'tempest/etc/tempest.conf']) From 456b08c032fddea41ab618292542d663fb2dacbf Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 4 Mar 2020 15:16:18 +0000 Subject: [PATCH 487/898] minor cleanup --- zaza/openstack/charm_tests/tempest/setup.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index f1c773e..d1d2ebe 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -168,11 +168,10 @@ def render_tempest_config_keystone_v3(): def clone_tempest(): if not os.path.isdir('tempest'): subprocess.check_call( - [ - 'git', - 'clone', - 'https://opendev.org/openstack/tempest', - 'tempest']) + ['git', + 'clone', + 'https://opendev.org/openstack/tempest', + 'tempest']) def add_cirros_alt_image(): From 4d8cd803ffb1663325e96294fc9c450e60dc25ef Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 4 Mar 2020 18:52:34 +0000 Subject: [PATCH 488/898] Convert list of (black-)regex's to a space-separated string --- zaza/openstack/charm_tests/tempest/tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 67ba09b..f621a8a 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -22,9 +22,13 @@ class TempestTest(): if config.get('smoke'): tempest_options.extend(['--smoke']) if config.get('regex'): - tempest_options.extend(['--regex', config.get('regex')]) + tempest_options.extend( + ['--regex', + ' '.join([reg for reg in config.get('regex')])]) if config.get('black-regex'): - tempest_options.extend(['--black-regex', config.get('black-regex')]) + tempest_options.extend( + ['--black-regex', + ' '.join([reg for reg in config.get('black-regex')])]) with tempfile.TemporaryDirectory() as tmpdirname: if config.get('whitelist'): white_file = os.path.join(tmpdirname, 'white.cfg') From 2eaa55bd51c078f82773d531d6be9431512c9439 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 4 Mar 2020 18:54:51 +0000 Subject: [PATCH 489/898] Drop git clone of tempest --- zaza/openstack/charm_tests/tempest/setup.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index d1d2ebe..09d0be9 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -165,15 +165,6 @@ def render_tempest_config_keystone_v3(): setup_tempest(tempest_v3, accounts) -def clone_tempest(): - if not os.path.isdir('tempest'): - subprocess.check_call( - ['git', - 'clone', - 'https://opendev.org/openstack/tempest', - 'tempest']) - - def add_cirros_alt_image(): """Add a cirros image to the current deployment. From a8a6e72ef30e4786246c4c2ae9ee786cafd98057 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 5 Mar 2020 14:24:55 +0000 Subject: [PATCH 490/898] Use tempest workspace to avoid conflicts with any charm unit test .stestr config --- zaza/openstack/charm_tests/tempest/setup.py | 6 +++--- zaza/openstack/charm_tests/tempest/tests.py | 10 +++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 09d0be9..cc344ba 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -148,15 +148,15 @@ def render_tempest_config(target_file, ctxt, tempest_template): def setup_tempest(tempest_template, accounts_template): try: - os.makedirs('tempest/etc/') + os.makedirs('tempest_workspace/etc/') except FileExistsError: pass render_tempest_config( - 'tempest/etc/tempest.conf', + 'tempest_workspace/etc/tempest.conf', get_tempest_context(), tempest_template) render_tempest_config( - 'tempest/etc/accounts.yaml', + 'tempest_workspace/etc/accounts.yaml', get_tempest_context(), accounts_template) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index f621a8a..58a31eb 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -12,8 +12,16 @@ class TempestTest(): test_runner = zaza.charm_lifecycle.test.DIRECT def run(self): + tempest_options = ['init', 'tempest_workspace'] + the_app = tempest.cmd.main.Main() + print(tempest_options) + _exec_tempest = the_app.run(tempest_options) + #if not _exec_tempest: + # return False + charm_config = zaza.charm_lifecycle.utils.get_charm_config() - tempest_options = ['run', '--config', 'tempest/etc/tempest.conf'] + tempest_options = ['run', '--config', 'tempest_workspace/etc/tempest.conf', + '--workspace', 'tempest_workspace'] for model_alias in zaza.model.get_juju_model_aliases().keys(): tempest_test_key = model_alias if model_alias == zaza.charm_lifecycle.utils.DEFAULT_MODEL_ALIAS: From 57594a466bfc19f42bfd23e34390639c26940363 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 5 Mar 2020 20:21:39 +0000 Subject: [PATCH 491/898] Fix the_app.run ret code checks and don't recreate workspace if already exists --- zaza/openstack/charm_tests/tempest/tests.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 58a31eb..10854e4 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -7,20 +7,24 @@ import tempest.cmd.main import tempfile + class TempestTest(): test_runner = zaza.charm_lifecycle.test.DIRECT def run(self): - tempest_options = ['init', 'tempest_workspace'] - the_app = tempest.cmd.main.Main() - print(tempest_options) - _exec_tempest = the_app.run(tempest_options) - #if not _exec_tempest: - # return False + tempest_workspace = 'tempst_workspace' + tempest_options = ['init', tempest_workspace] + if not os.path.isdir(tempest_workspace): + the_app = tempest.cmd.main.Main() + print(tempest_options) + _exec_tempest = the_app.run(tempest_options) + if _exec_tempest != 0: + return False charm_config = zaza.charm_lifecycle.utils.get_charm_config() - tempest_options = ['run', '--config', 'tempest_workspace/etc/tempest.conf', + tempest_options = ['run', '--config', + os.path.join(tempest_workspace, 'etc/tempest.conf'), '--workspace', 'tempest_workspace'] for model_alias in zaza.model.get_juju_model_aliases().keys(): tempest_test_key = model_alias @@ -53,6 +57,6 @@ class TempestTest(): print(tempest_options) the_app = tempest.cmd.main.Main() _exec_tempest = the_app.run(tempest_options) - if not _exec_tempest: + if _exec_tempest != 0: return False return True From eff595c6ee7d50adb8702908ec979889d33f6d82 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 9 Mar 2020 13:54:04 +0000 Subject: [PATCH 492/898] Add render_tempest_config_keystone_v2 for older deployments --- zaza/openstack/charm_tests/tempest/setup.py | 5 + .../tempest/templates/tempest_v2.py | 94 +++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 zaza/openstack/charm_tests/tempest/templates/tempest_v2.py diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index cc344ba..2fd95f4 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -21,6 +21,7 @@ import zaza.model import zaza.utilities.deployment_env as deployment_env import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.charm_tests.tempest.templates.tempest_v2 as tempest_v2 import zaza.openstack.charm_tests.tempest.templates.tempest_v3 as tempest_v3 import zaza.openstack.charm_tests.tempest.templates.accounts as accounts @@ -161,6 +162,10 @@ def setup_tempest(tempest_template, accounts_template): accounts_template) +def render_tempest_config_keystone_v2(): + setup_tempest(tempest_v2, accounts) + + def render_tempest_config_keystone_v3(): setup_tempest(tempest_v3, accounts) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py new file mode 100644 index 0000000..3421910 --- /dev/null +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py @@ -0,0 +1,94 @@ +file_contents = """ +[DEFAULT] +debug = true +use_stderr = false +log_file = tempest.log + +[auth] +test_accounts_file = accounts.yaml +default_credentials_domain_name = {default_credentials_domain_name} +admin_username = {admin_username} +admin_project_name = {admin_project_name} +admin_password = {admin_password} +admin_domain_name = {admin_domain_name} + +[compute] +image_ref = {image_id} +image_ref_alt = {image_alt_id} +flavor_ref = {flavor_ref} +flavor_ref_alt = {flavor_ref_alt} +min_compute_nodes = 3 + +# TODO: review this as its release specific +# min_microversion = 2.2 +# max_microversion = latest + +[compute-feature-enabled] +console_output = true +resize = true +live_migration = true +block_migration_for_live_migration = true +attach_encrypted_volume = false + +[identity] +uri = {proto}://{keystone}:5000/v2.0 +auth_version = v2 +admin_role = Admin +region = RegionOne +default_domain_id = {default_domain_id} +admin_domain_scope = true + +[identity-feature-enabled] +api_v2 = true +api_v3 = false + +[image] +http_image = http://{os_test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz + +[network] +project_network_cidr = {os_test_cidr_priv} +public_network_id = {ext_net} +dns_servers = {os_test_nameserver} +project_networks_reachable = false + +[network-feature-enabled] +ipv6 = false + +[orchestration] +stack_owner_role = Admin +instance_type = m1.small +keypair_name = testkey + +[oslo_concurrency] +lock_path = /tmp + +[scenario] +img_dir = /home/ubuntu/images +img_file = cirros-0.3.4-x86_64-disk.img +img_container_format = bare +img_disk_format = qcow2 + +[validation] +run_validation = true +image_ssh_user = cirros + +[service_available] +ceilometer = true +cinder = true +glance = true +heat = true +horizon = true +ironic = false +neutron = true +nova = true +sahara = false +swift = true +trove = false +zaqar = false + +[volume] +backend_names = cinder-ceph +storage_protocol = ceph + +[volume-feature-enabled] +backup = false""" From f017934c9b890a7e604107b17f63701e7a0753c0 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 9 Mar 2020 15:57:03 +0000 Subject: [PATCH 493/898] Fix typo in tempest workspace name --- zaza/openstack/charm_tests/tempest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 10854e4..1626dc5 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -13,7 +13,7 @@ class TempestTest(): test_runner = zaza.charm_lifecycle.test.DIRECT def run(self): - tempest_workspace = 'tempst_workspace' + tempest_workspace = 'tempest_workspace' tempest_options = ['init', tempest_workspace] if not os.path.isdir(tempest_workspace): the_app = tempest.cmd.main.Main() From 67de027d986df6c20fe6eeac7da653ebb49b8576 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 10 Mar 2020 20:53:40 +0000 Subject: [PATCH 494/898] Move tempest init to setup.py and use a local workspace path --- zaza/openstack/charm_tests/tempest/setup.py | 17 +++++++++++------ zaza/openstack/charm_tests/tempest/tests.py | 11 ++--------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 2fd95f4..9ffe663 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -15,6 +15,7 @@ """Code for configuring tempest.""" import urllib.parse import os +import shutil import subprocess import zaza.model @@ -24,6 +25,7 @@ import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.tempest.templates.tempest_v2 as tempest_v2 import zaza.openstack.charm_tests.tempest.templates.tempest_v3 as tempest_v3 import zaza.openstack.charm_tests.tempest.templates.accounts as accounts +import tempest.cmd.main import keystoneauth1 import novaclient @@ -148,16 +150,18 @@ def render_tempest_config(target_file, ctxt, tempest_template): def setup_tempest(tempest_template, accounts_template): - try: - os.makedirs('tempest_workspace/etc/') - except FileExistsError: - pass + tempest_workspace = 'tempest_workspace' + the_app = tempest.cmd.main.Main() + tempest_options = ['init', '--workspace-path', './.tempest/workspace.yaml', + tempest_workspace] + print(tempest_options) + _exec_tempest = the_app.run(tempest_options) render_tempest_config( - 'tempest_workspace/etc/tempest.conf', + os.path.join(tempest_workspace, 'etc/tempest.conf'), get_tempest_context(), tempest_template) render_tempest_config( - 'tempest_workspace/etc/accounts.yaml', + os.path.join(tempest_workspace, 'etc/accounts.yaml'), get_tempest_context(), accounts_template) @@ -216,3 +220,4 @@ def add_tempest_roles(): keystone_client.roles.create(role_name) except keystoneauth1.exceptions.http.Conflict: pass + diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 1626dc5..a4f97e4 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -13,18 +13,11 @@ class TempestTest(): test_runner = zaza.charm_lifecycle.test.DIRECT def run(self): - tempest_workspace = 'tempest_workspace' - tempest_options = ['init', tempest_workspace] - if not os.path.isdir(tempest_workspace): - the_app = tempest.cmd.main.Main() - print(tempest_options) - _exec_tempest = the_app.run(tempest_options) - if _exec_tempest != 0: - return False - charm_config = zaza.charm_lifecycle.utils.get_charm_config() + tempest_workspace = 'tempest_workspace' tempest_options = ['run', '--config', os.path.join(tempest_workspace, 'etc/tempest.conf'), + '--workspace-path', './.tempest/workspace.yaml', '--workspace', 'tempest_workspace'] for model_alias in zaza.model.get_juju_model_aliases().keys(): tempest_test_key = model_alias From a1f34e2c0148fafd3686752db37d088ca737dca2 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 11 Mar 2020 14:22:27 +0000 Subject: [PATCH 495/898] Handle tempest changing to workspace directory --- zaza/openstack/charm_tests/tempest/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index a4f97e4..0641520 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -49,7 +49,9 @@ class TempestTest(): tempest_options.extend(['--blacklist-file', black_file]) print(tempest_options) the_app = tempest.cmd.main.Main() + project_root = os.getcwd() _exec_tempest = the_app.run(tempest_options) + os.chdir(project_root) if _exec_tempest != 0: return False return True From 4a745e3ab0848927e2f87bafad53289faf2b535a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 12 Mar 2020 12:58:21 +0000 Subject: [PATCH 496/898] Align tempest_v2.py with o-c-t v2 tempest.conf.template --- .../charm_tests/tempest/templates/tempest_v2.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py index 3421910..4f40d90 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py @@ -6,17 +6,18 @@ log_file = tempest.log [auth] test_accounts_file = accounts.yaml -default_credentials_domain_name = {default_credentials_domain_name} +default_credentials_domain_name = Default admin_username = {admin_username} -admin_project_name = {admin_project_name} +admin_project_name = admin admin_password = {admin_password} -admin_domain_name = {admin_domain_name} +admin_domain_name = Default [compute] image_ref = {image_id} image_ref_alt = {image_alt_id} flavor_ref = {flavor_ref} flavor_ref_alt = {flavor_ref_alt} +region = RegionOne min_compute_nodes = 3 # TODO: review this as its release specific @@ -35,8 +36,6 @@ uri = {proto}://{keystone}:5000/v2.0 auth_version = v2 admin_role = Admin region = RegionOne -default_domain_id = {default_domain_id} -admin_domain_scope = true [identity-feature-enabled] api_v2 = true @@ -89,6 +88,10 @@ zaqar = false [volume] backend_names = cinder-ceph storage_protocol = ceph +# NOTE(coreycb): Need to enalbe catalog_type, determined by: +# volume_version=$(openstack endpoint list -c 'Service Type' -f value | grep -m 1 volumev3 || +# openstack endpoint list -c 'Service Type' -f value | grep -m 1 volumev2) +# catalog_type = __VOLUME_VERSION__ [volume-feature-enabled] backup = false""" From aa33797879fc20143dc471f11ef3609b38d18470 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 12 Mar 2020 13:32:21 +0000 Subject: [PATCH 497/898] Updates to add_access_protocol for keystone v2 support --- zaza/openstack/charm_tests/tempest/setup.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 9ffe663..6d7c26e 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -125,10 +125,14 @@ def add_access_protocol(ctxt): ctxt['proto'] = urllib.parse.urlparse(overcloud_auth['OS_AUTH_URL']).scheme ctxt['admin_username'] = overcloud_auth['OS_USERNAME'] ctxt['admin_password'] = overcloud_auth['OS_PASSWORD'] - ctxt['admin_project_name'] = overcloud_auth['OS_PROJECT_NAME'] - ctxt['admin_domain_name'] = overcloud_auth['OS_DOMAIN_NAME'] - ctxt['default_credentials_domain_name'] = overcloud_auth[ - 'OS_PROJECT_DOMAIN_NAME'] + if overcloud_auth['API_VERSION'] == 3: + ctxt['admin_project_name'] = overcloud_auth['OS_PROJECT_NAME'] + ctxt['admin_domain_name'] = overcloud_auth['OS_DOMAIN_NAME'] + ctxt['default_credentials_domain_name'] = overcloud_auth[ + 'OS_PROJECT_DOMAIN_NAME'] + elif overcloud_auth['API_VERSION'] == 2: + #ctxt['admin_tenant_name'] = overcloud_auth['OS_TENANT_NAME'] + pass def get_tempest_context(): From af2cff3c6ccce4b6dfe605813300fdda3d36e5c8 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 13 Mar 2020 14:20:23 +0000 Subject: [PATCH 498/898] Set tempest debug to false --- zaza/openstack/charm_tests/tempest/templates/tempest_v2.py | 2 +- zaza/openstack/charm_tests/tempest/templates/tempest_v3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py index 4f40d90..ec95577 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py @@ -1,6 +1,6 @@ file_contents = """ [DEFAULT] -debug = true +debug = false use_stderr = false log_file = tempest.log diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index ea2d88d..887b0b5 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -1,6 +1,6 @@ file_contents = """ [DEFAULT] -debug = true +debug = false use_stderr = false log_file = tempest.log From 82a1f1b75de8456d13fa32d405550cd98ccfd96d Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 13 Mar 2020 17:35:26 +0000 Subject: [PATCH 499/898] Fix tempest run config-file flag --- zaza/openstack/charm_tests/tempest/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 0641520..27e12ed 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -15,10 +15,10 @@ class TempestTest(): def run(self): charm_config = zaza.charm_lifecycle.utils.get_charm_config() tempest_workspace = 'tempest_workspace' - tempest_options = ['run', '--config', + tempest_options = ['run', '--config-file', os.path.join(tempest_workspace, 'etc/tempest.conf'), '--workspace-path', './.tempest/workspace.yaml', - '--workspace', 'tempest_workspace'] + '--workspace', tempest_workspace] for model_alias in zaza.model.get_juju_model_aliases().keys(): tempest_test_key = model_alias if model_alias == zaza.charm_lifecycle.utils.DEFAULT_MODEL_ALIAS: From 2b9de3a2b7dab0d53b5e91a11daa986f63f45f0e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 13 Mar 2020 20:47:54 +0000 Subject: [PATCH 500/898] Updates to tempest config rendering and init --- zaza/openstack/charm_tests/tempest/setup.py | 28 +++++++++++++++++---- zaza/openstack/charm_tests/tempest/tests.py | 13 +++++++--- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 6d7c26e..04d4bf9 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -154,18 +154,36 @@ def render_tempest_config(target_file, ctxt, tempest_template): def setup_tempest(tempest_template, accounts_template): - tempest_workspace = 'tempest_workspace' + config_dir = '.tempest' + config_etc_dir = os.path.join(config_dir, 'etc') + config_etc_tempest = os.path.join(config_etc_dir, 'tempest.conf') + config_workspace_yaml = os.path.join(config_dir, 'workspace.yaml') + workspace_name = 'workspace' + workspace_dir = os.path.join(config_dir, workspace_name) + workspace_etc_dir = os.path.join(workspace_dir, 'etc') + workspace_etc_accounts = os.path.join(workspace_etc_dir, 'accounts.yaml') + workspace_etc_tempest = os.path.join(workspace_etc_dir, 'tempest.conf') the_app = tempest.cmd.main.Main() - tempest_options = ['init', '--workspace-path', './.tempest/workspace.yaml', - tempest_workspace] + + # note sure this is needed or not + if not os.path.isdir(config_dir): + os.mkdir(config_dir) + os.mkdir(config_etc_dir) + render_tempest_config( + config_etc_tempest, + get_tempest_context(), + tempest_template) + tempest_options = ['init', '--workspace-path', config_workspace_yaml, + '--name', workspace_name, workspace_dir] print(tempest_options) _exec_tempest = the_app.run(tempest_options) + # This was mising /etc/tempest/ and just going to /etc/ render_tempest_config( - os.path.join(tempest_workspace, 'etc/tempest.conf'), + workspace_etc_tempest, get_tempest_context(), tempest_template) render_tempest_config( - os.path.join(tempest_workspace, 'etc/accounts.yaml'), + workspace_etc_accounts, get_tempest_context(), accounts_template) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 27e12ed..9bc6221 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -14,11 +14,16 @@ class TempestTest(): def run(self): charm_config = zaza.charm_lifecycle.utils.get_charm_config() - tempest_workspace = 'tempest_workspace' + config_dir = '.tempest' + config_workspace_yaml = os.path.join(config_dir, 'workspace.yaml') + workspace_name = 'workspace' + workspace_dir = os.path.join(config_dir, workspace_name) + workspace_etc_dir = os.path.join(workspace_dir, 'etc') + workspace_etc_tempest = os.path.join(workspace_etc_dir, 'tempest.conf') tempest_options = ['run', '--config-file', - os.path.join(tempest_workspace, 'etc/tempest.conf'), - '--workspace-path', './.tempest/workspace.yaml', - '--workspace', tempest_workspace] + workspace_etc_tempest, + '--workspace-path', config_workspace_yaml, + '--workspace', workspace_name] for model_alias in zaza.model.get_juju_model_aliases().keys(): tempest_test_key = model_alias if model_alias == zaza.charm_lifecycle.utils.DEFAULT_MODEL_ALIAS: From eaca793e522c4d121b074409e5ae61d9a7428721 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 16 Mar 2020 13:56:29 +0000 Subject: [PATCH 501/898] tmp debug testing --- zaza/openstack/charm_tests/tempest/setup.py | 14 +-- .../tempest/templates/tempest_v2.py | 96 +------------------ .../tempest/templates/tempest_v3.py | 94 +----------------- 3 files changed, 9 insertions(+), 195 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 04d4bf9..7336299 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -138,13 +138,13 @@ def add_access_protocol(ctxt): def get_tempest_context(): keystone_session = openstack_utils.get_overcloud_keystone_session() ctxt = {} - add_application_ips(ctxt) - add_nova_config(ctxt, keystone_session) - add_neutron_config(ctxt, keystone_session) - add_glance_config(ctxt, keystone_session) - add_keystone_config(ctxt, keystone_session) - add_environment_var_config(ctxt) - add_access_protocol(ctxt) + #add_application_ips(ctxt) + #add_nova_config(ctxt, keystone_session) + #add_neutron_config(ctxt, keystone_session) + #add_glance_config(ctxt, keystone_session) + #add_keystone_config(ctxt, keystone_session) + #add_environment_var_config(ctxt) + #add_access_protocol(ctxt) return ctxt diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py index ec95577..aa6b555 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py @@ -1,97 +1,3 @@ file_contents = """ [DEFAULT] -debug = false -use_stderr = false -log_file = tempest.log - -[auth] -test_accounts_file = accounts.yaml -default_credentials_domain_name = Default -admin_username = {admin_username} -admin_project_name = admin -admin_password = {admin_password} -admin_domain_name = Default - -[compute] -image_ref = {image_id} -image_ref_alt = {image_alt_id} -flavor_ref = {flavor_ref} -flavor_ref_alt = {flavor_ref_alt} -region = RegionOne -min_compute_nodes = 3 - -# TODO: review this as its release specific -# min_microversion = 2.2 -# max_microversion = latest - -[compute-feature-enabled] -console_output = true -resize = true -live_migration = true -block_migration_for_live_migration = true -attach_encrypted_volume = false - -[identity] -uri = {proto}://{keystone}:5000/v2.0 -auth_version = v2 -admin_role = Admin -region = RegionOne - -[identity-feature-enabled] -api_v2 = true -api_v3 = false - -[image] -http_image = http://{os_test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz - -[network] -project_network_cidr = {os_test_cidr_priv} -public_network_id = {ext_net} -dns_servers = {os_test_nameserver} -project_networks_reachable = false - -[network-feature-enabled] -ipv6 = false - -[orchestration] -stack_owner_role = Admin -instance_type = m1.small -keypair_name = testkey - -[oslo_concurrency] -lock_path = /tmp - -[scenario] -img_dir = /home/ubuntu/images -img_file = cirros-0.3.4-x86_64-disk.img -img_container_format = bare -img_disk_format = qcow2 - -[validation] -run_validation = true -image_ssh_user = cirros - -[service_available] -ceilometer = true -cinder = true -glance = true -heat = true -horizon = true -ironic = false -neutron = true -nova = true -sahara = false -swift = true -trove = false -zaqar = false - -[volume] -backend_names = cinder-ceph -storage_protocol = ceph -# NOTE(coreycb): Need to enalbe catalog_type, determined by: -# volume_version=$(openstack endpoint list -c 'Service Type' -f value | grep -m 1 volumev3 || -# openstack endpoint list -c 'Service Type' -f value | grep -m 1 volumev2) -# catalog_type = __VOLUME_VERSION__ - -[volume-feature-enabled] -backup = false""" +debug = false""" diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index 887b0b5..aa6b555 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -1,95 +1,3 @@ file_contents = """ [DEFAULT] -debug = false -use_stderr = false -log_file = tempest.log - -[auth] -test_accounts_file = accounts.yaml -default_credentials_domain_name = {default_credentials_domain_name} -admin_username = {admin_username} -admin_project_name = {admin_project_name} -admin_password = {admin_password} -admin_domain_name = {admin_domain_name} - -[compute] -image_ref = {image_id} -image_ref_alt = {image_alt_id} -flavor_ref = {flavor_ref} -flavor_ref_alt = {flavor_ref_alt} -min_compute_nodes = 3 - -# TODO: review this as its release specific -# min_microversion = 2.2 -# max_microversion = latest - -[compute-feature-enabled] -console_output = true -resize = true -live_migration = true -block_migration_for_live_migration = true -attach_encrypted_volume = false - -[identity] -uri = {proto}://{keystone}:5000/v2.0 -uri_v3 = {proto}://{keystone}:5000/v3 -auth_version = v3 -admin_role = Admin -region = RegionOne -default_domain_id = {default_domain_id} -admin_domain_scope = true - -[identity-feature-enabled] -api_v2 = false -api_v3 = true - -[image] -http_image = http://{os_test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz - -[network] -project_network_cidr = {os_test_cidr_priv} -public_network_id = {ext_net} -dns_servers = {os_test_nameserver} -project_networks_reachable = false - -[network-feature-enabled] -ipv6 = false - -[orchestration] -stack_owner_role = Admin -instance_type = m1.small -keypair_name = testkey - -[oslo_concurrency] -lock_path = /tmp - -[scenario] -img_dir = /home/ubuntu/images -img_file = cirros-0.3.4-x86_64-disk.img -img_container_format = bare -img_disk_format = qcow2 - -[validation] -run_validation = true -image_ssh_user = cirros - -[service_available] -ceilometer = true -cinder = true -glance = true -heat = true -horizon = true -ironic = false -neutron = true -nova = true -sahara = false -swift = true -trove = false -zaqar = false - -[volume] -backend_names = cinder-ceph -storage_protocol = ceph - -[volume-feature-enabled] -backup = false""" +debug = false""" From 816c6462ca19c15dd4aa9be8a6ad6b53fa87bdd0 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2020 17:42:54 +0000 Subject: [PATCH 502/898] Revert to remember --- zaza/openstack/charm_tests/tempest/setup.py | 56 ++++++++++++--------- zaza/openstack/charm_tests/tempest/tests.py | 21 ++++---- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 7336299..214b816 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -15,7 +15,7 @@ """Code for configuring tempest.""" import urllib.parse import os -import shutil +#import shutil import subprocess import zaza.model @@ -25,7 +25,7 @@ import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.tempest.templates.tempest_v2 as tempest_v2 import zaza.openstack.charm_tests.tempest.templates.tempest_v3 as tempest_v3 import zaza.openstack.charm_tests.tempest.templates.accounts as accounts -import tempest.cmd.main +#import tempest.cmd.main import keystoneauth1 import novaclient @@ -154,36 +154,42 @@ def render_tempest_config(target_file, ctxt, tempest_template): def setup_tempest(tempest_template, accounts_template): - config_dir = '.tempest' - config_etc_dir = os.path.join(config_dir, 'etc') - config_etc_tempest = os.path.join(config_etc_dir, 'tempest.conf') - config_workspace_yaml = os.path.join(config_dir, 'workspace.yaml') - workspace_name = 'workspace' - workspace_dir = os.path.join(config_dir, workspace_name) - workspace_etc_dir = os.path.join(workspace_dir, 'etc') - workspace_etc_accounts = os.path.join(workspace_etc_dir, 'accounts.yaml') - workspace_etc_tempest = os.path.join(workspace_etc_dir, 'tempest.conf') - the_app = tempest.cmd.main.Main() + try: + os.makedirs('tempest/etc/') + except FileExistsError: + pass + #config_dir = '.tempest' + #config_etc_dir = os.path.join(config_dir, 'etc') + #config_etc_tempest = os.path.join(config_etc_dir, 'tempest.conf') + #config_workspace_yaml = os.path.join(config_dir, 'workspace.yaml') + #workspace_name = 'workspace' + #workspace_dir = os.path.join(config_dir, workspace_name) + #workspace_etc_dir = os.path.join(workspace_dir, 'etc') + #workspace_etc_accounts = os.path.join(workspace_etc_dir, 'accounts.yaml') + #workspace_etc_tempest = os.path.join(workspace_etc_dir, 'tempest.conf') + #the_app = tempest.cmd.main.Main() # note sure this is needed or not - if not os.path.isdir(config_dir): - os.mkdir(config_dir) - os.mkdir(config_etc_dir) - render_tempest_config( - config_etc_tempest, - get_tempest_context(), - tempest_template) - tempest_options = ['init', '--workspace-path', config_workspace_yaml, - '--name', workspace_name, workspace_dir] - print(tempest_options) - _exec_tempest = the_app.run(tempest_options) + #if not os.path.isdir(config_dir): + # os.mkdir(config_dir) + # os.mkdir(config_etc_dir) + #render_tempest_config( + # config_etc_tempest, + # get_tempest_context(), + # tempest_template) + #tempest_options = ['init', '--workspace-path', config_workspace_yaml, + # '--name', workspace_name, workspace_dir] + #print(tempest_options) + #_exec_tempest = the_app.run(tempest_options) # This was mising /etc/tempest/ and just going to /etc/ render_tempest_config( - workspace_etc_tempest, + 'tempest/etc/tempest.conf', + #workspace_etc_tempest, get_tempest_context(), tempest_template) render_tempest_config( - workspace_etc_accounts, + 'tempest/etc/accounts.yaml', + #workspace_etc_accounts, get_tempest_context(), accounts_template) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 9bc6221..5e949aa 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -14,16 +14,17 @@ class TempestTest(): def run(self): charm_config = zaza.charm_lifecycle.utils.get_charm_config() - config_dir = '.tempest' - config_workspace_yaml = os.path.join(config_dir, 'workspace.yaml') - workspace_name = 'workspace' - workspace_dir = os.path.join(config_dir, workspace_name) - workspace_etc_dir = os.path.join(workspace_dir, 'etc') - workspace_etc_tempest = os.path.join(workspace_etc_dir, 'tempest.conf') - tempest_options = ['run', '--config-file', - workspace_etc_tempest, - '--workspace-path', config_workspace_yaml, - '--workspace', workspace_name] + tempest_options = ['run', '--config', 'tempest/etc/tempest.conf'] + #config_dir = '.tempest' + #config_workspace_yaml = os.path.join(config_dir, 'workspace.yaml') + #workspace_name = 'workspace' + #workspace_dir = os.path.join(config_dir, workspace_name) + #workspace_etc_dir = os.path.join(workspace_dir, 'etc') + #workspace_etc_tempest = os.path.join(workspace_etc_dir, 'tempest.conf') + #tempest_options = ['run', '--config-file', + # workspace_etc_tempest, + # '--workspace-path', config_workspace_yaml, + # '--workspace', workspace_name] for model_alias in zaza.model.get_juju_model_aliases().keys(): tempest_test_key = model_alias if model_alias == zaza.charm_lifecycle.utils.DEFAULT_MODEL_ALIAS: From 27cc4ebdcf21e5efe966de6e8d3d7d4198155f91 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2020 18:53:15 +0000 Subject: [PATCH 503/898] Switch to running tempest with subprocess and use workspace --- zaza/openstack/charm_tests/tempest/setup.py | 21 ++++++++++++++------- zaza/openstack/charm_tests/tempest/tests.py | 16 ++++++++++------ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 214b816..9789d1e 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -15,7 +15,7 @@ """Code for configuring tempest.""" import urllib.parse import os -#import shutil +import shutil import subprocess import zaza.model @@ -154,10 +154,17 @@ def render_tempest_config(target_file, ctxt, tempest_template): def setup_tempest(tempest_template, accounts_template): - try: - os.makedirs('tempest/etc/') - except FileExistsError: - pass + #try: + # os.makedirs('tempest/etc/') + #except FileExistsError: + # pass + if os.path.isdir('tempest-workspace'): + try: + subprocess.check_call(['tempest', 'workspace', 'remove', '--rmdir', '--name', 'tempest-workspace']) + except subprocess.CalledProcessError: + pass + #shutil.rmtree('tempest-workspace') + subprocess.check_call(['tempest', 'init', 'tempest-workspace']) #config_dir = '.tempest' #config_etc_dir = os.path.join(config_dir, 'etc') #config_etc_tempest = os.path.join(config_etc_dir, 'tempest.conf') @@ -183,12 +190,12 @@ def setup_tempest(tempest_template, accounts_template): #_exec_tempest = the_app.run(tempest_options) # This was mising /etc/tempest/ and just going to /etc/ render_tempest_config( - 'tempest/etc/tempest.conf', + 'tempest-workspace/etc/tempest.conf', #workspace_etc_tempest, get_tempest_context(), tempest_template) render_tempest_config( - 'tempest/etc/accounts.yaml', + 'tempest-workspace/etc/accounts.yaml', #workspace_etc_accounts, get_tempest_context(), accounts_template) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 5e949aa..a4360a9 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -1,4 +1,5 @@ import os +import subprocess import zaza import zaza.charm_lifecycle.utils @@ -14,7 +15,8 @@ class TempestTest(): def run(self): charm_config = zaza.charm_lifecycle.utils.get_charm_config() - tempest_options = ['run', '--config', 'tempest/etc/tempest.conf'] + tempest_options = ['run', '--workspace', 'tempest-workspace', + '--config', 'tempest-workspace/etc/tempest.conf'] #config_dir = '.tempest' #config_workspace_yaml = os.path.join(config_dir, 'workspace.yaml') #workspace_name = 'workspace' @@ -54,10 +56,12 @@ class TempestTest(): f.write('\n') tempest_options.extend(['--blacklist-file', black_file]) print(tempest_options) - the_app = tempest.cmd.main.Main() - project_root = os.getcwd() - _exec_tempest = the_app.run(tempest_options) - os.chdir(project_root) - if _exec_tempest != 0: + #the_app = tempest.cmd.main.Main() + #project_root = os.getcwd() + #_exec_tempest = the_app.run(tempest_options) + #os.chdir(project_root) + try: + subprocess.check_call(tempest_options) + except subprocess.CalledProcessError: return False return True From 411951d1a1f99308a3fd98940a3d6cb596ce3575 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2020 20:35:18 +0000 Subject: [PATCH 504/898] uncomment get_tempest_context --- zaza/openstack/charm_tests/tempest/setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 9789d1e..6bb900b 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -138,13 +138,13 @@ def add_access_protocol(ctxt): def get_tempest_context(): keystone_session = openstack_utils.get_overcloud_keystone_session() ctxt = {} - #add_application_ips(ctxt) - #add_nova_config(ctxt, keystone_session) - #add_neutron_config(ctxt, keystone_session) - #add_glance_config(ctxt, keystone_session) - #add_keystone_config(ctxt, keystone_session) - #add_environment_var_config(ctxt) - #add_access_protocol(ctxt) + add_application_ips(ctxt) + add_nova_config(ctxt, keystone_session) + add_neutron_config(ctxt, keystone_session) + add_glance_config(ctxt, keystone_session) + add_keystone_config(ctxt, keystone_session) + add_environment_var_config(ctxt) + add_access_protocol(ctxt) return ctxt From 2df890cabf8a0204d2ea74bdfe8b068c19ddb853 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2020 13:25:19 +0000 Subject: [PATCH 505/898] s/run/tempest run/ --- zaza/openstack/charm_tests/tempest/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index a4360a9..10a2298 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -15,7 +15,7 @@ class TempestTest(): def run(self): charm_config = zaza.charm_lifecycle.utils.get_charm_config() - tempest_options = ['run', '--workspace', 'tempest-workspace', + tempest_options = ['tempest', 'run', '--workspace', 'tempest-workspace', '--config', 'tempest-workspace/etc/tempest.conf'] #config_dir = '.tempest' #config_workspace_yaml = os.path.join(config_dir, 'workspace.yaml') From f4c150f3e70e71dfba4d039b90655bd3e93c37f2 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2020 15:23:49 +0000 Subject: [PATCH 506/898] restore templates --- .../tempest/templates/tempest_v2.py | 96 ++++++++++++++++++- .../tempest/templates/tempest_v3.py | 94 +++++++++++++++++- 2 files changed, 188 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py index aa6b555..ec95577 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py @@ -1,3 +1,97 @@ file_contents = """ [DEFAULT] -debug = false""" +debug = false +use_stderr = false +log_file = tempest.log + +[auth] +test_accounts_file = accounts.yaml +default_credentials_domain_name = Default +admin_username = {admin_username} +admin_project_name = admin +admin_password = {admin_password} +admin_domain_name = Default + +[compute] +image_ref = {image_id} +image_ref_alt = {image_alt_id} +flavor_ref = {flavor_ref} +flavor_ref_alt = {flavor_ref_alt} +region = RegionOne +min_compute_nodes = 3 + +# TODO: review this as its release specific +# min_microversion = 2.2 +# max_microversion = latest + +[compute-feature-enabled] +console_output = true +resize = true +live_migration = true +block_migration_for_live_migration = true +attach_encrypted_volume = false + +[identity] +uri = {proto}://{keystone}:5000/v2.0 +auth_version = v2 +admin_role = Admin +region = RegionOne + +[identity-feature-enabled] +api_v2 = true +api_v3 = false + +[image] +http_image = http://{os_test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz + +[network] +project_network_cidr = {os_test_cidr_priv} +public_network_id = {ext_net} +dns_servers = {os_test_nameserver} +project_networks_reachable = false + +[network-feature-enabled] +ipv6 = false + +[orchestration] +stack_owner_role = Admin +instance_type = m1.small +keypair_name = testkey + +[oslo_concurrency] +lock_path = /tmp + +[scenario] +img_dir = /home/ubuntu/images +img_file = cirros-0.3.4-x86_64-disk.img +img_container_format = bare +img_disk_format = qcow2 + +[validation] +run_validation = true +image_ssh_user = cirros + +[service_available] +ceilometer = true +cinder = true +glance = true +heat = true +horizon = true +ironic = false +neutron = true +nova = true +sahara = false +swift = true +trove = false +zaqar = false + +[volume] +backend_names = cinder-ceph +storage_protocol = ceph +# NOTE(coreycb): Need to enalbe catalog_type, determined by: +# volume_version=$(openstack endpoint list -c 'Service Type' -f value | grep -m 1 volumev3 || +# openstack endpoint list -c 'Service Type' -f value | grep -m 1 volumev2) +# catalog_type = __VOLUME_VERSION__ + +[volume-feature-enabled] +backup = false""" diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index aa6b555..887b0b5 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -1,3 +1,95 @@ file_contents = """ [DEFAULT] -debug = false""" +debug = false +use_stderr = false +log_file = tempest.log + +[auth] +test_accounts_file = accounts.yaml +default_credentials_domain_name = {default_credentials_domain_name} +admin_username = {admin_username} +admin_project_name = {admin_project_name} +admin_password = {admin_password} +admin_domain_name = {admin_domain_name} + +[compute] +image_ref = {image_id} +image_ref_alt = {image_alt_id} +flavor_ref = {flavor_ref} +flavor_ref_alt = {flavor_ref_alt} +min_compute_nodes = 3 + +# TODO: review this as its release specific +# min_microversion = 2.2 +# max_microversion = latest + +[compute-feature-enabled] +console_output = true +resize = true +live_migration = true +block_migration_for_live_migration = true +attach_encrypted_volume = false + +[identity] +uri = {proto}://{keystone}:5000/v2.0 +uri_v3 = {proto}://{keystone}:5000/v3 +auth_version = v3 +admin_role = Admin +region = RegionOne +default_domain_id = {default_domain_id} +admin_domain_scope = true + +[identity-feature-enabled] +api_v2 = false +api_v3 = true + +[image] +http_image = http://{os_test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz + +[network] +project_network_cidr = {os_test_cidr_priv} +public_network_id = {ext_net} +dns_servers = {os_test_nameserver} +project_networks_reachable = false + +[network-feature-enabled] +ipv6 = false + +[orchestration] +stack_owner_role = Admin +instance_type = m1.small +keypair_name = testkey + +[oslo_concurrency] +lock_path = /tmp + +[scenario] +img_dir = /home/ubuntu/images +img_file = cirros-0.3.4-x86_64-disk.img +img_container_format = bare +img_disk_format = qcow2 + +[validation] +run_validation = true +image_ssh_user = cirros + +[service_available] +ceilometer = true +cinder = true +glance = true +heat = true +horizon = true +ironic = false +neutron = true +nova = true +sahara = false +swift = true +trove = false +zaqar = false + +[volume] +backend_names = cinder-ceph +storage_protocol = ceph + +[volume-feature-enabled] +backup = false""" From 635866afe4a5d8c506d9b3c92142aba7ee9b4660 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2020 18:51:43 +0000 Subject: [PATCH 507/898] cleanup --- zaza/openstack/charm_tests/tempest/setup.py | 37 +++------------------ zaza/openstack/charm_tests/tempest/tests.py | 14 -------- 2 files changed, 4 insertions(+), 47 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 6bb900b..cae3ab5 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -25,7 +25,6 @@ import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.tempest.templates.tempest_v2 as tempest_v2 import zaza.openstack.charm_tests.tempest.templates.tempest_v3 as tempest_v3 import zaza.openstack.charm_tests.tempest.templates.accounts as accounts -#import tempest.cmd.main import keystoneauth1 import novaclient @@ -154,49 +153,21 @@ def render_tempest_config(target_file, ctxt, tempest_template): def setup_tempest(tempest_template, accounts_template): - #try: - # os.makedirs('tempest/etc/') - #except FileExistsError: - # pass if os.path.isdir('tempest-workspace'): try: subprocess.check_call(['tempest', 'workspace', 'remove', '--rmdir', '--name', 'tempest-workspace']) except subprocess.CalledProcessError: pass - #shutil.rmtree('tempest-workspace') - subprocess.check_call(['tempest', 'init', 'tempest-workspace']) - #config_dir = '.tempest' - #config_etc_dir = os.path.join(config_dir, 'etc') - #config_etc_tempest = os.path.join(config_etc_dir, 'tempest.conf') - #config_workspace_yaml = os.path.join(config_dir, 'workspace.yaml') - #workspace_name = 'workspace' - #workspace_dir = os.path.join(config_dir, workspace_name) - #workspace_etc_dir = os.path.join(workspace_dir, 'etc') - #workspace_etc_accounts = os.path.join(workspace_etc_dir, 'accounts.yaml') - #workspace_etc_tempest = os.path.join(workspace_etc_dir, 'tempest.conf') - #the_app = tempest.cmd.main.Main() - - # note sure this is needed or not - #if not os.path.isdir(config_dir): - # os.mkdir(config_dir) - # os.mkdir(config_etc_dir) - #render_tempest_config( - # config_etc_tempest, - # get_tempest_context(), - # tempest_template) - #tempest_options = ['init', '--workspace-path', config_workspace_yaml, - # '--name', workspace_name, workspace_dir] - #print(tempest_options) - #_exec_tempest = the_app.run(tempest_options) - # This was mising /etc/tempest/ and just going to /etc/ + try: + subprocess.check_call(['tempest', 'init', 'tempest-workspace']) + except subprocess.CalledProcessError: + pass render_tempest_config( 'tempest-workspace/etc/tempest.conf', - #workspace_etc_tempest, get_tempest_context(), tempest_template) render_tempest_config( 'tempest-workspace/etc/accounts.yaml', - #workspace_etc_accounts, get_tempest_context(), accounts_template) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 10a2298..dfdc247 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -17,16 +17,6 @@ class TempestTest(): charm_config = zaza.charm_lifecycle.utils.get_charm_config() tempest_options = ['tempest', 'run', '--workspace', 'tempest-workspace', '--config', 'tempest-workspace/etc/tempest.conf'] - #config_dir = '.tempest' - #config_workspace_yaml = os.path.join(config_dir, 'workspace.yaml') - #workspace_name = 'workspace' - #workspace_dir = os.path.join(config_dir, workspace_name) - #workspace_etc_dir = os.path.join(workspace_dir, 'etc') - #workspace_etc_tempest = os.path.join(workspace_etc_dir, 'tempest.conf') - #tempest_options = ['run', '--config-file', - # workspace_etc_tempest, - # '--workspace-path', config_workspace_yaml, - # '--workspace', workspace_name] for model_alias in zaza.model.get_juju_model_aliases().keys(): tempest_test_key = model_alias if model_alias == zaza.charm_lifecycle.utils.DEFAULT_MODEL_ALIAS: @@ -56,10 +46,6 @@ class TempestTest(): f.write('\n') tempest_options.extend(['--blacklist-file', black_file]) print(tempest_options) - #the_app = tempest.cmd.main.Main() - #project_root = os.getcwd() - #_exec_tempest = the_app.run(tempest_options) - #os.chdir(project_root) try: subprocess.check_call(tempest_options) except subprocess.CalledProcessError: From 9e3b2f74165d038cf61ed3db01569fea0017ac45 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 27 Apr 2020 16:57:24 +0000 Subject: [PATCH 508/898] Add catalog_type tempest config support --- zaza/openstack/charm_tests/tempest/setup.py | 12 ++++++++++++ .../charm_tests/tempest/templates/tempest_v2.py | 5 +---- .../charm_tests/tempest/templates/tempest_v3.py | 1 + 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index cae3ab5..73480df 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -98,6 +98,17 @@ def add_glance_config(ctxt, keystone_session): ctxt['image_alt_id'] = image_alt[0].id +def add_cinder_config(ctxt, keystone_session): + volume_types = ['volumev2', 'volumev3'] + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + for volume_type in volume_types: + service = keystone_client.services.list(type=volume_type) + if service: + ctxt['catalog_type'] = volume_type + break + + def add_keystone_config(ctxt, keystone_session): keystone_client = openstack_utils.get_keystone_session_client( keystone_session) @@ -141,6 +152,7 @@ def get_tempest_context(): add_nova_config(ctxt, keystone_session) add_neutron_config(ctxt, keystone_session) add_glance_config(ctxt, keystone_session) + add_cinder_config(ctxt, keystone_session) add_keystone_config(ctxt, keystone_session) add_environment_var_config(ctxt) add_access_protocol(ctxt) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py index ec95577..e4b94ba 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py @@ -88,10 +88,7 @@ zaqar = false [volume] backend_names = cinder-ceph storage_protocol = ceph -# NOTE(coreycb): Need to enalbe catalog_type, determined by: -# volume_version=$(openstack endpoint list -c 'Service Type' -f value | grep -m 1 volumev3 || -# openstack endpoint list -c 'Service Type' -f value | grep -m 1 volumev2) -# catalog_type = __VOLUME_VERSION__ +catalog_type = {catalog_type} [volume-feature-enabled] backup = false""" diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index 887b0b5..6f71765 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -90,6 +90,7 @@ zaqar = false [volume] backend_names = cinder-ceph storage_protocol = ceph +catalog_type = {catalog_type} [volume-feature-enabled] backup = false""" From 8a39e07cbf5919da9128759bbd8d15ec784352e6 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 30 Apr 2020 14:54:20 +0000 Subject: [PATCH 509/898] Set disable_ssl_certificate_validation to true to deal with self-signed certs --- zaza/openstack/charm_tests/tempest/templates/tempest_v2.py | 1 + zaza/openstack/charm_tests/tempest/templates/tempest_v3.py | 1 + 2 files changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py index e4b94ba..ecca5ce 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py @@ -36,6 +36,7 @@ uri = {proto}://{keystone}:5000/v2.0 auth_version = v2 admin_role = Admin region = RegionOne +disable_ssl_certificate_validation = true [identity-feature-enabled] api_v2 = true diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index 6f71765..f189ad5 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -38,6 +38,7 @@ admin_role = Admin region = RegionOne default_domain_id = {default_domain_id} admin_domain_scope = true +disable_ssl_certificate_validation = true [identity-feature-enabled] api_v2 = false From 19be6f7d48e9af8b49630dffc92e8f93149418b6 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 18 May 2020 18:04:04 +0000 Subject: [PATCH 510/898] Fix copyright --- zaza/openstack/charm_tests/tempest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/__init__.py b/zaza/openstack/charm_tests/tempest/__init__.py index ed3be2f..9703646 100644 --- a/zaza/openstack/charm_tests/tempest/__init__.py +++ b/zaza/openstack/charm_tests/tempest/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2018 Canonical Ltd. +# Copyright 2020 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From a21ddd33e6dcf1976d615561627f89a744a0c4e8 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 18 May 2020 20:04:47 +0000 Subject: [PATCH 511/898] Code cleanup --- zaza/openstack/charm_tests/tempest/setup.py | 138 ++++++++++++++++-- .../charm_tests/tempest/templates/__init__.py | 2 +- zaza/openstack/charm_tests/tempest/tests.py | 30 +++- 3 files changed, 153 insertions(+), 17 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 73480df..59b3b8b 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -13,6 +13,7 @@ # limitations under the License. """Code for configuring tempest.""" + import urllib.parse import os import shutil @@ -43,6 +44,13 @@ TEMPEST_CIRROS_ALT_IMAGE_NAME = 'cirros_alt' def get_app_access_ip(application_name): + """Get the application's access IP + + :param application_name: Name of application + :type application_name: str + :returns: Application's access IP + :rtype: str + """ try: app_config = zaza.model.get_application_config(application_name) except KeyError: @@ -57,12 +65,28 @@ def get_app_access_ip(application_name): def add_application_ips(ctxt): + """Add application access IPs to context + + :param ctxt: Context dictionary + :type ctxt: dict + :returns: None + :rtype: None + """ ctxt['keystone'] = get_app_access_ip('keystone') ctxt['dashboard'] = get_app_access_ip('openstack-dashboard') ctxt['ncc'] = get_app_access_ip('nova-cloud-controller') def add_nova_config(ctxt, keystone_session): + """Add nova config to context + + :param ctxt: Context dictionary + :type ctxt: dict + :returns keystone_session: keystoneauth1.session.Session object + :type: keystoneauth1.session.Session + :returns: None + :rtype: None + """ nova_client = openstack_utils.get_nova_session_client( keystone_session) for flavor in nova_client.flavors.list(): @@ -73,6 +97,15 @@ def add_nova_config(ctxt, keystone_session): def add_neutron_config(ctxt, keystone_session): + """Add neutron config to context + + :param ctxt: Context dictionary + :type ctxt: dict + :returns keystone_session: keystoneauth1.session.Session object + :type: keystoneauth1.session.Session + :returns: None + :rtype: None + """ neutron_client = openstack_utils.get_neutron_session_client( keystone_session) for net in neutron_client.list_networks()['networks']: @@ -86,6 +119,15 @@ def add_neutron_config(ctxt, keystone_session): def add_glance_config(ctxt, keystone_session): + """Add glance config to context + + :param ctxt: Context dictionary + :type ctxt: dict + :returns keystone_session: keystoneauth1.session.Session object + :type: keystoneauth1.session.Session + :returns: None + :rtype: None + """ glance_client = openstack_utils.get_glance_session_client( keystone_session) image = openstack_utils.get_images_by_name( @@ -99,6 +141,15 @@ def add_glance_config(ctxt, keystone_session): def add_cinder_config(ctxt, keystone_session): + """Add cinder config to context + + :param ctxt: Context dictionary + :type ctxt: dict + :returns keystone_session: keystoneauth1.session.Session object + :type: keystoneauth1.session.Session + :returns: None + :rtype: None + """ volume_types = ['volumev2', 'volumev3'] keystone_client = openstack_utils.get_keystone_session_client( keystone_session) @@ -110,6 +161,15 @@ def add_cinder_config(ctxt, keystone_session): def add_keystone_config(ctxt, keystone_session): + """Add keystone config to context + + :param ctxt: Context dictionary + :type ctxt: dict + :returns keystone_session: keystoneauth1.session.Session object + :type: keystoneauth1.session.Session + :returns: None + :rtype: None + """ keystone_client = openstack_utils.get_keystone_session_client( keystone_session) for domain in keystone_client.domains.list(): @@ -119,6 +179,13 @@ def add_keystone_config(ctxt, keystone_session): def add_environment_var_config(ctxt): + """Add environment variable config to context + + :param ctxt: Context dictionary + :type ctxt: dict + :returns: None + :rtype: None + """ deploy_env = deployment_env.get_deployment_context() for var in SETUP_ENV_VARS: value = deploy_env.get(var) @@ -130,7 +197,14 @@ def add_environment_var_config(ctxt): " test").format(', '.join(SETUP_ENV_VARS))) -def add_access_protocol(ctxt): +def add_auth_config(ctxt): + """Add authorization config to context + + :param ctxt: Context dictionary + :type ctxt: dict + :returns: None + :rtype: None + """ overcloud_auth = openstack_utils.get_overcloud_auth() ctxt['proto'] = urllib.parse.urlparse(overcloud_auth['OS_AUTH_URL']).scheme ctxt['admin_username'] = overcloud_auth['OS_USERNAME'] @@ -140,12 +214,14 @@ def add_access_protocol(ctxt): ctxt['admin_domain_name'] = overcloud_auth['OS_DOMAIN_NAME'] ctxt['default_credentials_domain_name'] = overcloud_auth[ 'OS_PROJECT_DOMAIN_NAME'] - elif overcloud_auth['API_VERSION'] == 2: - #ctxt['admin_tenant_name'] = overcloud_auth['OS_TENANT_NAME'] - pass def get_tempest_context(): + """Generate the tempest config context + + :returns: Context dictionary + :rtype: dict + """ keystone_session = openstack_utils.get_overcloud_keystone_session() ctxt = {} add_application_ips(ctxt) @@ -155,16 +231,37 @@ def get_tempest_context(): add_cinder_config(ctxt, keystone_session) add_keystone_config(ctxt, keystone_session) add_environment_var_config(ctxt) - add_access_protocol(ctxt) + add_auth_config(ctxt) return ctxt -def render_tempest_config(target_file, ctxt, tempest_template): +def render_tempest_config(target_file, ctxt, template): + """Render tempest config for specified config file and template + + :param target_file: Name of file to render config to + :type target_file: str + :param ctxt: Context dictionary + :type ctxt: dict + :param template: Template module + :type template: module + :returns: None + :rtype: None + """ + # TODO: switch to jinja2 and generate config based on available services with open(target_file, 'w') as f: f.write(tempest_template.file_contents.format(**ctxt)) def setup_tempest(tempest_template, accounts_template): + """Initialize tempest and render tempest config + + :param tempest_template: tempest.conf template + :type tempest_template: module + :param accounts_template: accounts.yaml template + :type accounts_template: module + :returns: None + :rtype: None + """ if os.path.isdir('tempest-workspace'): try: subprocess.check_call(['tempest', 'workspace', 'remove', '--rmdir', '--name', 'tempest-workspace']) @@ -185,20 +282,28 @@ def setup_tempest(tempest_template, accounts_template): def render_tempest_config_keystone_v2(): + """Render tempest config for Keystone V2 API + + :returns: None + :rtype: None + """ setup_tempest(tempest_v2, accounts) def render_tempest_config_keystone_v3(): + """Render tempest config for Keystone V3 API + + :returns: None + :rtype: None + """ setup_tempest(tempest_v3, accounts) def add_cirros_alt_image(): - """Add a cirros image to the current deployment. + """Add cirros alternate image to overcloud - :param glance: Authenticated glanceclient - :type glance: glanceclient.Client - :param image_name: Label for the image in glance - :type image_name: str + :returns: None + :rtype: None """ image_url = openstack_utils.find_cirros_image(arch='x86_64') glance_setup.add_image( @@ -208,6 +313,11 @@ def add_cirros_alt_image(): def add_tempest_flavors(): + """Add tempest flavors to overcloud + + :returns: None + :rtype: None + """ keystone_session = openstack_utils.get_overcloud_keystone_session() nova_client = openstack_utils.get_nova_session_client( keystone_session) @@ -230,6 +340,11 @@ def add_tempest_flavors(): def add_tempest_roles(): + """Add tempest roles overcloud + + :returns: None + :rtype: None + """ keystone_session = openstack_utils.get_overcloud_keystone_session() keystone_client = openstack_utils.get_keystone_session_client( keystone_session) @@ -238,4 +353,3 @@ def add_tempest_roles(): keystone_client.roles.create(role_name) except keystoneauth1.exceptions.http.Conflict: pass - diff --git a/zaza/openstack/charm_tests/tempest/templates/__init__.py b/zaza/openstack/charm_tests/tempest/templates/__init__.py index 1400887..9269e37 100644 --- a/zaza/openstack/charm_tests/tempest/templates/__init__.py +++ b/zaza/openstack/charm_tests/tempest/templates/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2018 Canonical Ltd. +# Copyright 2020 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index dfdc247..6fc58d7 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -1,3 +1,19 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for running tempest tests.""" + import os import subprocess @@ -7,13 +23,19 @@ import zaza.charm_lifecycle.test import tempest.cmd.main import tempfile - - class TempestTest(): - - test_runner = zaza.charm_lifecycle.test.DIRECT + """Tempest test class.""" def run(self): + """Run tempest tests as specified in tests/tests.yaml + + Test keys are parsed from ['tests_options']['tempest']['model'], where + valid test keys are: smoke (bool), whitelist (list of tests), blacklist + (list of tests), and regex (list of regex's). + + :returns: Status of tempest run + :rtype: bool + """ charm_config = zaza.charm_lifecycle.utils.get_charm_config() tempest_options = ['tempest', 'run', '--workspace', 'tempest-workspace', '--config', 'tempest-workspace/etc/tempest.conf'] From 9336e3efa74b65ffe3cbfd8b96b0b9984d8f0fb4 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 18 May 2020 20:07:31 +0000 Subject: [PATCH 512/898] Minor update --- zaza/openstack/charm_tests/tempest/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 59b3b8b..e36b026 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Code for configuring tempest.""" +"""Code for configuring and initializing tempest.""" import urllib.parse import os From f668784f410f0d606cfa24e04057460c67f03862 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 18 May 2020 20:09:37 +0000 Subject: [PATCH 513/898] Add the missing test_runner --- zaza/openstack/charm_tests/tempest/tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index 6fc58d7..bbc9d9e 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -25,6 +25,7 @@ import tempfile class TempestTest(): """Tempest test class.""" + test_runner = zaza.charm_lifecycle.test.DIRECT def run(self): """Run tempest tests as specified in tests/tests.yaml From db47aa1fb9fa627481bcd14e8cc947a6eadfe267 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 18 May 2020 20:10:52 +0000 Subject: [PATCH 514/898] Minor update --- zaza/openstack/charm_tests/tempest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/__init__.py b/zaza/openstack/charm_tests/tempest/__init__.py index 9703646..cf4c994 100644 --- a/zaza/openstack/charm_tests/tempest/__init__.py +++ b/zaza/openstack/charm_tests/tempest/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Collection of code for setting up and using tempest.""" +"""Collection of code for setting up and running tempest.""" From 5075e97470677c450c2e044493596c03bb55ac94 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 20 May 2020 18:13:48 +0000 Subject: [PATCH 515/898] Switch Member -> member --- zaza/openstack/charm_tests/tempest/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index e36b026..4f25686 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -348,7 +348,7 @@ def add_tempest_roles(): keystone_session = openstack_utils.get_overcloud_keystone_session() keystone_client = openstack_utils.get_keystone_session_client( keystone_session) - for role_name in ['Member', 'ResellerAdmin']: + for role_name in ['member', 'ResellerAdmin']: try: keystone_client.roles.create(role_name) except keystoneauth1.exceptions.http.Conflict: From 19725c73b54f86440d8bbf8a5b8ea329ad7540fd Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 20 May 2020 19:48:04 +0000 Subject: [PATCH 516/898] Fix template variable in render_tempest_config --- zaza/openstack/charm_tests/tempest/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 4f25686..5dd833e 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -249,7 +249,7 @@ def render_tempest_config(target_file, ctxt, template): """ # TODO: switch to jinja2 and generate config based on available services with open(target_file, 'w') as f: - f.write(tempest_template.file_contents.format(**ctxt)) + f.write(template.file_contents.format(**ctxt)) def setup_tempest(tempest_template, accounts_template): From c98aa001f997d2cc149e0c64e81f0339a83adc50 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 22 May 2020 19:43:10 +0000 Subject: [PATCH 517/898] Adjust basic_setup to run upgrade for < ocata --- zaza/openstack/charm_tests/ceilometer/setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceilometer/setup.py b/zaza/openstack/charm_tests/ceilometer/setup.py index c966077..0ff1fb5 100644 --- a/zaza/openstack/charm_tests/ceilometer/setup.py +++ b/zaza/openstack/charm_tests/ceilometer/setup.py @@ -28,11 +28,11 @@ def basic_setup(): tests. """ current_release = openstack_utils.get_os_release() - xenial_pike = openstack_utils.get_os_release('xenial_pike') + xenial_ocata = openstack_utils.get_os_release('xenial_ocata') - if current_release < xenial_pike: + if current_release < xenial_ocata: logging.info( - 'Skipping ceilometer-upgrade as it is not supported before Pike') + 'Skipping ceilometer-upgrade as it is not supported before ocata') return logging.debug('Checking ceilometer-upgrade') From 30b7b91904f3195d10ae138a1072b8eb376fd21e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 27 May 2020 13:50:43 +0000 Subject: [PATCH 518/898] Set api_extensions and floating_network_name api_extensions and floating_network_name are set in tempest.conf to fix tempest failures for: * test_list_show_extensions (missing 'l3_agent_scheduler' extension) * test_server_basic_ops (404 'Floating IP pool not found') --- zaza/openstack/charm_tests/tempest/setup.py | 23 +++++++++++++++++++ .../tempest/templates/tempest_v3.py | 2 ++ 2 files changed, 25 insertions(+) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 5dd833e..7f97014 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -106,6 +106,8 @@ def add_neutron_config(ctxt, keystone_session): :returns: None :rtype: None """ + current_release = openstack_utils.get_os_release() + focal_ussuri = openstack_utils.get_os_release('focal_ussuri') neutron_client = openstack_utils.get_neutron_session_client( keystone_session) for net in neutron_client.list_networks()['networks']: @@ -116,6 +118,27 @@ def add_neutron_config(ctxt, keystone_session): if router['name'] == 'provider-router': ctxt['provider_router_id'] = router['id'] break + # For focal+ with OVN, we use the same settings as upstream gate. + # This is because the l3_agent_scheduler extension is only + # applicable for OVN when conventional layer-3 agent enabled: + # https://docs.openstack.org/networking-ovn/2.0.1/features.html + # This enables test_list_show_extensions to run successfully. + if current_release >= focal_ussuri: + extensions = ('address-scope,agent,allowed-address-pairs,' + 'auto-allocated-topology,availability_zone,' + 'binding,default-subnetpools,external-net,' + 'extra_dhcp_opt,multi-provider,net-mtu,' + 'network_availability_zone,network-ip-availability,' + 'port-security,provider,quotas,rbac-address-scope,' + 'rbac-policies,standard-attr-revisions,security-group,' + 'standard-attr-description,subnet_allocation,' + 'standard-attr-tag,standard-attr-timestamp,trunk,' + 'quota_details,router,extraroute,ext-gw-mode,' + 'fip-port-details,pagination,sorting,project-id,' + 'dns-integration,qos') + ctxt['neutron_api_extensions'] = extensions + else: + ctxt['neutron_api_extensions'] = 'all' def add_glance_config(ctxt, keystone_session): diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index f189ad5..7c95e3e 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -52,9 +52,11 @@ project_network_cidr = {os_test_cidr_priv} public_network_id = {ext_net} dns_servers = {os_test_nameserver} project_networks_reachable = false +floating_network_name = {ext_net} [network-feature-enabled] ipv6 = false +api_extensions = {neutron_api_extensions} [orchestration] stack_owner_role = Admin From e7da28abc93803c00f1954ee45758887114b29e4 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 27 May 2020 21:19:35 +0000 Subject: [PATCH 519/898] Revert "Set api_extensions and floating_network_name" This reverts commit 1cc280292963ee70aeff9f9dbb96f457f398999f. --- zaza/openstack/charm_tests/tempest/setup.py | 23 ------------------- .../tempest/templates/tempest_v3.py | 2 -- 2 files changed, 25 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 7f97014..5dd833e 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -106,8 +106,6 @@ def add_neutron_config(ctxt, keystone_session): :returns: None :rtype: None """ - current_release = openstack_utils.get_os_release() - focal_ussuri = openstack_utils.get_os_release('focal_ussuri') neutron_client = openstack_utils.get_neutron_session_client( keystone_session) for net in neutron_client.list_networks()['networks']: @@ -118,27 +116,6 @@ def add_neutron_config(ctxt, keystone_session): if router['name'] == 'provider-router': ctxt['provider_router_id'] = router['id'] break - # For focal+ with OVN, we use the same settings as upstream gate. - # This is because the l3_agent_scheduler extension is only - # applicable for OVN when conventional layer-3 agent enabled: - # https://docs.openstack.org/networking-ovn/2.0.1/features.html - # This enables test_list_show_extensions to run successfully. - if current_release >= focal_ussuri: - extensions = ('address-scope,agent,allowed-address-pairs,' - 'auto-allocated-topology,availability_zone,' - 'binding,default-subnetpools,external-net,' - 'extra_dhcp_opt,multi-provider,net-mtu,' - 'network_availability_zone,network-ip-availability,' - 'port-security,provider,quotas,rbac-address-scope,' - 'rbac-policies,standard-attr-revisions,security-group,' - 'standard-attr-description,subnet_allocation,' - 'standard-attr-tag,standard-attr-timestamp,trunk,' - 'quota_details,router,extraroute,ext-gw-mode,' - 'fip-port-details,pagination,sorting,project-id,' - 'dns-integration,qos') - ctxt['neutron_api_extensions'] = extensions - else: - ctxt['neutron_api_extensions'] = 'all' def add_glance_config(ctxt, keystone_session): diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index 7c95e3e..f189ad5 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -52,11 +52,9 @@ project_network_cidr = {os_test_cidr_priv} public_network_id = {ext_net} dns_servers = {os_test_nameserver} project_networks_reachable = false -floating_network_name = {ext_net} [network-feature-enabled] ipv6 = false -api_extensions = {neutron_api_extensions} [orchestration] stack_owner_role = Admin From 924da664cff843e159dcd4091986d00f6ad2ae90 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 28 May 2020 01:29:14 +0000 Subject: [PATCH 520/898] Revert "Revert "Set api_extensions and floating_network_name"" This reverts commit 21ee604329552faacfe7984180eb34e003d0cc1b. --- zaza/openstack/charm_tests/tempest/setup.py | 23 +++++++++++++++++++ .../tempest/templates/tempest_v3.py | 2 ++ 2 files changed, 25 insertions(+) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 5dd833e..7f97014 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -106,6 +106,8 @@ def add_neutron_config(ctxt, keystone_session): :returns: None :rtype: None """ + current_release = openstack_utils.get_os_release() + focal_ussuri = openstack_utils.get_os_release('focal_ussuri') neutron_client = openstack_utils.get_neutron_session_client( keystone_session) for net in neutron_client.list_networks()['networks']: @@ -116,6 +118,27 @@ def add_neutron_config(ctxt, keystone_session): if router['name'] == 'provider-router': ctxt['provider_router_id'] = router['id'] break + # For focal+ with OVN, we use the same settings as upstream gate. + # This is because the l3_agent_scheduler extension is only + # applicable for OVN when conventional layer-3 agent enabled: + # https://docs.openstack.org/networking-ovn/2.0.1/features.html + # This enables test_list_show_extensions to run successfully. + if current_release >= focal_ussuri: + extensions = ('address-scope,agent,allowed-address-pairs,' + 'auto-allocated-topology,availability_zone,' + 'binding,default-subnetpools,external-net,' + 'extra_dhcp_opt,multi-provider,net-mtu,' + 'network_availability_zone,network-ip-availability,' + 'port-security,provider,quotas,rbac-address-scope,' + 'rbac-policies,standard-attr-revisions,security-group,' + 'standard-attr-description,subnet_allocation,' + 'standard-attr-tag,standard-attr-timestamp,trunk,' + 'quota_details,router,extraroute,ext-gw-mode,' + 'fip-port-details,pagination,sorting,project-id,' + 'dns-integration,qos') + ctxt['neutron_api_extensions'] = extensions + else: + ctxt['neutron_api_extensions'] = 'all' def add_glance_config(ctxt, keystone_session): diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index f189ad5..7c95e3e 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -52,9 +52,11 @@ project_network_cidr = {os_test_cidr_priv} public_network_id = {ext_net} dns_servers = {os_test_nameserver} project_networks_reachable = false +floating_network_name = {ext_net} [network-feature-enabled] ipv6 = false +api_extensions = {neutron_api_extensions} [orchestration] stack_owner_role = Admin From 9eefa08ce4d20a9e5afb850744831642a2a428cb Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 28 May 2020 10:57:29 +0000 Subject: [PATCH 521/898] Refactor Designate tests so API tests can be called Refactor Designate tests so that the api tests can be called without running pause/resume tests. This is useful in the mojo replacement functional tests. --- zaza/openstack/charm_tests/designate/tests.py | 68 +++++++++++-------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index f8a32db..5583f0b 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -83,8 +83,8 @@ class BaseDesignateTest(test_utils.OpenStackBaseTest): cls.server_delete = cls.designate.servers.delete -class DesignateTests(BaseDesignateTest): - """Designate charm restart and pause tests.""" +class DesignateAPITests(BaseDesignateTest): + """Tests interact with designate api.""" TEST_DOMAIN = 'amuletexample.com.' TEST_NS1_RECORD = 'ns1.{}'.format(TEST_DOMAIN) @@ -92,33 +92,6 @@ class DesignateTests(BaseDesignateTest): TEST_WWW_RECORD = "www.{}".format(TEST_DOMAIN) TEST_RECORD = {TEST_WWW_RECORD: '10.0.0.23'} - def test_900_restart_on_config_change(self): - """Checking restart happens on config change. - - Change debug mode and assert that change propagates to the correct - file and that services are restarted as a result - """ - # Services which are expected to restart upon config change, - # and corresponding config files affected by the change - conf_file = '/etc/designate/designate.conf' - - # Make config change, check for service restarts - self.restart_on_changed_debug_oslo_config_file( - conf_file, - self.designate_svcs, - ) - - def test_910_pause_and_resume(self): - """Run pause and resume tests. - - Pause service and check services are stopped then resume and check - they are started - """ - with self.pause_resume( - self.designate_svcs, - pgrep_full=False): - logging.info("Testing pause resume") - def _get_server_id(self, server_name=None, server_id=None): for srv in self.server_list(): if isinstance(srv, dict): @@ -245,3 +218,40 @@ class DesignateTests(BaseDesignateTest): logging.debug('Tidy up delete test record') self._wait_on_domain_gone(domain_id) logging.debug('OK') + + +class DesignateCharmTests(BaseDesignateTest): + """Designate charm restart and pause tests.""" + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change. + + Change debug mode and assert that change propagates to the correct + file and that services are restarted as a result + """ + # Services which are expected to restart upon config change, + # and corresponding config files affected by the change + conf_file = '/etc/designate/designate.conf' + + # Make config change, check for service restarts + self.restart_on_changed_debug_oslo_config_file( + conf_file, + self.designate_svcs, + ) + + def test_910_pause_and_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + with self.pause_resume( + self.designate_svcs, + pgrep_full=False): + logging.info("Testing pause resume") + + +class DesignateTests(DesignateAPITests, DesignateCharmTests): + """Collection of all Designate test classes.""" + + pass From 8c747390d3505c0f9699a0e4c51a7df2398212ba Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 28 May 2020 13:23:11 +0200 Subject: [PATCH 522/898] Fail early when no units found for external port creation Fixes #298 --- .../test_zaza_utilities_openstack.py | 30 +++++++++++++++++++ zaza/openstack/utilities/openstack.py | 8 +++++ 2 files changed, 38 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 5c344bf..3083204 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1216,3 +1216,33 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.assertTrue(openstack_utils.ovn_present()) self.get_application.side_effect = [KeyError, KeyError] self.assertFalse(openstack_utils.ovn_present()) + + def test_configure_gateway_ext_port(self): + # FIXME: this is not a complete unit test for the function as one did + # not exist at all I'm adding this to test one bit and we'll add more + # as we go. + self.patch_object(openstack_utils, 'deprecated_external_networking') + self.patch_object(openstack_utils, 'dvr_enabled') + self.patch_object(openstack_utils, 'ovn_present') + self.patch_object(openstack_utils, 'get_gateway_uuids') + self.patch_object(openstack_utils, 'get_admin_net') + self.dvr_enabled = False + self.ovn_present = False + self.get_admin_net.return_value = {'id': 'fakeid'} + + novaclient = mock.MagicMock() + neutronclient = mock.MagicMock() + + def _fake_empty_generator(empty=True): + if empty: + return + yield + + self.get_gateway_uuids.side_effect = _fake_empty_generator + with self.assertRaises(RuntimeError): + openstack_utils.configure_gateway_ext_port( + novaclient, neutronclient) + # provide a uuid and check that we don't raise RuntimeError + self.get_gateway_uuids.side_effect = ['fake-uuid'] + openstack_utils.configure_gateway_ext_port( + novaclient, neutronclient) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 47cabf3..ea2a6ba 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -732,6 +732,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, if not net_id: net_id = get_admin_net(neutronclient)['id'] + ports_created = 0 for uuid in uuids: server = novaclient.servers.get(uuid) ext_port_name = "{}_ext-port".format(server.name) @@ -752,12 +753,19 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, } } port = neutronclient.create_port(body=body_value) + ports_created += 1 server.interface_attach(port_id=port['port']['id'], net_id=None, fixed_ip=None) if add_dataport_to_netplan: mac_address = get_mac_from_port(port, neutronclient) add_interface_to_netplan(server.name, mac_address=mac_address) + if not ports_created: + # NOTE: uuids is an iterator so testing it for contents or length prior + # to iterating over it is futile. + raise RuntimeError('Unable to determine UUIDs for machines to attach ' + 'external networking to.') + ext_br_macs = [] for port in neutronclient.list_ports(network_id=net_id)['ports']: if 'ext-port' in port['name']: From 35a9e55ea543bda3b26ac5193e168fe18d74dc58 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 28 May 2020 14:19:04 +0000 Subject: [PATCH 523/898] Fix pep8 issues --- zaza/openstack/charm_tests/tempest/setup.py | 42 +++++++++---------- .../charm_tests/tempest/templates/accounts.py | 1 + .../tempest/templates/tempest_v2.py | 1 + .../tempest/templates/tempest_v3.py | 1 + zaza/openstack/charm_tests/tempest/tests.py | 10 +++-- 5 files changed, 30 insertions(+), 25 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 7f97014..9f442bc 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -16,7 +16,6 @@ import urllib.parse import os -import shutil import subprocess import zaza.model @@ -44,7 +43,7 @@ TEMPEST_CIRROS_ALT_IMAGE_NAME = 'cirros_alt' def get_app_access_ip(application_name): - """Get the application's access IP + """Get the application's access IP. :param application_name: Name of application :type application_name: str @@ -65,7 +64,7 @@ def get_app_access_ip(application_name): def add_application_ips(ctxt): - """Add application access IPs to context + """Add application access IPs to context. :param ctxt: Context dictionary :type ctxt: dict @@ -78,7 +77,7 @@ def add_application_ips(ctxt): def add_nova_config(ctxt, keystone_session): - """Add nova config to context + """Add nova config to context. :param ctxt: Context dictionary :type ctxt: dict @@ -97,7 +96,7 @@ def add_nova_config(ctxt, keystone_session): def add_neutron_config(ctxt, keystone_session): - """Add neutron config to context + """Add neutron config to context. :param ctxt: Context dictionary :type ctxt: dict @@ -142,7 +141,7 @@ def add_neutron_config(ctxt, keystone_session): def add_glance_config(ctxt, keystone_session): - """Add glance config to context + """Add glance config to context. :param ctxt: Context dictionary :type ctxt: dict @@ -164,7 +163,7 @@ def add_glance_config(ctxt, keystone_session): def add_cinder_config(ctxt, keystone_session): - """Add cinder config to context + """Add cinder config to context. :param ctxt: Context dictionary :type ctxt: dict @@ -184,7 +183,7 @@ def add_cinder_config(ctxt, keystone_session): def add_keystone_config(ctxt, keystone_session): - """Add keystone config to context + """Add keystone config to context. :param ctxt: Context dictionary :type ctxt: dict @@ -202,7 +201,7 @@ def add_keystone_config(ctxt, keystone_session): def add_environment_var_config(ctxt): - """Add environment variable config to context + """Add environment variable config to context. :param ctxt: Context dictionary :type ctxt: dict @@ -221,7 +220,7 @@ def add_environment_var_config(ctxt): def add_auth_config(ctxt): - """Add authorization config to context + """Add authorization config to context. :param ctxt: Context dictionary :type ctxt: dict @@ -235,12 +234,12 @@ def add_auth_config(ctxt): if overcloud_auth['API_VERSION'] == 3: ctxt['admin_project_name'] = overcloud_auth['OS_PROJECT_NAME'] ctxt['admin_domain_name'] = overcloud_auth['OS_DOMAIN_NAME'] - ctxt['default_credentials_domain_name'] = overcloud_auth[ - 'OS_PROJECT_DOMAIN_NAME'] + ctxt['default_credentials_domain_name'] = ( + overcloud_auth['OS_PROJECT_DOMAIN_NAME']) def get_tempest_context(): - """Generate the tempest config context + """Generate the tempest config context. :returns: Context dictionary :rtype: dict @@ -259,7 +258,7 @@ def get_tempest_context(): def render_tempest_config(target_file, ctxt, template): - """Render tempest config for specified config file and template + """Render tempest config for specified config file and template. :param target_file: Name of file to render config to :type target_file: str @@ -276,7 +275,7 @@ def render_tempest_config(target_file, ctxt, template): def setup_tempest(tempest_template, accounts_template): - """Initialize tempest and render tempest config + """Initialize tempest and render tempest config. :param tempest_template: tempest.conf template :type tempest_template: module @@ -287,7 +286,8 @@ def setup_tempest(tempest_template, accounts_template): """ if os.path.isdir('tempest-workspace'): try: - subprocess.check_call(['tempest', 'workspace', 'remove', '--rmdir', '--name', 'tempest-workspace']) + subprocess.check_call(['tempest', 'workspace', 'remove', '--rmdir', + '--name', 'tempest-workspace']) except subprocess.CalledProcessError: pass try: @@ -305,7 +305,7 @@ def setup_tempest(tempest_template, accounts_template): def render_tempest_config_keystone_v2(): - """Render tempest config for Keystone V2 API + """Render tempest config for Keystone V2 API. :returns: None :rtype: None @@ -314,7 +314,7 @@ def render_tempest_config_keystone_v2(): def render_tempest_config_keystone_v3(): - """Render tempest config for Keystone V3 API + """Render tempest config for Keystone V3 API. :returns: None :rtype: None @@ -323,7 +323,7 @@ def render_tempest_config_keystone_v3(): def add_cirros_alt_image(): - """Add cirros alternate image to overcloud + """Add cirros alternate image to overcloud. :returns: None :rtype: None @@ -336,7 +336,7 @@ def add_cirros_alt_image(): def add_tempest_flavors(): - """Add tempest flavors to overcloud + """Add tempest flavors to overcloud. :returns: None :rtype: None @@ -363,7 +363,7 @@ def add_tempest_flavors(): def add_tempest_roles(): - """Add tempest roles overcloud + """Add tempest roles overcloud. :returns: None :rtype: None diff --git a/zaza/openstack/charm_tests/tempest/templates/accounts.py b/zaza/openstack/charm_tests/tempest/templates/accounts.py index 3e5329d..0c5cf5a 100644 --- a/zaza/openstack/charm_tests/tempest/templates/accounts.py +++ b/zaza/openstack/charm_tests/tempest/templates/accounts.py @@ -1,3 +1,4 @@ +# flake8: noqa file_contents = """ - username: 'demo' tenant_name: 'demo' diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py index ecca5ce..c5649d0 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py @@ -1,3 +1,4 @@ +# flake8: noqa file_contents = """ [DEFAULT] debug = false diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index 7c95e3e..4f183dd 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -1,3 +1,4 @@ +# flake8: noqa file_contents = """ [DEFAULT] debug = false diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index bbc9d9e..d726930 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -20,15 +20,16 @@ import subprocess import zaza import zaza.charm_lifecycle.utils import zaza.charm_lifecycle.test -import tempest.cmd.main import tempfile + class TempestTest(): """Tempest test class.""" + test_runner = zaza.charm_lifecycle.test.DIRECT def run(self): - """Run tempest tests as specified in tests/tests.yaml + """Run tempest tests as specified in tests/tests.yaml. Test keys are parsed from ['tests_options']['tempest']['model'], where valid test keys are: smoke (bool), whitelist (list of tests), blacklist @@ -38,8 +39,9 @@ class TempestTest(): :rtype: bool """ charm_config = zaza.charm_lifecycle.utils.get_charm_config() - tempest_options = ['tempest', 'run', '--workspace', 'tempest-workspace', - '--config', 'tempest-workspace/etc/tempest.conf'] + tempest_options = ['tempest', 'run', '--workspace', + 'tempest-workspace', '--config', + 'tempest-workspace/etc/tempest.conf'] for model_alias in zaza.model.get_juju_model_aliases().keys(): tempest_test_key = model_alias if model_alias == zaza.charm_lifecycle.utils.DEFAULT_MODEL_ALIAS: From 820667011db87f4b0282dd55bf25095036484e9e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 28 May 2020 15:23:54 +0000 Subject: [PATCH 524/898] Add initial tempest branches to requirements.txt --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index e43c4e5..1446e9d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,3 +40,7 @@ paramiko sphinx sphinxcontrib-asyncio git+https://github.com/openstack-charmers/zaza#egg=zaza + +# Tempest tests +git+https://opendev.org/openstack/tempest.git#egg=tempest +git+https://opendev.org/openstack/cinder-tempest-plugin.git#egg=cinder-tempest-plugin From d84622d088bf3f17302a3f1c292fc3eb27a3efc4 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 29 May 2020 16:52:16 +0000 Subject: [PATCH 525/898] Drop tempest from requirements.txt as it doesn't support py35 --- requirements.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1446e9d..e43c4e5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,7 +40,3 @@ paramiko sphinx sphinxcontrib-asyncio git+https://github.com/openstack-charmers/zaza#egg=zaza - -# Tempest tests -git+https://opendev.org/openstack/tempest.git#egg=tempest -git+https://opendev.org/openstack/cinder-tempest-plugin.git#egg=cinder-tempest-plugin From 1beef2ddb2426b70160055432ff59f7369d7630b Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 29 May 2020 13:49:03 +0000 Subject: [PATCH 526/898] Port designate-bind expand and shrink test Port designate-bind expand and shrink test (and helpers) from mojo to zaza. The mojo designate-bind test is here *1 *1 https://github.com/openstack-charmers/openstack-mojo-specs/blob/master/helper/tests/expand_and_shrink_bind.py --- zaza/openstack/charm_tests/designate/tests.py | 62 ++++++ zaza/openstack/charm_tests/designate/utils.py | 205 ++++++++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 zaza/openstack/charm_tests/designate/utils.py diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 5583f0b..94c5f31 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -14,14 +14,19 @@ """Encapsulate designate testing.""" import logging +import unittest import tenacity import subprocess + import designateclient.v1.domains as domains import designateclient.v1.records as records import designateclient.v1.servers as servers + +import zaza.model import zaza.openstack.utilities.juju as zaza_juju import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.designate.utils as designate_utils class BaseDesignateTest(test_utils.OpenStackBaseTest): @@ -255,3 +260,60 @@ class DesignateTests(DesignateAPITests, DesignateCharmTests): """Collection of all Designate test classes.""" pass + + +class DesignateBindExpand(BaseDesignateTest): + """Test expanding and shrinking bind.""" + + TEST_DOMAIN = 'zazabindtesting.com.' + TEST_NS1_RECORD = 'ns1.{}'.format(TEST_DOMAIN) + TEST_NS2_RECORD = 'ns2.{}'.format(TEST_DOMAIN) + TEST_WWW_RECORD = "www.{}".format(TEST_DOMAIN) + TEST_RECORD = {TEST_WWW_RECORD: '10.0.0.24'} + + def test_expand_and_contract(self): + """Test expanding and shrinking bind.""" + if not self.post_xenial_queens: + raise unittest.SkipTest("Test not supported before Queens") + + domain = designate_utils.create_or_return_zone( + self.designate, + name=self.TEST_DOMAIN, + email="test@zaza.com") + + designate_utils.create_or_return_recordset( + self.designate, + domain['id'], + 'www', + 'A', + [self.TEST_RECORD[self.TEST_WWW_RECORD]]) + + # Test record is in bind and designate + designate_utils.check_dns_entry( + self.designate, + self.TEST_RECORD[self.TEST_WWW_RECORD], + self.TEST_DOMAIN, + record_name=self.TEST_WWW_RECORD) + + logging.debug('Adding a designate-bind unit') + zaza.model.add_unit('designate-bind') + zaza.model.block_until_all_units_idle() + + logging.debug('Performing DNS lookup on all units') + designate_utils.check_dns_entry( + self.designate, + self.TEST_RECORD[self.TEST_WWW_RECORD], + self.TEST_DOMAIN, + record_name=self.TEST_WWW_RECORD) + + units = zaza.model.get_status().applications['designate-bind']['units'] + doomed_unit = sorted(units.keys())[0] + logging.debug('Removing {}'.format(doomed_unit)) + zaza.model.destroy_unit('designate-bind', doomed_unit) + + logging.debug('Performing DNS lookup on all units') + designate_utils.check_dns_entry( + self.designate, + self.TEST_RECORD[self.TEST_WWW_RECORD], + self.TEST_DOMAIN, + record_name=self.TEST_WWW_RECORD) diff --git a/zaza/openstack/charm_tests/designate/utils.py b/zaza/openstack/charm_tests/designate/utils.py new file mode 100644 index 0000000..bde69ff --- /dev/null +++ b/zaza/openstack/charm_tests/designate/utils.py @@ -0,0 +1,205 @@ +"""Utilities for interacting with designate.""" + +import dns.resolver +import logging +import tenacity + +import designateclient.exceptions + +import zaza.model + + +def create_or_return_zone(client, name, email): + """Create zone or return matching existing zone. + + :param designate_client: Client to query designate + :type designate_client: designateclient.v2.Client + :param name: Name of zone + :type name: str + :param email: Email address to associate with zone. + :type email: str + :returns: Zone + :rtype: designateclient.v2.zones.Zone + """ + try: + zone = client.zones.create( + name=name, + email=email) + except designateclient.exceptions.Conflict: + logging.info('{} zone already exists.'.format(name)) + zones = [z for z in client.zones.list() if z['name'] == name] + assert len(zones) == 1, "Wrong number of zones found {}".format(zones) + zone = zones[0] + return zone + + +def create_or_return_recordset(client, zone_id, sub_domain, record_type, data): + """Create recordset or return matching existing recordset. + + :param designate_client: Client to query designate + :type designate_client: designateclient.v2.Client + :param zone_id: uuid of zone + :type zone_id: str + :param sub_domain: Subdomain to associate records with + :type sub_domain: str + :param data: Dictionary of entries eg {'www.test.com': '10.0.0.24'} + :type data: dict + :returns: RecordSet + :rtype: designateclient.v2.recordsets.RecordSet + """ + try: + rs = client.recordsets.create( + zone_id, + sub_domain, + record_type, + data) + except designateclient.exceptions.Conflict: + logging.info('{} record already exists.'.format(data)) + for r in client.recordsets.list(zone_id): + if r['name'].split('.')[0] == sub_domain: + rs = r + return rs + + +def get_designate_zone_objects(designate_client, domain_name=None, + domain_id=None): + """Get all domains matching a given domain_name or domain_id. + + :param designate_client: Client to query designate + :type designate_client: designateclient.v2.Client + :param domain_name: Name of domain to lookup + :type domain_name: str + :param domain_id: UUID of domain to lookup + :type domain_id: str + :returns: List of Domain objects matching domain_name or domain_id + :rtype: [designateclient.v2.domains.Domain,] + """ + all_zones = designate_client.zones.list() + a = [z for z in all_zones + if z['name'] == domain_name or z['id'] == domain_id] + return a + + +def get_designate_domain_object(designate_client, domain_name): + """Get the one and only domain matching the given domain_name. + + :param designate_client: Client to query designate + :type designate_client: designateclient.v2.Client + :param domain_name: Name of domain to lookup + :type domain_name:str + :returns: Domain with name domain_name + :rtype: designateclient.v2.domains.Domain + :raises: AssertionError + """ + dns_zone_id = get_designate_zone_objects(designate_client, + domain_name=domain_name) + msg = "Found {} domains for {}".format( + len(dns_zone_id), + domain_name) + assert len(dns_zone_id) == 1, msg + return dns_zone_id[0] + + +def get_designate_dns_records(designate_client, domain_name, ip): + """Look for records in designate that match the given ip. + + :param designate_client: Client to query designate + :type designate_client: designateclient.v2.Client + :param domain_name: Name of domain to lookup + :type domain_name:str + :returns: List of Record objects matching matching IP address + :rtype: [designateclient.v2.records.Record,] + """ + dns_zone = get_designate_domain_object(designate_client, domain_name) + return [r for r in designate_client.recordsets.list(dns_zone['id']) + if r['records'] == ip] + + +def check_dns_record_exists(dns_server_ip, query_name, expected_ip, + retry_count=3): + """Lookup a DNS record against the given dns server address. + + :param dns_server_ip: IP address to run query against + :type dns_server_ip: str + :param query_name: Record to lookup + :type query_name: str + :param expected_ip: IP address expected to be associated with record. + :type expected_ip: str + :param retry_count: Number of times to retry query. Useful if waiting + for record to propagate. + :type retry_count: int + :raises: AssertionError + """ + my_resolver = dns.resolver.Resolver() + my_resolver.nameservers = [dns_server_ip] + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(retry_count), + wait=tenacity.wait_exponential(multiplier=1, min=2, max=10), + reraise=True): + with attempt: + logging.info("Checking record {} against {}".format( + query_name, + dns_server_ip)) + answers = my_resolver.query(query_name) + for rdata in answers: + logging.info("Checking address returned by {} is correct".format( + dns_server_ip)) + assert str(rdata) == expected_ip + + +def check_dns_entry(des_client, ip, domain, record_name): + """Check that record for ip is in designate and in bind. + + :param ip: IP address to lookup + :type ip: str + :param domain_name: Domain to look for record in + :type domain_name:str + :param record_name: record name + :type record_name: str + """ + check_dns_entry_in_designate(des_client, [ip], domain, + record_name=record_name) + check_dns_entry_in_bind(ip, record_name) + + +def check_dns_entry_in_designate(des_client, ip, domain, record_name=None): + """Look for records in designate that match the given ip domain. + + :param designate_client: Client to query designate + :type designate_client: designateclient.v2.Client + :param ip: IP address to lookup in designate + :type ip: str + :param domain_name: Name of domain to lookup + :type domain_name:str + :param record_name: Retrieved record should have this name + :type record_name: str + :raises: AssertionError + """ + records = get_designate_dns_records(des_client, domain, ip) + assert records, "Record not found for {} in designate".format(ip) + logging.info('Found record in {} for {} in designate'.format(domain, ip)) + + if record_name: + recs = [r for r in records if r['name'] == record_name] + assert recs, "No DNS entry name matches expected name {}".format( + record_name) + logging.info('Found record in {} for {} in designate'.format( + domain, + record_name)) + + +def check_dns_entry_in_bind(ip, record_name, model_name=None): + """Check that record for ip address is in bind. + + :param ip: IP address to lookup + :type ip: str + :param record_name: record name + :type record_name: str + """ + for addr in zaza.model.get_app_ips('designate-bind', + model_name=model_name): + logging.info("Checking {} is {} against ({})".format( + record_name, + ip, + addr)) + check_dns_record_exists(addr, record_name, ip, retry_count=6) From 3a6002b61f3bfc86829f16569f7a624d0454bb0c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sat, 30 May 2020 16:56:25 +0000 Subject: [PATCH 527/898] Port aodh server alarm test from mojo Port over the aodh alarm test from mojo. Mojo source is here: *1 *1: https://github.com/openstack-charmers/openstack-mojo-specs/blob/master/helper/tests/validate_aodh.py --- zaza/openstack/charm_tests/aodh/tests.py | 79 +++++++++++++++++++++++- zaza/openstack/configure/telemetry.py | 24 +++++++ 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index c11e054..e0178df 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -19,7 +19,11 @@ import logging import tenacity +import novaclient.exceptions + import zaza.model +import zaza.openstack.configure.guest +import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.configure.telemetry as telemetry_utils @@ -33,7 +37,7 @@ class AodhTest(test_utils.OpenStackBaseTest): @classmethod def setUpClass(cls): """Run class setup for running tests.""" - super(AodhTest, cls).setUpClass() + super(AodhTest, cls).setUpClass(application_name='aodh') cls.xenial_ocata = openstack_utils.get_os_release('xenial_ocata') cls.xenial_newton = openstack_utils.get_os_release('xenial_newton') cls.bionic_stein = openstack_utils.get_os_release('bionic_stein') @@ -134,3 +138,76 @@ class AodhTest(test_utils.OpenStackBaseTest): pgrep_full=False): logging.info("Testing pause resume") self.query_aodh_api() + + +class AodhServerAlarmTest(test_utils.OpenStackBaseTest): + """Test server events trigger Aodh alarms.""" + + RESOURCE_PREFIX = 'zaza-aodhtests' + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(AodhServerAlarmTest, cls).setUpClass(application_name='aodh') + cls.aodh_client = openstack_utils.get_aodh_session_client( + cls.keystone_session) + cls.nova_client = openstack_utils.get_nova_session_client( + cls.keystone_session) + cls.run_resource_cleanup = True + + @classmethod + def resource_cleanup(cls): + """Remove test resources.""" + logging.info('Running teardown') + for alarm in cls.aodh_client.alarm.list(): + if alarm['name'].startswith(cls.RESOURCE_PREFIX): + logging.info('Removing Alarm {}'.format(alarm['name'])) + telemetry_utils.delete_alarm( + cls.aodh_client, + alarm['name'], + cache_wait=False) + for server in cls.nova_client.servers.list(): + if server.name.startswith(cls.RESOURCE_PREFIX): + logging.info('Removing server {}'.format(server.name)) + openstack_utils.delete_resource( + cls.nova_client.servers, + server.id, + msg="server") + + def test_alarm_on_power_off(self): + """Test server alarm is triggered when server is powered off.""" + server_name = '{}-server'.format(self.RESOURCE_PREFIX) + alarm_name = '{}_instance_off'.format(self.RESOURCE_PREFIX) + try: + server = self.nova_client.servers.find(name=server_name) + logging.info("Found existing server {}".format(server_name)) + except novaclient.exceptions.NotFound: + logging.info("Launching new server {}".format(server_name)) + server = zaza.openstack.configure.guest.launch_instance( + glance_setup.LTS_IMAGE_NAME, + vm_name=server_name) + assert server.status == 'ACTIVE', "Server {} not active".format( + server.name) + + logging.info('Deleting alarm {} if it exists'.format(alarm_name)) + telemetry_utils.delete_alarm( + self.aodh_client, + alarm_name, + cache_wait=True) + logging.info('Creating alarm {}'.format(alarm_name)) + alarm_info = telemetry_utils.create_server_power_off_alarm( + self.aodh_client, + alarm_name, + server.id) + alarm_state = telemetry_utils.get_alarm_state( + self.aodh_client, + alarm_info['alarm_id']) + logging.info('Alarm in state {}'.format(alarm_state)) + # Until data is collected alarm come up in an 'insufficient data' + # state. + self.assertEqual(alarm_state, 'insufficient data') + logging.info('Stopping server {}'.format(server.name)) + server.stop() + telemetry_utils.block_until_alarm_state( + self.aodh_client, + alarm_info['alarm_id']) diff --git a/zaza/openstack/configure/telemetry.py b/zaza/openstack/configure/telemetry.py index 86102ac..8bc01b8 100644 --- a/zaza/openstack/configure/telemetry.py +++ b/zaza/openstack/configure/telemetry.py @@ -18,6 +18,8 @@ Functions for managing masakari resources and simulating compute node loss and recovery. """ +import logging +import tenacity import time import zaza.model @@ -119,3 +121,25 @@ def create_server_power_off_alarm(aodh_client, alarm_name, server_uuid): 'type': 'string', 'value': server_uuid}]}} return aodh_client.alarm.create(alarm_def) + + +def block_until_alarm_state(aodh_client, alarm_id, target_state='alarm'): + """Block until alarm has reached target state. + + :param aodh_client: Authenticated aodh v2 client + :type aodh_client: aodhclient.v2.client.Client + :param alarm_id: ID of provided alarm + :type alarm_id: str + :param target_state: uuid of alarm to check + :stype target_state: str + """ + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential(multiplier=1, min=2, max=10)): + with attempt: + alarm_state = get_alarm_state( + aodh_client, + alarm_id) + + logging.info('Alarm in state {}'.format(alarm_state)) + assert alarm_state == target_state From 944335a39ff4f7cd3e74a5c61eaa86fe7ddf5b8f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 3 Jun 2020 07:19:11 +0000 Subject: [PATCH 528/898] Add dnspython dep and wait for add/remove units --- setup.py | 1 + zaza/openstack/charm_tests/designate/tests.py | 21 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 6d08832..a77d425 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ install_require = [ 'async_generator', 'boto3', 'cryptography', + 'dnspython', 'hvac<0.7.0', 'jinja2', 'juju', diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 94c5f31..3c56ee1 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -27,6 +27,7 @@ import zaza.openstack.utilities.juju as zaza_juju import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.designate.utils as designate_utils +import zaza.charm_lifecycle.utils as lifecycle_utils class BaseDesignateTest(test_utils.OpenStackBaseTest): @@ -273,6 +274,8 @@ class DesignateBindExpand(BaseDesignateTest): def test_expand_and_contract(self): """Test expanding and shrinking bind.""" + test_config = lifecycle_utils.get_charm_config(fatal=False) + states = test_config.get("target_deploy_status", {}) if not self.post_xenial_queens: raise unittest.SkipTest("Test not supported before Queens") @@ -295,11 +298,12 @@ class DesignateBindExpand(BaseDesignateTest): self.TEST_DOMAIN, record_name=self.TEST_WWW_RECORD) - logging.debug('Adding a designate-bind unit') - zaza.model.add_unit('designate-bind') + logging.info('Adding a designate-bind unit') + zaza.model.add_unit('designate-bind', wait_appear=True) zaza.model.block_until_all_units_idle() + zaza.model.wait_for_application_states(states=states) - logging.debug('Performing DNS lookup on all units') + logging.info('Performing DNS lookup on all units') designate_utils.check_dns_entry( self.designate, self.TEST_RECORD[self.TEST_WWW_RECORD], @@ -308,10 +312,15 @@ class DesignateBindExpand(BaseDesignateTest): units = zaza.model.get_status().applications['designate-bind']['units'] doomed_unit = sorted(units.keys())[0] - logging.debug('Removing {}'.format(doomed_unit)) - zaza.model.destroy_unit('designate-bind', doomed_unit) + logging.info('Removing {}'.format(doomed_unit)) + zaza.model.destroy_unit( + 'designate-bind', + doomed_unit, + wait_disappear=True) + zaza.model.block_until_all_units_idle() + zaza.model.wait_for_application_states(states=states) - logging.debug('Performing DNS lookup on all units') + logging.info('Performing DNS lookup on all units') designate_utils.check_dns_entry( self.designate, self.TEST_RECORD[self.TEST_WWW_RECORD], From a6b2a9799de9219c23cc06238fc151f9066c3184 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 4 Jun 2020 12:32:54 +0000 Subject: [PATCH 529/898] Fix hacluster tests Tidy up hacluster tests. In the process this fixes the error: ``` UnboundLocalError: local variable 'primary_status' referenced before assignment ``` This was caused by libjuju now returning an empty dict rather than None when listing a subordinates units. --- zaza/openstack/charm_tests/hacluster/tests.py | 76 ++----------------- 1 file changed, 6 insertions(+), 70 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index ebffb84..199d915 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -20,7 +20,6 @@ import logging import os import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.configure.hacluster @@ -35,78 +34,15 @@ class HaclusterTest(test_utils.OpenStackBaseTest): def test_900_action_cleanup(self): """The services can be cleaned up.""" - status = zaza.model.get_status().applications[self.application_name] - - # libjuju juju status no longer has units for subordinate charms - # Use the application it is subordinate-to to check workload status - if status.get("units") is None and status.get("subordinate-to"): - primary_status = juju_utils.get_application_status( - status.get("subordinate-to")[0]) - leader = None - for unit in primary_status["units"]: - if primary_status["units"][unit].get('leader'): - leader = unit - - if primary_status["units"][leader].get("subordinates"): - for subordinate in primary_status["units"][leader]["subordinates"]: - # mysql-router is a subordinate from focal onwards - _app = subordinate.split('/')[0] - if _app != 'hacluster': - continue - logging.info("Cleaning {}".format(subordinate)) - _action = "cleanup" - action_id = zaza.model.run_action(subordinate, "cleanup") - assert "success" in action_id.data["results"]["result"], ( - "Set hacluster action {} failed: {}" - .format(_action, action_id.data)) - - logging.info("Cleaning action w/resource {}" - .format(subordinate)) - params = {'resource': 'res_ks_haproxy'} - _action = "cleanup res_ks_haproxy" - zaza.model.run_action(subordinate, "cleanup", - action_params=params) - assert "success" in action_id.data["results"]["result"], ( - "Set hacluster action {} failed: {}" - .format(_action, action_id.data)) + zaza.model.run_action_on_leader( + self.application_name, + 'cleanup', + raise_on_failure=True) def test_910_pause_and_resume(self): """The services can be paused and resumed.""" - logging.debug('Checking pause and resume actions...') - - status = zaza.model.get_status().applications[self.application_name] - - # libjuju juju status no longer has units for subordinate charms - # Use the application it is subordinate-to to check workload status - if status.get("units") is None and status.get("subordinate-to"): - primary_status = juju_utils.get_application_status( - status.get("subordinate-to")[0]) - leader = None - for unit in primary_status["units"]: - if primary_status["units"][unit].get('leader'): - leader = unit - - if primary_status["units"][leader].get("subordinates"): - for subordinate in primary_status["units"][leader]["subordinates"]: - # mysql-router is a subordinate from focal onwards - _app = subordinate.split('/')[0] - if _app != 'hacluster': - continue - logging.info("Pausing {}".format(subordinate)) - zaza.model.run_action(subordinate, "pause") - zaza.model.block_until_unit_wl_status( - subordinate, - "maintenance") - - logging.info("Resuming {}".format(subordinate)) - zaza.model.run_action(subordinate, "resume") - zaza.model.block_until_unit_wl_status(subordinate, "active") - - _states = {"hacluster": { - "workload-status": "active", - "workload-status-message": "Unit is ready and clustered"}} - zaza.model.wait_for_application_states(states=_states) - logging.debug('OK') + with self.pause_resume([]): + logging.info("Testing pause resume") def _toggle_maintenance_and_wait(self, expected): """Configure cluster maintenance-mode. From 46d9887b45a63468f1aaf7e752d1c25ef5ee3dcb Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 5 Jun 2020 14:35:29 +0000 Subject: [PATCH 530/898] Move get_application_ip to zaza/openstack/utilities/juju.py --- zaza/openstack/charm_tests/tempest/setup.py | 28 +++------------------ zaza/openstack/utilities/juju.py | 21 ++++++++++++++++ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 9f442bc..f8772ea 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -20,6 +20,7 @@ import subprocess import zaza.model import zaza.utilities.deployment_env as deployment_env +import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.tempest.templates.tempest_v2 as tempest_v2 @@ -42,27 +43,6 @@ TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest' TEMPEST_CIRROS_ALT_IMAGE_NAME = 'cirros_alt' -def get_app_access_ip(application_name): - """Get the application's access IP. - - :param application_name: Name of application - :type application_name: str - :returns: Application's access IP - :rtype: str - """ - try: - app_config = zaza.model.get_application_config(application_name) - except KeyError: - return '' - vip = app_config.get("vip").get("value") - if vip: - ip = vip - else: - unit = zaza.model.get_units(application_name)[0] - ip = unit.public_address - return ip - - def add_application_ips(ctxt): """Add application access IPs to context. @@ -71,9 +51,9 @@ def add_application_ips(ctxt): :returns: None :rtype: None """ - ctxt['keystone'] = get_app_access_ip('keystone') - ctxt['dashboard'] = get_app_access_ip('openstack-dashboard') - ctxt['ncc'] = get_app_access_ip('nova-cloud-controller') + ctxt['keystone'] = juju_utils.get_application_ip('keystone') + ctxt['dashboard'] = juju_utils.get_application_ip('openstack-dashboard') + ctxt['ncc'] = juju_utils.get_application_ip('nova-cloud-controller') def add_nova_config(ctxt, keystone_session): diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index b3e54bc..191bc99 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -49,6 +49,27 @@ def get_application_status(application=None, unit=None, model_name=None): return status +def get_application_ip(application): + """Get the application's IP address. + + :param application: Application name + :type application: str + :returns: Application's IP address + :rtype: str + """ + try: + app_config = model.get_application_config(application) + except KeyError: + return '' + vip = app_config.get("vip").get("value") + if vip: + ip = vip + else: + unit = zaza.model.get_units(application_name)[0] + ip = unit.public_address + return ip + + def get_cloud_configs(cloud=None): """Get cloud configuration from local clouds.yaml. From 616f04f0bc8f67b1a744e3f8881a1fc3c88a9e97 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 5 Jun 2020 14:54:26 +0000 Subject: [PATCH 531/898] Move add_cirros_alt_image to glance setup.py and fix up loops --- zaza/openstack/charm_tests/glance/setup.py | 13 ++++++++ zaza/openstack/charm_tests/tempest/setup.py | 34 +++++---------------- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index c992917..8c7bd3e 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -18,6 +18,7 @@ import logging import zaza.openstack.utilities.openstack as openstack_utils CIRROS_IMAGE_NAME = "cirros" +CIRROS_ALT_IMAGE_NAME = "cirros_alt" LTS_RELEASE = "bionic" LTS_IMAGE_NAME = "bionic" @@ -77,6 +78,18 @@ def add_cirros_image(glance_client=None, image_name=None): image_name=image_name) +def add_cirros_alt_image(glance_client=None, image_name=None): + """Add alt cirros image to the current deployment. + + :param glance: Authenticated glanceclient + :type glance: glanceclient.Client + :param image_name: Label for the image in glance + :type image_name: str + """ + image_name = image_name or CIRROS_ALT_IMAGE_NAME + add_cirros_image(glance_client, image_name) + + def add_lts_image(glance_client=None, image_name=None, release=None): """Add an Ubuntu LTS image to the current deployment. diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index f8772ea..0e90597 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -40,7 +40,6 @@ SETUP_ENV_VARS = [ ] TEMPEST_FLAVOR_NAME = 'm1.tempest' TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest' -TEMPEST_CIRROS_ALT_IMAGE_NAME = 'cirros_alt' def add_application_ips(ctxt): @@ -89,14 +88,10 @@ def add_neutron_config(ctxt, keystone_session): focal_ussuri = openstack_utils.get_os_release('focal_ussuri') neutron_client = openstack_utils.get_neutron_session_client( keystone_session) - for net in neutron_client.list_networks()['networks']: - if net['name'] == 'ext_net': - ctxt['ext_net'] = net['id'] - break - for router in neutron_client.list_routers()['routers']: - if router['name'] == 'provider-router': - ctxt['provider_router_id'] = router['id'] - break + net = neutron_client.find_resource("network", "ext_net") + ctxt['ext_net'] = net['id'] + router = neutron_client.find_resource("router", "provider-router") + ctxt['provider_router_id'] = router['id'] # For focal+ with OVN, we use the same settings as upstream gate. # This is because the l3_agent_scheduler extension is only # applicable for OVN when conventional layer-3 agent enabled: @@ -135,7 +130,7 @@ def add_glance_config(ctxt, keystone_session): image = openstack_utils.get_images_by_name( glance_client, glance_setup.CIRROS_IMAGE_NAME) image_alt = openstack_utils.get_images_by_name( - glance_client, TEMPEST_CIRROS_ALT_IMAGE_NAME) + glance_client, glance_setup.CIRROS_ALT_IMAGE_NAME) if image: ctxt['image_id'] = image[0].id if image_alt: @@ -174,10 +169,8 @@ def add_keystone_config(ctxt, keystone_session): """ keystone_client = openstack_utils.get_keystone_session_client( keystone_session) - for domain in keystone_client.domains.list(): - if domain.name == 'admin_domain': - ctxt['default_domain_id'] = domain.id - break + domain = keystone_client.domains.find(name="admin_domain") + ctxt['default_domain_id'] = domain.id def add_environment_var_config(ctxt): @@ -302,19 +295,6 @@ def render_tempest_config_keystone_v3(): setup_tempest(tempest_v3, accounts) -def add_cirros_alt_image(): - """Add cirros alternate image to overcloud. - - :returns: None - :rtype: None - """ - image_url = openstack_utils.find_cirros_image(arch='x86_64') - glance_setup.add_image( - image_url, - glance_client=None, - image_name=TEMPEST_CIRROS_ALT_IMAGE_NAME) - - def add_tempest_flavors(): """Add tempest flavors to overcloud. From 3837b1ac85f4852c5314ab23e0180725a3e22c96 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 5 Jun 2020 15:54:33 +0000 Subject: [PATCH 532/898] Drop add_tempest_flavors in favor of using nova.setup.create_flavors --- zaza/openstack/charm_tests/nova/utils.py | 10 ++++++++ zaza/openstack/charm_tests/tempest/setup.py | 27 --------------------- 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/utils.py b/zaza/openstack/charm_tests/nova/utils.py index 974edd7..f2fcefc 100644 --- a/zaza/openstack/charm_tests/nova/utils.py +++ b/zaza/openstack/charm_tests/nova/utils.py @@ -35,5 +35,15 @@ FLAVORS = { 'ram': 8192, 'disk': 40, 'vcpus': 4}, + 'm1.tempest': { + 'flavorid': 5, + 'ram': 256, + 'disk': 1, + 'vcpus': 1}, + 'm2.tempest': { + 'flavorid': 6, + 'ram': 512, + 'disk': 1, + 'vcpus': 1}, } KEYPAIR_NAME = 'zaza' diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 0e90597..f93b3d3 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -295,33 +295,6 @@ def render_tempest_config_keystone_v3(): setup_tempest(tempest_v3, accounts) -def add_tempest_flavors(): - """Add tempest flavors to overcloud. - - :returns: None - :rtype: None - """ - keystone_session = openstack_utils.get_overcloud_keystone_session() - nova_client = openstack_utils.get_nova_session_client( - keystone_session) - try: - nova_client.flavors.create( - name=TEMPEST_FLAVOR_NAME, - ram=256, - vcpus=1, - disk=1) - except novaclient.exceptions.Conflict: - pass - try: - nova_client.flavors.create( - name=TEMPEST_ALT_FLAVOR_NAME, - ram=512, - vcpus=1, - disk=1) - except novaclient.exceptions.Conflict: - pass - - def add_tempest_roles(): """Add tempest roles overcloud. From 4e993e425167078a7fea63368b8a487440316c12 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 5 Jun 2020 17:56:10 +0000 Subject: [PATCH 533/898] Move add_tempest_roles to keystone setup.py --- .../charm_tests/keystone/__init__.py | 2 ++ zaza/openstack/charm_tests/keystone/setup.py | 30 +++++++++++++++++++ zaza/openstack/charm_tests/tempest/setup.py | 29 ++++-------------- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/__init__.py b/zaza/openstack/charm_tests/keystone/__init__.py index 6c4cca6..4d2e829 100644 --- a/zaza/openstack/charm_tests/keystone/__init__.py +++ b/zaza/openstack/charm_tests/keystone/__init__.py @@ -26,6 +26,8 @@ DEMO_ADMIN_USER_PASSWORD = 'password' DEMO_USER = 'demo' DEMO_PASSWORD = 'password' +TEMPEST_ROLES = ['member', 'ResellerAdmin'] + class BaseKeystoneTest(test_utils.OpenStackBaseTest): """Base for Keystone charm tests.""" diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index 688e3cd..6dbb7c1 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -14,6 +14,8 @@ """Code for setting up keystone.""" +import keystoneauth1 + import zaza.openstack.utilities.openstack as openstack_utils from zaza.openstack.charm_tests.keystone import ( BaseKeystoneTest, @@ -24,6 +26,7 @@ from zaza.openstack.charm_tests.keystone import ( DEMO_ADMIN_USER_PASSWORD, DEMO_USER, DEMO_PASSWORD, + TEMPEST_ROLES, ) @@ -115,3 +118,30 @@ def add_demo_user(): else: # create only V3 user _v3() + + +def _add_additional_roles(roles): + """Add additional roles to this deployment. + + :param ctxt: roles + :type ctxt: list + :returns: None + :rtype: None + """ + keystone_session = openstack_utils.get_overcloud_keystone_session() + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + for role_name in roles: + try: + keystone_client.roles.create(role_name) + except keystoneauth1.exceptions.http.Conflict: + pass + + +def add_tempest_roles(): + """Add tempest roles to this deployment. + + :returns: None + :rtype: None + """ + _add_additional_roles(TEMPEST_ROLES) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index f93b3d3..1b5b4de 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -27,9 +27,6 @@ import zaza.openstack.charm_tests.tempest.templates.tempest_v2 as tempest_v2 import zaza.openstack.charm_tests.tempest.templates.tempest_v3 as tempest_v3 import zaza.openstack.charm_tests.tempest.templates.accounts as accounts -import keystoneauth1 -import novaclient - SETUP_ENV_VARS = [ 'OS_TEST_GATEWAY', 'OS_TEST_CIDR_EXT', @@ -60,7 +57,7 @@ def add_nova_config(ctxt, keystone_session): :param ctxt: Context dictionary :type ctxt: dict - :returns keystone_session: keystoneauth1.session.Session object + :param keystone_session: keystoneauth1.session.Session object :type: keystoneauth1.session.Session :returns: None :rtype: None @@ -79,7 +76,7 @@ def add_neutron_config(ctxt, keystone_session): :param ctxt: Context dictionary :type ctxt: dict - :returns keystone_session: keystoneauth1.session.Session object + :param keystone_session: keystoneauth1.session.Session object :type: keystoneauth1.session.Session :returns: None :rtype: None @@ -120,7 +117,7 @@ def add_glance_config(ctxt, keystone_session): :param ctxt: Context dictionary :type ctxt: dict - :returns keystone_session: keystoneauth1.session.Session object + :param keystone_session: keystoneauth1.session.Session object :type: keystoneauth1.session.Session :returns: None :rtype: None @@ -142,7 +139,7 @@ def add_cinder_config(ctxt, keystone_session): :param ctxt: Context dictionary :type ctxt: dict - :returns keystone_session: keystoneauth1.session.Session object + :param keystone_session: keystoneauth1.session.Session object :type: keystoneauth1.session.Session :returns: None :rtype: None @@ -162,7 +159,7 @@ def add_keystone_config(ctxt, keystone_session): :param ctxt: Context dictionary :type ctxt: dict - :returns keystone_session: keystoneauth1.session.Session object + :param keystone_session: keystoneauth1.session.Session object :type: keystoneauth1.session.Session :returns: None :rtype: None @@ -293,19 +290,3 @@ def render_tempest_config_keystone_v3(): :rtype: None """ setup_tempest(tempest_v3, accounts) - - -def add_tempest_roles(): - """Add tempest roles overcloud. - - :returns: None - :rtype: None - """ - keystone_session = openstack_utils.get_overcloud_keystone_session() - keystone_client = openstack_utils.get_keystone_session_client( - keystone_session) - for role_name in ['member', 'ResellerAdmin']: - try: - keystone_client.roles.create(role_name) - except keystoneauth1.exceptions.http.Conflict: - pass From 7d0dfadc80016f31ac156c459e1bd85577c2bc31 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 5 Jun 2020 20:02:45 +0000 Subject: [PATCH 534/898] Fix incorrect reference to model --- zaza/openstack/utilities/juju.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index 191bc99..069801d 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -65,7 +65,7 @@ def get_application_ip(application): if vip: ip = vip else: - unit = zaza.model.get_units(application_name)[0] + unit = model.get_units(application_name)[0] ip = unit.public_address return ip From 5401be3a9700782d84bef0f78cdd3ccc06d6378a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 5 Jun 2020 20:04:51 +0000 Subject: [PATCH 535/898] Fix variable name --- zaza/openstack/utilities/juju.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index 069801d..3bdf4b8 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -65,7 +65,7 @@ def get_application_ip(application): if vip: ip = vip else: - unit = model.get_units(application_name)[0] + unit = model.get_units(application)[0] ip = unit.public_address return ip From 5d59fa43a5907eb5d939f05f43d0196b75905778 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 5 Jun 2020 22:55:24 +0000 Subject: [PATCH 536/898] Update flavor ids --- zaza/openstack/charm_tests/nova/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/utils.py b/zaza/openstack/charm_tests/nova/utils.py index f2fcefc..b16eef4 100644 --- a/zaza/openstack/charm_tests/nova/utils.py +++ b/zaza/openstack/charm_tests/nova/utils.py @@ -36,12 +36,12 @@ FLAVORS = { 'disk': 40, 'vcpus': 4}, 'm1.tempest': { - 'flavorid': 5, + 'flavorid': 6, 'ram': 256, 'disk': 1, 'vcpus': 1}, 'm2.tempest': { - 'flavorid': 6, + 'flavorid': 7, 'ram': 512, 'disk': 1, 'vcpus': 1}, From 2e6a2ef534d61e7dc11526eb8408aaf41dd07263 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sat, 6 Jun 2020 14:38:38 +0200 Subject: [PATCH 537/898] Add helper to format IP addresses appropriately The ``format_addr`` helper will allow you to write IP version agnostic code by encapsulating IPv6 addresses in brackets ('[]'). --- unit_tests/charm_tests/test_utils.py | 12 ++++++++++++ zaza/openstack/charm_tests/test_utils.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/unit_tests/charm_tests/test_utils.py b/unit_tests/charm_tests/test_utils.py index 280e2e2..a415360 100644 --- a/unit_tests/charm_tests/test_utils.py +++ b/unit_tests/charm_tests/test_utils.py @@ -30,3 +30,15 @@ class TestOpenStackBaseTest(unittest.TestCase): MyTestClass.setUpClass('foo', 'bar') _setUpClass.assert_called_with('foo', 'bar') + + +class TestUtils(unittest.TestCase): + + def test_format_addr(self): + self.assertEquals('1.2.3.4', test_utils.format_addr('1.2.3.4')) + self.assertEquals( + '[2001:db8::42]', test_utils.format_addr('2001:db8::42')) + with self.assertRaises(ValueError): + test_utils.format_addr('999.999.999.999') + with self.assertRaises(ValueError): + test_utils.format_addr('2001:db8::g') diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index bcb5aff..47c12d7 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -14,6 +14,7 @@ """Module containing base class for implementing charm tests.""" import contextlib import logging +import ipaddress import subprocess import unittest @@ -431,3 +432,20 @@ class OpenStackBaseTest(BaseCharmTest): cls.keystone_session = openstack_utils.get_overcloud_keystone_session( model_name=cls.model_name) cls.cacert = openstack_utils.get_cacert() + + +def format_addr(addr): + """Validate and format IP address. + + :param addr: IPv6 or IPv4 address + :type addr: str + :returns: Address string, optionally encapsulated in brackets([]) + :rtype: str + :raises: ValueError + """ + ipaddr = ipaddress.ip_address(addr) + if isinstance(ipaddr, ipaddress.IPv6Address): + fmt = '[{}]' + else: + fmt = '{}' + return fmt.format(ipaddr) From 37311df9c2592e9e0fa20a717aca17a4baece643 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sat, 6 Jun 2020 14:40:06 +0200 Subject: [PATCH 538/898] vault: fix formatting of IP addresses to support IPv6 At present an invalid URL to Vault will be produced in the event of the IP address used being an IPv6 address. --- zaza/openstack/charm_tests/vault/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index c05fcf6..a708d78 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -27,6 +27,7 @@ import yaml import collections import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils AUTH_FILE = "vault_tests.yaml" CharmVaultClient = collections.namedtuple( @@ -101,7 +102,7 @@ def get_unit_api_url(ip): transport = 'http' if vault_config['ssl-cert']['value']: transport = 'https' - return '{}://{}:8200'.format(transport, ip) + return '{}://{}:8200'.format(transport, test_utils.format_addr(ip)) def get_hvac_client(vault_url, cacert=None): From 7f6ed4665d83bd7ff243856e999c8ba718898a29 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 9 Jun 2020 14:01:27 +0000 Subject: [PATCH 539/898] Fix pep8 error --- zaza/openstack/charm_tests/tempest/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 1b5b4de..e6d7295 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -18,7 +18,6 @@ import urllib.parse import os import subprocess -import zaza.model import zaza.utilities.deployment_env as deployment_env import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils From bdc2e96e93339a2fa3ec5a292ce90288de9118b6 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 9 Jun 2020 14:22:26 +0000 Subject: [PATCH 540/898] Drop OS_ prefix from SETUP_ENV_VARS --- zaza/openstack/charm_tests/tempest/setup.py | 12 ++++++------ .../charm_tests/tempest/templates/tempest_v2.py | 6 +++--- .../charm_tests/tempest/templates/tempest_v3.py | 6 +++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index e6d7295..4266ddc 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -27,12 +27,12 @@ import zaza.openstack.charm_tests.tempest.templates.tempest_v3 as tempest_v3 import zaza.openstack.charm_tests.tempest.templates.accounts as accounts SETUP_ENV_VARS = [ - 'OS_TEST_GATEWAY', - 'OS_TEST_CIDR_EXT', - 'OS_TEST_FIP_RANGE', - 'OS_TEST_NAMESERVER', - 'OS_TEST_CIDR_PRIV', - 'OS_TEST_SWIFT_IP', + 'TEST_GATEWAY', + 'TEST_CIDR_EXT', + 'TEST_FIP_RANGE', + 'TEST_NAMESERVER', + 'TEST_CIDR_PRIV', + 'TEST_SWIFT_IP', ] TEMPEST_FLAVOR_NAME = 'm1.tempest' TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest' diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py index c5649d0..8b0f939 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py @@ -44,12 +44,12 @@ api_v2 = true api_v3 = false [image] -http_image = http://{os_test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz +http_image = http://{test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz [network] -project_network_cidr = {os_test_cidr_priv} +project_network_cidr = {test_cidr_priv} public_network_id = {ext_net} -dns_servers = {os_test_nameserver} +dns_servers = {test_nameserver} project_networks_reachable = false [network-feature-enabled] diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py index 4f183dd..5140deb 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py @@ -46,12 +46,12 @@ api_v2 = false api_v3 = true [image] -http_image = http://{os_test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz +http_image = http://{test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz [network] -project_network_cidr = {os_test_cidr_priv} +project_network_cidr = {test_cidr_priv} public_network_id = {ext_net} -dns_servers = {os_test_nameserver} +dns_servers = {test_nameserver} project_networks_reachable = false floating_network_name = {ext_net} From 9d9b47a27587df9db340a2c7eb87761f74c646ce Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 9 Jun 2020 15:06:01 +0000 Subject: [PATCH 541/898] Add test for networking when gateways are stopped Add test for networking when gateways are stopped. This includes refactoring the existing network test so I can reuse some of the code. --- zaza/openstack/charm_tests/neutron/tests.py | 191 ++++++++++++++++---- 1 file changed, 157 insertions(+), 34 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 6e2759b..7e28112 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -25,6 +25,8 @@ import logging import tenacity import unittest +import novaclient + import zaza import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.nova.utils as nova_utils @@ -598,8 +600,8 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): logging.info('Testing pause resume') -class NeutronNetworkingTest(unittest.TestCase): - """Ensure that openstack instances have valid networking.""" +class NeutronNetworkingBase(unittest.TestCase): + """Base for checking openstack instances have valid networking.""" RESOURCE_PREFIX = 'zaza-neutrontests' @@ -610,6 +612,8 @@ class NeutronNetworkingTest(unittest.TestCase): openstack_utils.get_overcloud_keystone_session()) cls.nova_client = ( openstack_utils.get_nova_session_client(cls.keystone_session)) + cls.neutron_client = ( + openstack_utils.get_neutron_session_client(cls.keystone_session)) # NOTE(fnordahl): in the event of a test failure we do not want to run # tear down code as it will make debugging a problem virtually # impossible. To alleviate each test method will set the @@ -629,38 +633,6 @@ class NeutronNetworkingTest(unittest.TestCase): server.id, msg="server") - def test_instances_have_networking(self): - """Validate North/South and East/West networking.""" - guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX)) - guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX)) - - instance_1 = self.nova_client.servers.find( - name='{}-ins-1'.format(self.RESOURCE_PREFIX)) - - instance_2 = self.nova_client.servers.find( - name='{}-ins-2'.format(self.RESOURCE_PREFIX)) - - def verify(stdin, stdout, stderr): - """Validate that the SSH command exited 0.""" - self.assertEqual(stdout.channel.recv_exit_status(), 0) - - # Verify network from 1 to 2 - self.validate_instance_can_reach_other(instance_1, instance_2, verify) - - # Verify network from 2 to 1 - self.validate_instance_can_reach_other(instance_2, instance_1, verify) - - # Validate tenant to external network routing - self.validate_instance_can_reach_router(instance_1, verify) - self.validate_instance_can_reach_router(instance_2, verify) - - # If we get here, it means the tests passed - self.run_tearDown = True - @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(8)) def validate_instance_can_reach_other(self, @@ -718,6 +690,99 @@ class NeutronNetworkingTest(unittest.TestCase): username, address, 'instance', 'ping -c 1 192.168.0.1', password=password, privkey=privkey, verify=verify) + @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60), + reraise=True, stop=tenacity.stop_after_attempt(8), + retry=tenacity.retry_if_exception_type(AssertionError)) + def check_server_state(self, nova_client, state, server_id=None, + server_name=None): + """Wait for server to reach desired state. + + :param nova_client: Nova client to use when checking status + :type nova_client: nova client + :param state: Target state for server + :type state: str + :param server_id: UUID of server to check + :type server_id: str + :param server_name: Name of server to check + :type server_name: str + :raises: AssertionError + """ + if server_name: + server_id = nova_client.servers.find(name=server_name).id + server = nova_client.servers.find(id=server_id) + assert server.status == state + + @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60), + reraise=True, stop=tenacity.stop_after_attempt(8), + retry=tenacity.retry_if_exception_type(AssertionError)) + def check_neutron_agent_up(self, neutron_client, host_name): + """Wait for agents to come up. + + :param neutron_client: Neutron client to use when checking status + :type neutron_client: neutron client + :param host_name: The name of the host whose agents need checking + :type host_name: str + :raises: AssertionError + """ + for agent in neutron_client.list_agents()['agents']: + if agent['host'] == host_name: + assert agent['admin_state_up'] + assert agent['alive'] + + def launch_guests(self): + """Launch two guests to use in tests.""" + guest.launch_instance( + glance_setup.LTS_IMAGE_NAME, + vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX)) + guest.launch_instance( + glance_setup.LTS_IMAGE_NAME, + vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX)) + + def retrieve_guest(self, nova_client, guest_name): + """Return guest matching name. + + :param nova_client: Nova client to use when checking status + :type nova_client: Nova client + :returns: the matching guest + :rtype: Union[novaclient.Server, None] + """ + try: + return nova_client.servers.find(name=guest_name) + except novaclient.exceptions.NotFound: + return None + + def retrieve_guests(self, nova_client): + """Return test guests. + + :param nova_client: Nova client to use when checking status + :type nova_client: Nova client + :returns: the matching guest + :rtype: Union[novaclient.Server, None] + """ + instance_1 = self.retrieve_guest( + nova_client, + '{}-ins-1'.format(self.RESOURCE_PREFIX)) + instance_2 = self.retrieve_guest( + nova_client, + '{}-ins-1'.format(self.RESOURCE_PREFIX)) + return instance_1, instance_2 + + def check_connectivity(self, instance_1, instance_2): + """Run North/South and East/West connectivity tests.""" + def verify(stdin, stdout, stderr): + """Validate that the SSH command exited 0.""" + self.assertEqual(stdout.channel.recv_exit_status(), 0) + + # Verify network from 1 to 2 + self.validate_instance_can_reach_other(instance_1, instance_2, verify) + + # Verify network from 2 to 1 + self.validate_instance_can_reach_other(instance_2, instance_1, verify) + + # Validate tenant to external network routing + self.validate_instance_can_reach_router(instance_1, verify) + self.validate_instance_can_reach_router(instance_2, verify) + def floating_ips_from_instance(instance): """ @@ -764,3 +829,61 @@ def ips_from_instance(instance, ip_type): return list([ ip['addr'] for ip in instance.addresses['private'] if ip['OS-EXT-IPS:type'] == ip_type]) + + +class NeutronNetworkingTest(NeutronNetworkingBase): + """Ensure that openstack instances have valid networking.""" + + def test_instances_have_networking(self): + """Validate North/South and East/West networking.""" + self.launch_guests() + instance_1, instance_2 = self.retrieve_guests(self.nova_client) + self.check_connectivity(instance_1, instance_2) + self.run_tearDown = True + + +class NeutronNetworkingVRRPTests(NeutronNetworkingBase): + """Check networking when gateways are restarted.""" + + def test_gateway_failure(self): + """Validate networking in the case of a gateway failure.""" + instance_1, instance_2 = self.retrieve_guests(self.nova_client) + if not all([instance_1, instance_2]): + self.launch_guests() + instance_1, instance_2 = self.retrieve_guests(self.nova_client) + self.check_connectivity(instance_1, instance_2) + + routers = self.neutron_client.list_routers( + name='provider-router')['routers'] + assert len(routers) == 1, "Unexpected router count {}".format( + len(routers)) + provider_router = routers[0] + l3_agents = self.neutron_client.list_l3_agent_hosting_routers( + router=provider_router['id'])['agents'] + logging.info( + 'Checking there are multiple L3 agents running tenant router') + assert len(l3_agents) == 2, "Unexpected l3 agent count {}".format( + len(l3_agents)) + uc_ks_session = openstack_utils.get_undercloud_keystone_session() + uc_nova_client = openstack_utils.get_nova_session_client(uc_ks_session) + uc_neutron_client = openstack_utils.get_neutron_session_client( + uc_ks_session) + for agent in l3_agents: + gateway_hostname = agent['host'] + gateway_server = uc_nova_client.servers.find(name=gateway_hostname) + logging.info("Shutting down {}".format(gateway_hostname)) + gateway_server.stop() + self.check_server_state( + uc_nova_client, + 'SHUTOFF', + server_name=gateway_hostname) + self.check_connectivity(instance_1, instance_2) + gateway_server.start() + self.check_server_state( + uc_nova_client, + 'ACTIVE', + server_name=gateway_hostname) + self.check_neutron_agent_up( + uc_neutron_client, + gateway_hostname) + self.check_connectivity(instance_1, instance_2) From 9844701d1984f181f6215202a36ac5e754bac714 Mon Sep 17 00:00:00 2001 From: Dmitrii Shcherbakov Date: Wed, 10 Jun 2020 14:18:29 +0300 Subject: [PATCH 542/898] Wait until a share status reaches 'available' (#310) * Wait until a share status reaches 'available' It appears to be that the test_manila_share test case does not wait until a share becomes available which results in failures like this: manilaclient.common.apiclient.exceptions.BadRequest: New access rules cannot be applied while the share or any of its replicas or migration copies lacks a valid host or is in an invalid state. (HTTP 400) (Request-ID: req-8d609e13-9a80-428b-953b-17ab8d0e0cae) Fixes: #309 --- zaza/openstack/charm_tests/manila_ganesha/tests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index e24a9f3..2f9f8be 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -74,6 +74,15 @@ packages: fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] + # Wait for the created share to become available before it gets used. + openstack_utils.resource_reaches_status( + self.manila_client.shares, + share.id, + wait_iteration_max_time=120, + stop_after_attempt=2, + expected_status="available", + msg="Waiting for a share to become available") + share.allow(access_type='ip', access=fip_1, access_level='rw') share.allow(access_type='ip', access=fip_2, access_level='rw') From 8336b75c4029717b0aad09259169265fe39df2cf Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 11 Jun 2020 10:01:00 +0000 Subject: [PATCH 543/898] Add test to perform charm upgrade. Add test to perform charm upgrade and a few small tweaks to the upgrade utils. --- .../charm_tests/charm_upgrade/__init__.py | 15 ++++ .../charm_tests/charm_upgrade/tests.py | 78 +++++++++++++++++++ zaza/openstack/utilities/upgrade_utils.py | 57 +++++++++----- 3 files changed, 130 insertions(+), 20 deletions(-) create mode 100644 zaza/openstack/charm_tests/charm_upgrade/__init__.py create mode 100644 zaza/openstack/charm_tests/charm_upgrade/tests.py diff --git a/zaza/openstack/charm_tests/charm_upgrade/__init__.py b/zaza/openstack/charm_tests/charm_upgrade/__init__.py new file mode 100644 index 0000000..fc5baa5 --- /dev/null +++ b/zaza/openstack/charm_tests/charm_upgrade/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Test charm upgrade.""" diff --git a/zaza/openstack/charm_tests/charm_upgrade/tests.py b/zaza/openstack/charm_tests/charm_upgrade/tests.py new file mode 100644 index 0000000..0ffd30b --- /dev/null +++ b/zaza/openstack/charm_tests/charm_upgrade/tests.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define class for Charm Upgrade.""" + +import logging +import unittest + +import zaza.model +from zaza.openstack.utilities import ( + cli as cli_utils, + upgrade_utils as upgrade_utils, +) +from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest + + +class FullCloudCharmUpgradeTest(unittest.TestCase): + """Class to encapsulate Charm Upgrade Tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for Charm Upgrades.""" + cli_utils.setup_logging() + cls.lts = LTSGuestCreateTest() + cls.target_charm_namespace = '~openstack-charmers-next' + + def get_upgrade_url(self, charm_url): + """Return the charm_url to upgrade to. + + :param charm_url: Current charm url. + :type charm_url: str + """ + charm_name = upgrade_utils.extract_charm_name_from_url( + charm_url) + next_charm_url = zaza.model.get_latest_charm_url( + "cs:{}/{}".format(self.target_charm_namespace, charm_name)) + return next_charm_url + + def test_200_run_charm_upgrade(self): + """Run charm upgrade.""" + self.lts.test_launch_small_instance() + applications = zaza.model.get_status().applications + groups = upgrade_utils.get_charm_upgrade_groups() + for group_name, group in groups.items(): + logging.info("About to upgrade {} ({})".format(group_name, group)) + for application, app_details in applications.items(): + if application not in group: + continue + target_url = self.get_upgrade_url(app_details['charm']) + if target_url == app_details['charm']: + logging.warn( + "Skipping upgrade of {}, already using {}".format( + application, + target_url)) + else: + logging.info("Upgrading {} to {}".format( + application, + target_url)) + zaza.model.upgrade_charm( + application, + switch=target_url) + logging.info("Waiting for charm url to update") + zaza.model.block_until_charm_url(application, target_url) + zaza.model.block_until_all_units_idle() + self.lts.test_launch_small_instance() diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 4066d19..b134a69 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -20,7 +20,8 @@ import zaza.model SERVICE_GROUPS = collections.OrderedDict([ - ('Stateful Services', ['percona-cluster', 'rabbitmq-server', 'ceph-mon']), + ('Stateful Services', ['percona-cluster', 'rabbitmq-server', 'ceph-mon', + 'mysql-innodb-cluster']), ('Core Identity', ['keystone']), ('Control Plane', [ 'aodh', 'barbican', 'ceilometer', 'ceph-fs', @@ -92,6 +93,19 @@ def _filter_non_openstack_services(app, app_config, model_name=None): return False +def _apply_extra_filters(filters, extra_filters): + if extra_filters: + if isinstance(extra_filters, list): + filters.extend(extra_filters) + elif callable(extra_filters): + filters.append(extra_filters) + else: + raise RuntimeError( + "extra_filters should be a list of " + "callables") + return filters + + def get_upgrade_groups(model_name=None, extra_filters=None): """Place apps in the model into their upgrade groups. @@ -108,18 +122,10 @@ def get_upgrade_groups(model_name=None, extra_filters=None): _filter_openstack_upgrade_list, _filter_non_openstack_services, ] - if extra_filters: - if isinstance(extra_filters, list): - filters.extend(extra_filters) - elif callable(extra_filters): - filters.append(extra_filters) - else: - raise RuntimeError( - "extra_filters should be a list of " - "callables") + filters = _apply_extra_filters(filters, extra_filters) apps_in_model = get_upgrade_candidates( model_name=model_name, - filters=filters,) + filters=filters) return _build_service_groups(apps_in_model) @@ -136,15 +142,26 @@ def get_series_upgrade_groups(model_name=None, extra_filters=None): :rtype: collections.OrderedDict """ filters = [_filter_subordinates] - if extra_filters: - if isinstance(extra_filters, list): - filters.extend(extra_filters) - elif callable(extra_filters): - filters.append(extra_filters) - else: - raise RuntimeError( - "extra_filters should be a list of " - "callables") + filters = _apply_extra_filters(filters, extra_filters) + apps_in_model = get_upgrade_candidates( + model_name=model_name, + filters=filters) + + return _build_service_groups(apps_in_model) + + +def get_charm_upgrade_groups(model_name=None, extra_filters=None): + """Place apps in the model into their upgrade groups for a charm upgrade. + + Place apps in the model into their upgrade groups. If an app is deployed + but is not in SERVICE_GROUPS then it is placed in a sweep_up group. + + :param model_name: Name of model to query. + :type model_name: str + :returns: Dict of group lists keyed on group name. + :rtype: collections.OrderedDict + """ + filters = _apply_extra_filters([], extra_filters) apps_in_model = get_upgrade_candidates( model_name=model_name, filters=filters) From d837c0ed97b9bc9d4401c56f040c63371acbca89 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 11 Jun 2020 14:12:16 +0000 Subject: [PATCH 544/898] Automatically enable/disable config for tempest. Automatically enable/disable config for tempest based on the contents of the keystone service catalogue. --- zaza/openstack/charm_tests/tempest/setup.py | 90 ++++++++++------ .../templates/{accounts.py => accounts.j2} | 3 - .../{tempest_v2.py => tempest_v2.j2} | 56 +++++----- .../tempest/templates/tempest_v3.j2 | 101 ++++++++++++++++++ .../tempest/templates/tempest_v3.py | 100 ----------------- 5 files changed, 187 insertions(+), 163 deletions(-) rename zaza/openstack/charm_tests/tempest/templates/{accounts.py => accounts.j2} (76%) rename zaza/openstack/charm_tests/tempest/templates/{tempest_v2.py => tempest_v2.j2} (57%) create mode 100644 zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 delete mode 100644 zaza/openstack/charm_tests/tempest/templates/tempest_v3.py diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 4266ddc..8e338f6 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -14,6 +14,7 @@ """Code for configuring and initializing tempest.""" +import jinja2 import urllib.parse import os import subprocess @@ -22,20 +23,18 @@ import zaza.utilities.deployment_env as deployment_env import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.glance.setup as glance_setup -import zaza.openstack.charm_tests.tempest.templates.tempest_v2 as tempest_v2 -import zaza.openstack.charm_tests.tempest.templates.tempest_v3 as tempest_v3 -import zaza.openstack.charm_tests.tempest.templates.accounts as accounts -SETUP_ENV_VARS = [ - 'TEST_GATEWAY', - 'TEST_CIDR_EXT', - 'TEST_FIP_RANGE', - 'TEST_NAMESERVER', - 'TEST_CIDR_PRIV', - 'TEST_SWIFT_IP', -] +SETUP_ENV_VARS = { + 'neutron': ['TEST_GATEWAY', 'TEST_CIDR_EXT', 'TEST_FIP_RANGE', + 'TEST_NAMESERVER', 'TEST_CIDR_PRIV'], + 'swift': ['TEST_SWIFT_IP'], +} + TEMPEST_FLAVOR_NAME = 'm1.tempest' TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest' +TEMPEST_SVC_LIST = ['ceilometer', 'cinder', 'glance', 'heat', 'horizon', + 'ironic', 'neutron', 'nova', 'sahara', 'swift', 'trove', + 'zaqar'] def add_application_ips(ctxt): @@ -169,7 +168,20 @@ def add_keystone_config(ctxt, keystone_session): ctxt['default_domain_id'] = domain.id -def add_environment_var_config(ctxt): +def get_service_list(keystone_session): + """Add keystone config to context. + + :param keystone_session: keystoneauth1.session.Session object + :type: keystoneauth1.session.Session + :returns: None + :rtype: None + """ + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + return [s.name for s in keystone_client.services.list() if s.enabled] + + +def add_environment_var_config(ctxt, services): """Add environment variable config to context. :param ctxt: Context dictionary @@ -178,14 +190,16 @@ def add_environment_var_config(ctxt): :rtype: None """ deploy_env = deployment_env.get_deployment_context() - for var in SETUP_ENV_VARS: - value = deploy_env.get(var) - if value: - ctxt[var.lower()] = value - else: - raise ValueError( - ("Environment variables {} must all be set to run this" - " test").format(', '.join(SETUP_ENV_VARS))) + for svc, env_vars in SETUP_ENV_VARS.items(): + if svc in services: + for var in env_vars: + value = deploy_env.get(var) + if value: + ctxt[var.lower()] = value + else: + raise ValueError( + ("Environment variables {} must all be set to run this" + " test").format(', '.join(env_vars))) def add_auth_config(ctxt): @@ -215,32 +229,42 @@ def get_tempest_context(): """ keystone_session = openstack_utils.get_overcloud_keystone_session() ctxt = {} + ctxt_funcs = { + 'nova': add_nova_config, + 'neutron': add_nova_config, + 'glance': add_keystone_config, + 'cinder': add_cinder_config, + 'keystone': add_keystone_config} + ctxt['enabled_services'] = get_service_list(keystone_session) + ctxt['disabled_services'] = list( + set(TEMPEST_SVC_LIST) - set(ctxt['enabled_services'])) add_application_ips(ctxt) - add_nova_config(ctxt, keystone_session) - add_neutron_config(ctxt, keystone_session) - add_glance_config(ctxt, keystone_session) - add_cinder_config(ctxt, keystone_session) - add_keystone_config(ctxt, keystone_session) - add_environment_var_config(ctxt) + for svc_name, ctxt_func in ctxt_funcs.items(): + if svc_name in ctxt['enabled_services']: + ctxt_func(ctxt, keystone_session) + add_environment_var_config(ctxt, ctxt['enabled_services']) add_auth_config(ctxt) return ctxt -def render_tempest_config(target_file, ctxt, template): +def render_tempest_config(target_file, ctxt, template_name): """Render tempest config for specified config file and template. :param target_file: Name of file to render config to :type target_file: str :param ctxt: Context dictionary :type ctxt: dict - :param template: Template module - :type template: module + :param template_name: Name of template file + :type template_name: str :returns: None :rtype: None """ - # TODO: switch to jinja2 and generate config based on available services + jenv = jinja2.Environment(loader=jinja2.PackageLoader( + 'zaza.openstack', + 'charm_tests/tempest/templates')) + template = jenv.get_template(template_name) with open(target_file, 'w') as f: - f.write(template.file_contents.format(**ctxt)) + f.write(template.render(ctxt)) def setup_tempest(tempest_template, accounts_template): @@ -279,7 +303,7 @@ def render_tempest_config_keystone_v2(): :returns: None :rtype: None """ - setup_tempest(tempest_v2, accounts) + setup_tempest('tempest_v2.j2', 'accounts.j2') def render_tempest_config_keystone_v3(): @@ -288,4 +312,4 @@ def render_tempest_config_keystone_v3(): :returns: None :rtype: None """ - setup_tempest(tempest_v3, accounts) + setup_tempest('tempest_v3.j2', 'accounts.j2') diff --git a/zaza/openstack/charm_tests/tempest/templates/accounts.py b/zaza/openstack/charm_tests/tempest/templates/accounts.j2 similarity index 76% rename from zaza/openstack/charm_tests/tempest/templates/accounts.py rename to zaza/openstack/charm_tests/tempest/templates/accounts.j2 index 0c5cf5a..c4dd21a 100644 --- a/zaza/openstack/charm_tests/tempest/templates/accounts.py +++ b/zaza/openstack/charm_tests/tempest/templates/accounts.j2 @@ -1,9 +1,6 @@ -# flake8: noqa -file_contents = """ - username: 'demo' tenant_name: 'demo' password: 'pass' - username: 'alt_demo' tenant_name: 'alt_demo' password: 'secret' -""" diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 similarity index 57% rename from zaza/openstack/charm_tests/tempest/templates/tempest_v2.py rename to zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 index 8b0f939..b4f2dfc 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.py +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 @@ -1,5 +1,3 @@ -# flake8: noqa -file_contents = """ [DEFAULT] debug = false use_stderr = false @@ -8,16 +6,17 @@ log_file = tempest.log [auth] test_accounts_file = accounts.yaml default_credentials_domain_name = Default -admin_username = {admin_username} +admin_username = {{ admin_username }} admin_project_name = admin -admin_password = {admin_password} +admin_password = {{ admin_password }} admin_domain_name = Default +{% if 'nova' in enabled_services %} [compute] -image_ref = {image_id} -image_ref_alt = {image_alt_id} -flavor_ref = {flavor_ref} -flavor_ref_alt = {flavor_ref_alt} +image_ref = {{ image_id }} +image_ref_alt = {{ image_alt_id }} +flavor_ref = {{ flavor_ref }} +flavor_ref_alt = {{ flavor_ref_alt }} region = RegionOne min_compute_nodes = 3 @@ -31,9 +30,11 @@ resize = true live_migration = true block_migration_for_live_migration = true attach_encrypted_volume = false +{% endif %} +{% if 'keystone' in enabled_services %} [identity] -uri = {proto}://{keystone}:5000/v2.0 +uri = {proto}://{{ keystone }}:5000/v2.0 auth_version = v2 admin_role = Admin region = RegionOne @@ -42,23 +43,28 @@ disable_ssl_certificate_validation = true [identity-feature-enabled] api_v2 = true api_v3 = false +{% endif %} [image] -http_image = http://{test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz +http_image = http://{{ test_swift_ip }}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz +{% if 'neutron' in enabled_services %} [network] -project_network_cidr = {test_cidr_priv} -public_network_id = {ext_net} -dns_servers = {test_nameserver} +project_network_cidr = {{ test_cidr_priv }} +public_network_id = {{ ext_net }} +dns_servers = {{ test_nameserver }} project_networks_reachable = false [network-feature-enabled] ipv6 = false +{% endif %} +{% if 'heat' in enabled_services %} [orchestration] stack_owner_role = Admin instance_type = m1.small keypair_name = testkey +{% endif %} [oslo_concurrency] lock_path = /tmp @@ -74,23 +80,19 @@ run_validation = true image_ssh_user = cirros [service_available] -ceilometer = true -cinder = true -glance = true -heat = true -horizon = true -ironic = false -neutron = true -nova = true -sahara = false -swift = true -trove = false -zaqar = false +{% for svc in enabled_services -%} +{{ svc }} = true +{% endfor -%} +{% for svc in disabled_services -%} +{{ svc }} = false +{% endfor %} +{% if 'cinder' in enabled_services %} [volume] backend_names = cinder-ceph storage_protocol = ceph -catalog_type = {catalog_type} +catalog_type = {{ catalog_type }} [volume-feature-enabled] -backup = false""" +backup = false +{% endif %} diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 new file mode 100644 index 0000000..dc7d8db --- /dev/null +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 @@ -0,0 +1,101 @@ +[DEFAULT] +debug = false +use_stderr = false +log_file = tempest.log + +[auth] +test_accounts_file = accounts.yaml +default_credentials_domain_name = {{ default_credentials_domain_name }} +admin_username = {{ admin_username }} +admin_project_name = {{ admin_project_name }} +admin_password = {{ admin_password }} +admin_domain_name = {{ admin_domain_name }} + +{% if 'nova' in enabled_services %} +[compute] +image_ref = {{ image_id }} +image_ref_alt = {{ image_alt_id }} +flavor_ref = {{ flavor_ref }} +flavor_ref_alt = {{ flavor_ref_alt }} +min_compute_nodes = 3 + +# TODO: review this as its release specific +# min_microversion = 2.2 +# max_microversion = latest + +[compute-feature-enabled] +console_output = true +resize = true +live_migration = true +block_migration_for_live_migration = true +attach_encrypted_volume = false +{% endif %} + +{% if 'keystone' in enabled_services %} +[identity] +uri = {{ proto }}://{{ keystone }}:5000/v2.0 +uri_v3 = {{ proto }}://{{ keystone }}:5000/v3 +auth_version = v3 +admin_role = Admin +region = RegionOne +default_domain_id = {{ default_domain_id }} +admin_domain_scope = true +disable_ssl_certificate_validation = true + +[identity-feature-enabled] +api_v2 = false +api_v3 = true +{% endif %} + +[image] +http_image = http://{{ test_swift_ip }}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz + +{% if 'neutron' in enabled_services %} +[network] +project_network_cidr = {{ test_cidr_priv }} +public_network_id = {{ ext_net }} +dns_servers = {{ test_nameserver }} +project_networks_reachable = false +floating_network_name = {{ ext_net }} + +[network-feature-enabled] +ipv6 = false +api_extensions = {{ neutron_api_extensions }} +{% endif %} + +{% if 'heat' in enabled_services %} +[orchestration] +stack_owner_role = Admin +instance_type = m1.small +keypair_name = testkey +{% endif %} + +[oslo_concurrency] +lock_path = /tmp + +[scenario] +img_dir = /home/ubuntu/images +img_file = cirros-0.3.4-x86_64-disk.img +img_container_format = bare +img_disk_format = qcow2 + +[validation] +run_validation = true +image_ssh_user = cirros + +[service_available] +{% for svc in enabled_services -%} +{{ svc }} = true +{% endfor -%} +{% for svc in disabled_services -%} +{{ svc }} = false +{% endfor %} + +{% if 'cinder' in enabled_services %} +[volume] +backend_names = cinder-ceph +storage_protocol = ceph +catalog_type = {{ catalog_type }} + +[volume-feature-enabled] +{% endif %} diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py deleted file mode 100644 index 5140deb..0000000 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.py +++ /dev/null @@ -1,100 +0,0 @@ -# flake8: noqa -file_contents = """ -[DEFAULT] -debug = false -use_stderr = false -log_file = tempest.log - -[auth] -test_accounts_file = accounts.yaml -default_credentials_domain_name = {default_credentials_domain_name} -admin_username = {admin_username} -admin_project_name = {admin_project_name} -admin_password = {admin_password} -admin_domain_name = {admin_domain_name} - -[compute] -image_ref = {image_id} -image_ref_alt = {image_alt_id} -flavor_ref = {flavor_ref} -flavor_ref_alt = {flavor_ref_alt} -min_compute_nodes = 3 - -# TODO: review this as its release specific -# min_microversion = 2.2 -# max_microversion = latest - -[compute-feature-enabled] -console_output = true -resize = true -live_migration = true -block_migration_for_live_migration = true -attach_encrypted_volume = false - -[identity] -uri = {proto}://{keystone}:5000/v2.0 -uri_v3 = {proto}://{keystone}:5000/v3 -auth_version = v3 -admin_role = Admin -region = RegionOne -default_domain_id = {default_domain_id} -admin_domain_scope = true -disable_ssl_certificate_validation = true - -[identity-feature-enabled] -api_v2 = false -api_v3 = true - -[image] -http_image = http://{test_swift_ip}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz - -[network] -project_network_cidr = {test_cidr_priv} -public_network_id = {ext_net} -dns_servers = {test_nameserver} -project_networks_reachable = false -floating_network_name = {ext_net} - -[network-feature-enabled] -ipv6 = false -api_extensions = {neutron_api_extensions} - -[orchestration] -stack_owner_role = Admin -instance_type = m1.small -keypair_name = testkey - -[oslo_concurrency] -lock_path = /tmp - -[scenario] -img_dir = /home/ubuntu/images -img_file = cirros-0.3.4-x86_64-disk.img -img_container_format = bare -img_disk_format = qcow2 - -[validation] -run_validation = true -image_ssh_user = cirros - -[service_available] -ceilometer = true -cinder = true -glance = true -heat = true -horizon = true -ironic = false -neutron = true -nova = true -sahara = false -swift = true -trove = false -zaqar = false - -[volume] -backend_names = cinder-ceph -storage_protocol = ceph -catalog_type = {catalog_type} - -[volume-feature-enabled] -backup = false""" From effbf131908f67c7eda55eeb56f1a621292bfef2 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 10 Jun 2020 08:29:50 +0200 Subject: [PATCH 545/898] Add CephFS pause/resume test --- zaza/openstack/charm_tests/ceph/fs/tests.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/fs/tests.py b/zaza/openstack/charm_tests/ceph/fs/tests.py index 63cc492..4b0842b 100644 --- a/zaza/openstack/charm_tests/ceph/fs/tests.py +++ b/zaza/openstack/charm_tests/ceph/fs/tests.py @@ -14,6 +14,7 @@ """Encapsulate CephFS testing.""" +import logging from tenacity import Retrying, stop_after_attempt, wait_exponential import zaza.model as model @@ -124,3 +125,18 @@ write_files: def _indent(text, amount, ch=' '): padding = amount * ch return ''.join(padding+line for line in text.splitlines(True)) + + +class CharmOperationTest(test_utils.BaseCharmTest): + """CephFS Charm operation tests.""" + + def test_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped, then resume and check + they are started. + """ + services = ['ceph-mds'] + with self.pause_resume(services): + logging.info('Testing pause resume (services="{}")' + .format(services)) From b95c79c9e3f4d0b373b604eb2e165942b5a12c05 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 11 Jun 2020 15:53:51 +0000 Subject: [PATCH 546/898] Include templates --- MANIFEST.in | 1 + setup.py | 1 + .../charm_tests/tempest/templates/__init__.py | 15 --------------- 3 files changed, 2 insertions(+), 15 deletions(-) create mode 100644 MANIFEST.in delete mode 100644 zaza/openstack/charm_tests/tempest/templates/__init__.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..96aeb92 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include zaza/openstack *.j2 diff --git a/setup.py b/setup.py index a77d425..f4e31f8 100644 --- a/setup.py +++ b/setup.py @@ -108,6 +108,7 @@ setup( license='Apache-2.0: http://www.apache.org/licenses/LICENSE-2.0', packages=find_packages(exclude=["unit_tests"]), zip_safe=False, + include_package_data=True, cmdclass={'test': Tox}, install_requires=install_require, extras_require={ diff --git a/zaza/openstack/charm_tests/tempest/templates/__init__.py b/zaza/openstack/charm_tests/tempest/templates/__init__.py deleted file mode 100644 index 9269e37..0000000 --- a/zaza/openstack/charm_tests/tempest/templates/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Collection of templates for tempest.""" From e8898979b5cd860edb85f3b10f88fbf647bcdb9c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 12 Jun 2020 01:56:17 +0000 Subject: [PATCH 547/898] Drop check for tempest workspace directory This check is not necessary. The workspace is recreated on each run, so just attempt to remove the workspace each time. --- zaza/openstack/charm_tests/tempest/setup.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 4266ddc..8f37f02 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -15,7 +15,6 @@ """Code for configuring and initializing tempest.""" import urllib.parse -import os import subprocess import zaza.utilities.deployment_env as deployment_env @@ -253,12 +252,11 @@ def setup_tempest(tempest_template, accounts_template): :returns: None :rtype: None """ - if os.path.isdir('tempest-workspace'): - try: - subprocess.check_call(['tempest', 'workspace', 'remove', '--rmdir', - '--name', 'tempest-workspace']) - except subprocess.CalledProcessError: - pass + try: + subprocess.check_call(['tempest', 'workspace', 'remove', '--rmdir', + '--name', 'tempest-workspace']) + except subprocess.CalledProcessError: + pass try: subprocess.check_call(['tempest', 'init', 'tempest-workspace']) except subprocess.CalledProcessError: From de893b66b838f369eb7638e51fab2d10a66eec3c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 12 Jun 2020 01:58:10 +0000 Subject: [PATCH 548/898] Drop OS_ prefix from OS_TEST_HTTP_PROXY As part of this change also switch to using deployment_env. --- unit_tests/utilities/test_zaza_utilities_openstack.py | 10 ++++++---- zaza/openstack/utilities/openstack.py | 8 +++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 3083204..a6aa190 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -293,8 +293,9 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.patch_object(openstack_utils.urllib.request, "ProxyHandler") self.patch_object(openstack_utils.urllib.request, "HTTPHandler") self.patch_object(openstack_utils.urllib.request, "build_opener") - self.patch_object(openstack_utils.os, "getenv") - self.getenv.return_value = None + self.patch_object(openstack_utils.deployment_env, + "get_deployment_context", + return_value=dict(TEST_HTTP_PROXY=None)) HTTPHandler_mock = mock.MagicMock() self.HTTPHandler.return_value = HTTPHandler_mock openstack_utils.get_urllib_opener() @@ -305,8 +306,9 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.patch_object(openstack_utils.urllib.request, "ProxyHandler") self.patch_object(openstack_utils.urllib.request, "HTTPHandler") self.patch_object(openstack_utils.urllib.request, "build_opener") - self.patch_object(openstack_utils.os, "getenv") - self.getenv.return_value = 'http://squidy' + self.patch_object(openstack_utils.deployment_env, + "get_deployment_context", + return_value=dict(TEST_HTTP_PROXY='http://squidy')) ProxyHandler_mock = mock.MagicMock() self.ProxyHandler.return_value = ProxyHandler_mock openstack_utils.get_urllib_opener() diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index ea2a6ba..4963247 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -39,6 +39,7 @@ from keystoneauth1.identity import ( v2, ) import zaza.openstack.utilities.cert as cert +import zaza.utilities.deployment_env as deployment_env from novaclient import client as novaclient_client from neutronclient.v2_0 import client as neutronclient from neutronclient.common import exceptions as neutronexceptions @@ -1749,14 +1750,15 @@ def get_urllib_opener(): Using urllib.request.urlopen will automatically handle proxies so none of this function is needed except we are currently specifying proxies - via OS_TEST_HTTP_PROXY rather than http_proxy so a ProxyHandler is needed + via TEST_HTTP_PROXY rather than http_proxy so a ProxyHandler is needed explicitly stating the proxies. :returns: An opener which opens URLs via BaseHandlers chained together :rtype: urllib.request.OpenerDirector """ - http_proxy = os.getenv('OS_TEST_HTTP_PROXY') - logging.debug('OS_TEST_HTTP_PROXY: {}'.format(http_proxy)) + deploy_env = deployment_env.get_deployment_context() + http_proxy = deploy_env.get('TEST_HTTP_PROXY') + logging.debug('TEST_HTTP_PROXY: {}'.format(http_proxy)) if http_proxy: handler = urllib.request.ProxyHandler({'http': http_proxy}) From 931fcb4aa7b72bcc04010e7375af59514a76aabc Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 12 Jun 2020 10:40:11 +0100 Subject: [PATCH 549/898] Add setup for glance-simplestreams-sync Use action to complete initial image sync for the gss charm. This avoids races where the images end up in the wrong locations and allows the tests to actually know when images should be discoverable. Update tests to wait for at least four images (20.04 is synced by default). --- .../glance_simplestreams_sync/setup.py | 40 +++++++++++++++++++ .../glance_simplestreams_sync/tests.py | 5 ++- 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py new file mode 100644 index 0000000..06e172d --- /dev/null +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for configuring glance-simplestreams-sync.""" + +import logging + +import zaza.model as zaza_model +import zaza.openstack.utilities.generic as generic_utils + + +def sync_images(): + """Run image sync using an action. + + Execute an initial image sync using an action to ensure that the + cloud is populated with images at the right point in time during + deployment. + """ + logging.info("Synchronising images using glance-simplestreams-sync") + generic_utils.assertActionRanOK( + zaza_model.run_action_on_leader( + "glance-simplestreams-sync", + "sync-images", + raise_on_failure=True, + action_params={}, + ) + ) diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py index 648172a..adbc87b 100644 --- a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py @@ -24,7 +24,7 @@ import zaza.openstack.utilities.openstack as openstack_utils @tenacity.retry( - retry=tenacity.retry_if_result(lambda images: len(images) < 3), + retry=tenacity.retry_if_result(lambda images: len(images) < 4), wait=tenacity.wait_fixed(6), # interval between retries stop=tenacity.stop_after_attempt(100)) # retry times def retry_image_sync(glance_client): @@ -61,7 +61,7 @@ class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): cls.keystone_session) def test_010_wait_for_image_sync(self): - """Wait for images to be synced. Expect at least three.""" + """Wait for images to be synced. Expect at least four.""" self.assertTrue(retry_image_sync(self.glance_client)) def test_050_gss_permissions_regression_check_lp1611987(self): @@ -94,6 +94,7 @@ class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): 'com.ubuntu.cloud:server:14.04:amd64', 'com.ubuntu.cloud:server:16.04:amd64', 'com.ubuntu.cloud:server:18.04:amd64', + 'com.ubuntu.cloud:server:20.04:amd64', ] uri = "streams/v1/auto.sync.json" key = "url" From 12cf1725cb56d9ba3a1739c7095be795bc9bfa46 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 12 Jun 2020 12:54:57 +0000 Subject: [PATCH 550/898] Fix service -> setup f map --- zaza/openstack/charm_tests/tempest/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 8e338f6..eaa855a 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -231,8 +231,8 @@ def get_tempest_context(): ctxt = {} ctxt_funcs = { 'nova': add_nova_config, - 'neutron': add_nova_config, - 'glance': add_keystone_config, + 'neutron': add_neutron_config, + 'glance': add_glance_config, 'cinder': add_cinder_config, 'keystone': add_keystone_config} ctxt['enabled_services'] = get_service_list(keystone_session) From 68648aede5121dee0f7ed8d2726392976aaeea5e Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 12 Jun 2020 13:14:28 +0000 Subject: [PATCH 551/898] Tidyup following review feedback --- zaza/openstack/charm_tests/tempest/setup.py | 2 +- zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 | 2 ++ zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index eaa855a..23308af 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -169,7 +169,7 @@ def add_keystone_config(ctxt, keystone_session): def get_service_list(keystone_session): - """Add keystone config to context. + """Retrieve list of services from keystone. :param keystone_session: keystoneauth1.session.Session object :type: keystoneauth1.session.Session diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 index b4f2dfc..f5505ad 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 @@ -45,8 +45,10 @@ api_v2 = true api_v3 = false {% endif %} +{% if 'glance' in enabled_services %} [image] http_image = http://{{ test_swift_ip }}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz +{% endif %} {% if 'neutron' in enabled_services %} [network] diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 index dc7d8db..0ed401a 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 @@ -47,8 +47,10 @@ api_v2 = false api_v3 = true {% endif %} +{% if 'glance' in enabled_services %} [image] http_image = http://{{ test_swift_ip }}:80/swift/v1/images/cirros-0.3.4-x86_64-uec.tar.gz +{% endif %} {% if 'neutron' in enabled_services %} [network] @@ -98,4 +100,5 @@ storage_protocol = ceph catalog_type = {{ catalog_type }} [volume-feature-enabled] +backup = false {% endif %} From 06ed9eecf02aaeeb4166d3b7b042e030beaa2f0b Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Tue, 16 Jun 2020 17:01:05 -0500 Subject: [PATCH 552/898] Add gnocchi s3 test --- zaza/openstack/charm_tests/gnocchi/tests.py | 42 ++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index f789cff..269851f 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -16,11 +16,14 @@ """Encapsulate Gnocchi testing.""" +import boto3 import logging - from gnocchiclient.v1 import client as gnocchi_client + +import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils +from zaza.openstack.charm_tests.swift.tests import S3APITest class GnocchiTest(test_utils.OpenStackBaseTest): @@ -58,3 +61,40 @@ class GnocchiTest(test_utils.OpenStackBaseTest): """ with self.pause_resume(self.services): logging.info("Testing pause and resume") + + +class GnocchiS3Test(test_utils.OpenStackBaseTest): + """Tests for S3 storage backend""" + + swift = S3APITest + kwargs = { + 'region_name': swift.s3_region, + 'aws_access_key_id': swift.ec2_creds.access, + 'aws_secret_access_key': swift.ec2_creds.secret, + 'endpoint_url': swift.s3_endpoint, + 'verify': swift.cacert, + } + + def update_gnocchi_config_for_s3(self): + """Update Gnocchi with the correct values for the S3 backend""" + logging.debug('Changing charm setting to connect to S3') + model.set_application_config( + 'gnocchi', + {'s3-endpoint-url': self.swift.s3_endpoint, + 's3-region-name': self.swift.s3_region, + 's3-access-key-id': self.swift.ec2_creds.access, + 's3-secret-access-key': self.swift.ec2_creds.secret}, + model_name=self.model_name + ) + logging.debug( + 'Waiting for units to execute config-changed hook') + model.wait_for_agent_status(model_name=self.model_name) + logging.debug( + 'Waiting for units to reach target states') + model.wait_for_application_states( + model_name=self.model_name, + states={'gnocchi': { + 'workload-status-': 'active', + 'workload-status-message': 'Unit is ready'}} + ) + model.block_until_all_units_idle() \ No newline at end of file From e66ef5ac6c039d2687326d669726c001766da65b Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Tue, 16 Jun 2020 17:38:59 -0500 Subject: [PATCH 553/898] Get S3 info --- zaza/openstack/charm_tests/gnocchi/tests.py | 50 +++++++++++++++------ 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 269851f..6148675 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -66,24 +66,48 @@ class GnocchiTest(test_utils.OpenStackBaseTest): class GnocchiS3Test(test_utils.OpenStackBaseTest): """Tests for S3 storage backend""" - swift = S3APITest - kwargs = { - 'region_name': swift.s3_region, - 'aws_access_key_id': swift.ec2_creds.access, - 'aws_secret_access_key': swift.ec2_creds.secret, - 'endpoint_url': swift.s3_endpoint, - 'verify': swift.cacert, - } + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(GnocchiS3Test, cls).setUpClass() + + session = openstack_utils.get_overcloud_keystone_session() + ks_client = openstack_utils.get_keystone_session_client(session) + + # Get token data so we can clean our user_id and project_id + token_data = ks_client.tokens.get_token_data(session.get_token()) + project_id = token_data['token']['project']['id'] + user_id = token_data['token']['user']['id'] + + # Store URL to service providing S3 compatible API + for entry in token_data['token']['catalog']: + if entry['type'] == 's3': + for endpoint in entry['endpoints']: + if endpoint['interface'] == 'public': + cls.s3_region = endpoint['region'] + cls.s3_endpoint = endpoint['url'] + + # Create AWS compatible application credentials in Keystone + cls.ec2_creds = ks_client.ec2.create(user_id, project_id) + + # kwargs = { + # 'region_name': swift.s3_region, + # 'aws_access_key_id': swift.ec2_creds.access, + # 'aws_secret_access_key': swift.ec2_creds.secret, + # 'endpoint_url': swift.s3_endpoint, + # 'verify': swift.cacert, + # } def update_gnocchi_config_for_s3(self): """Update Gnocchi with the correct values for the S3 backend""" + logging.debug('Changing charm setting to connect to S3') model.set_application_config( 'gnocchi', - {'s3-endpoint-url': self.swift.s3_endpoint, - 's3-region-name': self.swift.s3_region, - 's3-access-key-id': self.swift.ec2_creds.access, - 's3-secret-access-key': self.swift.ec2_creds.secret}, + {'s3-endpoint-url': self.s3_endpoint, + 's3-region-name': self.s3_region, + 's3-access-key-id': self.ec2_creds.access, + 's3-secret-access-key': self.ec2_creds.secret}, model_name=self.model_name ) logging.debug( @@ -97,4 +121,4 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): 'workload-status-': 'active', 'workload-status-message': 'Unit is ready'}} ) - model.block_until_all_units_idle() \ No newline at end of file + model.block_until_all_units_idle() From 0c33bcc4a1705cbf36a9f3d369749c38f1436ba0 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Wed, 17 Jun 2020 15:17:02 -0500 Subject: [PATCH 554/898] Gnocchi S3 Tests --- zaza/openstack/charm_tests/gnocchi/tests.py | 58 +++++++++++++++------ 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 6148675..2f1cb83 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -64,7 +64,7 @@ class GnocchiTest(test_utils.OpenStackBaseTest): class GnocchiS3Test(test_utils.OpenStackBaseTest): - """Tests for S3 storage backend""" + """Test object storage S3 API.""" @classmethod def setUpClass(cls): @@ -74,7 +74,7 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): session = openstack_utils.get_overcloud_keystone_session() ks_client = openstack_utils.get_keystone_session_client(session) - # Get token data so we can clean our user_id and project_id + # Get token data so we can glean our user_id and project_id token_data = ks_client.tokens.get_token_data(session.get_token()) project_id = token_data['token']['project']['id'] user_id = token_data['token']['user']['id'] @@ -90,18 +90,10 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): # Create AWS compatible application credentials in Keystone cls.ec2_creds = ks_client.ec2.create(user_id, project_id) - # kwargs = { - # 'region_name': swift.s3_region, - # 'aws_access_key_id': swift.ec2_creds.access, - # 'aws_secret_access_key': swift.ec2_creds.secret, - # 'endpoint_url': swift.s3_endpoint, - # 'verify': swift.cacert, - # } - - def update_gnocchi_config_for_s3(self): - """Update Gnocchi with the correct values for the S3 backend""" + def test_s3_connection_for_gnocchi(self): + """Use S3 API to list buckets.""" - logging.debug('Changing charm setting to connect to S3') + logging.info('Changing charm config to connect to swift S3 backend') model.set_application_config( 'gnocchi', {'s3-endpoint-url': self.s3_endpoint, @@ -110,15 +102,47 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): 's3-secret-access-key': self.ec2_creds.secret}, model_name=self.model_name ) - logging.debug( + logging.info( 'Waiting for units to execute config-changed hook') model.wait_for_agent_status(model_name=self.model_name) - logging.debug( + logging.info( 'Waiting for units to reach target states') model.wait_for_application_states( model_name=self.model_name, states={'gnocchi': { - 'workload-status-': 'active', - 'workload-status-message': 'Unit is ready'}} + 'workload-status-': 'active', + 'workload-status-message': 'Unit is ready' + }, + 'ceilometer': { + 'workload-status' : 'blocked', + 'workload-status-message': 'Run the ceilometer-upgrade action on the leader to initialize ceilometer and gnocchi' + } + } ) model.block_until_all_units_idle() + + def test_s3_list_gnocchi_buckets(self): + """Verify that the gnocchi buckets were created in the S3 backend """ + + kwargs = { + 'region_name': self.s3_region, + 'aws_access_key_id': self.ec2_creds.access, + 'aws_secret_access_key': self.ec2_creds.secret, + 'endpoint_url': self.s3_endpoint, + 'verify': self.cacert, + } + s3_client = boto3.client('s3', **kwargs) + s3 = boto3.resource('s3', **kwargs) + + bucket_names = ['gnocchi-measure', 'gnocchi-aggregates'] + # Validate their presence + bucket_list = s3_client.list_buckets() + logging.info(pprint.pformat(bucket_list)) + for bkt in bucket_list['Buckets']: + for gnocchi_bkt in bucket_names: + print(gnocchi_bkt) + if bkt['Name'] == gnocchi_bkt: + print('break out of 1st loop') + break + else: + AssertionError('Bucket "{}" not found'.format(gnocchi_bkt)) From a1d4a3c4ae17d7929352406c002beccd5f1753bb Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Wed, 17 Jun 2020 15:30:04 -0500 Subject: [PATCH 555/898] pep8 --- zaza/openstack/charm_tests/gnocchi/tests.py | 40 ++++++++++----------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 2f1cb83..6f7b986 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -18,12 +18,12 @@ import boto3 import logging +import pprint from gnocchiclient.v1 import client as gnocchi_client import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils -from zaza.openstack.charm_tests.swift.tests import S3APITest class GnocchiTest(test_utils.OpenStackBaseTest): @@ -92,38 +92,39 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): def test_s3_connection_for_gnocchi(self): """Use S3 API to list buckets.""" - logging.info('Changing charm config to connect to swift S3 backend') model.set_application_config( 'gnocchi', {'s3-endpoint-url': self.s3_endpoint, - 's3-region-name': self.s3_region, - 's3-access-key-id': self.ec2_creds.access, - 's3-secret-access-key': self.ec2_creds.secret}, + 's3-region-name': self.s3_region, + 's3-access-key-id': self.ec2_creds.access, + 's3-secret-access-key': self.ec2_creds.secret}, model_name=self.model_name ) logging.info( - 'Waiting for units to execute config-changed hook') + 'Waiting for units to execute config-changed hook') model.wait_for_agent_status(model_name=self.model_name) logging.info( - 'Waiting for units to reach target states') + 'Waiting for units to reach target states') model.wait_for_application_states( model_name=self.model_name, - states={'gnocchi': { - 'workload-status-': 'active', - 'workload-status-message': 'Unit is ready' - }, - 'ceilometer': { - 'workload-status' : 'blocked', - 'workload-status-message': 'Run the ceilometer-upgrade action on the leader to initialize ceilometer and gnocchi' - } - } + states={ + 'gnocchi': { + 'workload-status-': 'active', + 'workload-status-message': 'Unit is ready' + }, + 'ceilometer': { + 'workload-status': 'blocked', + 'workload-status-message': 'Run the ' + + 'ceilometer-upgrade action on the leader ' + + 'to initialize ceilometer and gnocchi' + } + } ) model.block_until_all_units_idle() def test_s3_list_gnocchi_buckets(self): - """Verify that the gnocchi buckets were created in the S3 backend """ - + """Verify that the gnocchi buckets were created in the S3 backend.""" kwargs = { 'region_name': self.s3_region, 'aws_access_key_id': self.ec2_creds.access, @@ -132,7 +133,6 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): 'verify': self.cacert, } s3_client = boto3.client('s3', **kwargs) - s3 = boto3.resource('s3', **kwargs) bucket_names = ['gnocchi-measure', 'gnocchi-aggregates'] # Validate their presence @@ -140,9 +140,7 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): logging.info(pprint.pformat(bucket_list)) for bkt in bucket_list['Buckets']: for gnocchi_bkt in bucket_names: - print(gnocchi_bkt) if bkt['Name'] == gnocchi_bkt: - print('break out of 1st loop') break else: AssertionError('Bucket "{}" not found'.format(gnocchi_bkt)) From a3836e06e2f25cacd10fb2b1f788e3f48b06e635 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Thu, 18 Jun 2020 16:13:27 -0500 Subject: [PATCH 556/898] pep8 --- zaza/openstack/charm_tests/gnocchi/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 6f7b986..0a0a0cc 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -64,7 +64,7 @@ class GnocchiTest(test_utils.OpenStackBaseTest): class GnocchiS3Test(test_utils.OpenStackBaseTest): - """Test object storage S3 API.""" + """Test Gnocchi with S3 storage backend.""" @classmethod def setUpClass(cls): @@ -91,7 +91,7 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): cls.ec2_creds = ks_client.ec2.create(user_id, project_id) def test_s3_connection_for_gnocchi(self): - """Use S3 API to list buckets.""" + """Set S3 config for gnocchi-upgrade.""" logging.info('Changing charm config to connect to swift S3 backend') model.set_application_config( 'gnocchi', From 1f2b4890ef0a44b089b71b96dd8c582e774ba77c Mon Sep 17 00:00:00 2001 From: Drew Freiberger Date: Thu, 18 Jun 2020 08:36:41 -0500 Subject: [PATCH 557/898] Fix racy swift global replication testing During some runs of functional testing, swift global replication would only have 2 copies rather than 3 of the object that was just written. This patch attempts to resolve the race condition and also adds ordering to more recently added tests. Closes-Bug: #1882247 --- zaza/openstack/charm_tests/swift/tests.py | 41 +++++++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index 32441df..75f82b7 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -178,15 +178,28 @@ class SwiftGlobalReplicationTests(test_utils.OpenStackBaseTest): logging.info('Deleting container {}'.format(container['name'])) cls.swift_region1.delete_container(container['name']) - def test_two_regions_any_zones_two_replicas(self): - """Create an object with two replicas across two regions.""" + def test_901_two_regions_any_zones_two_replicas(self): + """Create an object with two replicas across two regions. + + We set write affinity to write the first copy in the local + region of the proxy used to perform the write, the other + replica will land in the remote region. + """ swift_utils.apply_proxy_config( self.region1_proxy_app, { - 'write-affinity': 'r1, r2', + 'write-affinity': 'r1', 'write-affinity-node-count': '1', 'replicas': '2'}, self.region1_model_name) + swift_utils.apply_proxy_config( + self.region2_proxy_app, + { + 'write-affinity': 'r2', + 'write-affinity-node-count': '1', + 'replicas': '2'}, + self.region2_model_name) + logging.info('Proxy configs updated in both regions') container_name, obj_name, obj_replicas = swift_utils.create_object( self.swift_region1, self.region1_proxy_app, @@ -204,15 +217,29 @@ class SwiftGlobalReplicationTests(test_utils.OpenStackBaseTest): len(obj_replicas.all_zones), 2) - def test_two_regions_any_zones_three_replicas(self): - """Create an object with three replicas across two regions.""" + def test_902_two_regions_any_zones_three_replicas(self): + """Create an object with three replicas across two regions. + + We set write affinity to write the first copy in the local + region of the proxy used to perform the write, at least one + of the other two replicas will end up in the opposite region + based on primary partitions in the ring. + """ swift_utils.apply_proxy_config( self.region1_proxy_app, { - 'write-affinity': 'r1, r2', + 'write-affinity': 'r1', 'write-affinity-node-count': '1', 'replicas': '3'}, self.region1_model_name) + swift_utils.apply_proxy_config( + self.region2_proxy_app, + { + 'write-affinity': 'r2', + 'write-affinity-node-count': '1', + 'replicas': '3'}, + self.region2_model_name) + logging.info('Proxy configs updated in both regions') container_name, obj_name, obj_replicas = swift_utils.create_object( self.swift_region1, self.region1_proxy_app, @@ -258,7 +285,7 @@ class S3APITest(test_utils.OpenStackBaseTest): # Create AWS compatible application credentials in Keystone cls.ec2_creds = ks_client.ec2.create(user_id, project_id) - def test_s3_list_buckets(self): + def test_901_s3_list_buckets(self): """Use S3 API to list buckets.""" # We use a mix of the high- and low-level API with common arguments kwargs = { From a3da91e643df477cffb17156486bfa6e0fa881e8 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Thu, 18 Jun 2020 16:32:57 -0500 Subject: [PATCH 558/898] Move configure step to setup.py --- zaza/openstack/charm_tests/gnocchi/setup.py | 69 +++++++++++++++++++++ zaza/openstack/charm_tests/gnocchi/tests.py | 34 +--------- 2 files changed, 70 insertions(+), 33 deletions(-) create mode 100644 zaza/openstack/charm_tests/gnocchi/setup.py diff --git a/zaza/openstack/charm_tests/gnocchi/setup.py b/zaza/openstack/charm_tests/gnocchi/setup.py new file mode 100644 index 0000000..086c263 --- /dev/null +++ b/zaza/openstack/charm_tests/gnocchi/setup.py @@ -0,0 +1,69 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup for Gnocchi tests.""" + +import logging + +import zaza.model as model +import zaza.openstack.utilities.openstack as openstack_utils + + +def configure_s3_backend(): + """Inject S3 parameters from Swift for Gnocchi config.""" + session = openstack_utils.get_overcloud_keystone_session() + ks_client = openstack_utils.get_keystone_session_client(session) + + logging.info('Retrieving S3 connection data from Swift') + token_data = ks_client.tokens.get_token_data(session.get_token()) + project_id = token_data['token']['project']['id'] + user_id = token_data['token']['user']['id'] + + # Store URL to service providing S3 compatible API + for entry in token_data['token']['catalog']: + if entry['type'] == 's3': + for endpoint in entry['endpoints']: + if endpoint['interface'] == 'public': + s3_region = endpoint['region'] + s3_endpoint = endpoint['url'] + + # Create AWS compatible application credentials in Keystone + ec2_creds = ks_client.ec2.create(user_id, project_id) + + logging.info('Changing Gnocchi charm config to connect to S3') + model.set_application_config( + 'gnocchi', + {'s3-endpoint-url': s3_endpoint, + 's3-region-name': s3_region, + 's3-access-key-id': ec2_creds.access, + 's3-secret-access-key': ec2_creds.secret} + ) + logging.info('Waiting for units to execute config-changed hook') + model.wait_for_agent_status() + logging.info('Waiting for units to reach target states') + model.wait_for_application_states( + states={ + 'gnocchi': { + 'workload-status-': 'active', + 'workload-status-message': 'Unit is ready' + }, + 'ceilometer': { + 'workload-status': 'blocked', + 'workload-status-message': 'Run the ' + + 'ceilometer-upgrade action on the leader ' + + 'to initialize ceilometer and gnocchi' + } + } + ) + model.block_until_all_units_idle() \ No newline at end of file diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 0a0a0cc..d7d5736 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -64,7 +64,7 @@ class GnocchiTest(test_utils.OpenStackBaseTest): class GnocchiS3Test(test_utils.OpenStackBaseTest): - """Test Gnocchi with S3 storage backend.""" + """Test Gnocchi for S3 storage backend.""" @classmethod def setUpClass(cls): @@ -90,38 +90,6 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): # Create AWS compatible application credentials in Keystone cls.ec2_creds = ks_client.ec2.create(user_id, project_id) - def test_s3_connection_for_gnocchi(self): - """Set S3 config for gnocchi-upgrade.""" - logging.info('Changing charm config to connect to swift S3 backend') - model.set_application_config( - 'gnocchi', - {'s3-endpoint-url': self.s3_endpoint, - 's3-region-name': self.s3_region, - 's3-access-key-id': self.ec2_creds.access, - 's3-secret-access-key': self.ec2_creds.secret}, - model_name=self.model_name - ) - logging.info( - 'Waiting for units to execute config-changed hook') - model.wait_for_agent_status(model_name=self.model_name) - logging.info( - 'Waiting for units to reach target states') - model.wait_for_application_states( - model_name=self.model_name, - states={ - 'gnocchi': { - 'workload-status-': 'active', - 'workload-status-message': 'Unit is ready' - }, - 'ceilometer': { - 'workload-status': 'blocked', - 'workload-status-message': 'Run the ' + - 'ceilometer-upgrade action on the leader ' + - 'to initialize ceilometer and gnocchi' - } - } - ) - model.block_until_all_units_idle() def test_s3_list_gnocchi_buckets(self): """Verify that the gnocchi buckets were created in the S3 backend.""" From 95eb158e7e00f3958f6f3c698fa2900ec5648abd Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 19 Jun 2020 08:28:18 +0200 Subject: [PATCH 559/898] Support undercloud running TLS (#330) Consume the `OS_CACERT` environment variable when setting up undercloud auth. Fixes #329 --- zaza/openstack/utilities/openstack.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 4963247..780d2df 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1621,6 +1621,10 @@ def get_undercloud_auth(): if os_project_id is not None: auth_settings['OS_PROJECT_ID'] = os_project_id + _os_cacert = os.environ.get('OS_CACERT') + if _os_cacert: + auth_settings.update({'OS_CACERT': _os_cacert}) + # Validate settings for key, settings in list(auth_settings.items()): if settings is None: From 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 19 Jun 2020 09:50:28 +0200 Subject: [PATCH 560/898] Store local overcloud CACERT copy in relative path At present the overcloud CACERT is copied to /tmp and as such it is not possbile to run multiple tests at once without them stepping on each other. Store the copy in a path relative to where the test is executed, in line with how the SSH keys are stored etc. Fixes #331 --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 780d2df..64d42a8 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -163,7 +163,7 @@ WORKLOAD_STATUS_EXCEPTIONS = { KEYSTONE_CACERT = "keystone_juju_ca_cert.crt" KEYSTONE_REMOTE_CACERT = ( "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT)) -KEYSTONE_LOCAL_CACERT = ("/tmp/{}".format(KEYSTONE_CACERT)) +KEYSTONE_LOCAL_CACERT = ("tests/{}".format(KEYSTONE_CACERT)) def get_cacert(): From 692853e242bb66ba426032ecec162a840112335a Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Fri, 19 Jun 2020 14:19:25 -0500 Subject: [PATCH 561/898] pep8 --- zaza/openstack/charm_tests/gnocchi/setup.py | 2 +- zaza/openstack/charm_tests/gnocchi/tests.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/setup.py b/zaza/openstack/charm_tests/gnocchi/setup.py index 086c263..4995f22 100644 --- a/zaza/openstack/charm_tests/gnocchi/setup.py +++ b/zaza/openstack/charm_tests/gnocchi/setup.py @@ -66,4 +66,4 @@ def configure_s3_backend(): } } ) - model.block_until_all_units_idle() \ No newline at end of file + model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index d7d5736..680bf3c 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -21,7 +21,6 @@ import logging import pprint from gnocchiclient.v1 import client as gnocchi_client -import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -90,7 +89,6 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): # Create AWS compatible application credentials in Keystone cls.ec2_creds = ks_client.ec2.create(user_id, project_id) - def test_s3_list_gnocchi_buckets(self): """Verify that the gnocchi buckets were created in the S3 backend.""" kwargs = { From f0ceb33f3a31fc5a3c8ed18971cc19009a10259f Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 18 Jun 2020 15:03:17 +0200 Subject: [PATCH 562/898] Add functional tests for neutron-arista --- zaza/openstack/charm_tests/neutron/tests.py | 21 +++++- .../charm_tests/neutron_arista/__init__.py | 15 +++++ .../charm_tests/neutron_arista/setup.py | 39 +++++++++++ .../charm_tests/neutron_arista/tests.py | 53 +++++++++++++++ .../charm_tests/neutron_arista/utils.py | 67 +++++++++++++++++++ 5 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 zaza/openstack/charm_tests/neutron_arista/__init__.py create mode 100644 zaza/openstack/charm_tests/neutron_arista/setup.py create mode 100644 zaza/openstack/charm_tests/neutron_arista/tests.py create mode 100644 zaza/openstack/charm_tests/neutron_arista/utils.py diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 7e28112..ba014e2 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -293,8 +293,23 @@ class NeutronCreateNetworkTest(test_utils.OpenStackBaseTest): cls.neutron_client = ( openstack_utils.get_neutron_session_client(cls.keystone_session)) + def _test_400_additional_validation(self, expected_network_names): + """Additional assertions for test_400_create_network. + + Can be overridden in derived classes. + + :type expected_network_names: List[str] + """ + pass + def test_400_create_network(self): - """Create a network, verify that it exists, and then delete it.""" + """Create a network, verify that it exists, and then delete it. + + Additional verifications on the created network can be performed by + deriving this class and overriding _test_400_additional_validation(). + """ + self._test_400_additional_validation([]) + logging.debug('Creating neutron network...') self.neutron_client.format = 'json' net_name = 'test_net' @@ -319,10 +334,14 @@ class NeutronCreateNetworkTest(test_utils.OpenStackBaseTest): network = networks['networks'][0] assert network['name'] == net_name, "network ext_net not found" + self._test_400_additional_validation([net_name]) + # Cleanup logging.debug('Deleting neutron network...') self.neutron_client.delete_network(network['id']) + self._test_400_additional_validation([]) + class NeutronApiTest(NeutronCreateNetworkTest): """Test basic Neutron API Charm functionality.""" diff --git a/zaza/openstack/charm_tests/neutron_arista/__init__.py b/zaza/openstack/charm_tests/neutron_arista/__init__.py new file mode 100644 index 0000000..8d3f5c9 --- /dev/null +++ b/zaza/openstack/charm_tests/neutron_arista/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing neutron-arista.""" diff --git a/zaza/openstack/charm_tests/neutron_arista/setup.py b/zaza/openstack/charm_tests/neutron_arista/setup.py new file mode 100644 index 0000000..50878e6 --- /dev/null +++ b/zaza/openstack/charm_tests/neutron_arista/setup.py @@ -0,0 +1,39 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for setting up neutron-arista.""" + +import logging +import tenacity +import zaza +import zaza.openstack.charm_tests.neutron_arista.utils as arista_utils + + +def test_fixture(): + """Pass arista-virt-test-fixture's IP address to neutron-arista.""" + fixture_ip_addr = arista_utils.fixture_ip_addr() + logging.info( + "{}'s IP address is '{}'. Passing it to neutron-arista...".format( + arista_utils.FIXTURE_APP_NAME, fixture_ip_addr)) + zaza.model.set_application_config('neutron-arista', + {'eapi-host': fixture_ip_addr}) + + logging.info('Waiting for {} to become ready...'.format( + arista_utils.FIXTURE_APP_NAME)) + for attempt in tenacity.Retrying( + wait=tenacity.wait_fixed(10), # seconds + stop=tenacity.stop_after_attempt(30), + reraise=True): + with attempt: + arista_utils.query_fixture_networks(fixture_ip_addr) diff --git a/zaza/openstack/charm_tests/neutron_arista/tests.py b/zaza/openstack/charm_tests/neutron_arista/tests.py new file mode 100644 index 0000000..c47c9c8 --- /dev/null +++ b/zaza/openstack/charm_tests/neutron_arista/tests.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulating `neutron-arista` testing.""" + +import logging +import tenacity +import zaza.openstack.charm_tests.neutron.tests as neutron_tests +import zaza.openstack.charm_tests.neutron_arista.utils as arista_utils + + +class NeutronCreateAristaNetworkTest(neutron_tests.NeutronCreateNetworkTest): + """Test creating an Arista Neutron network through the API.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Neutron Arista tests.""" + super(NeutronCreateAristaNetworkTest, cls).setUpClass() + + logging.info('Waiting for Neutron to become ready...') + for attempt in tenacity.Retrying( + wait=tenacity.wait_fixed(5), # seconds + stop=tenacity.stop_after_attempt(12), + reraise=True): + with attempt: + cls.neutron_client.list_networks() + + def _test_400_additional_validation(self, expected_network_names): + """Arista-specific assertions for test_400_create_network. + + :type expected_network_names: List[str] + """ + logging.info("Querying Arista CVX's networks...") + actual_network_names = arista_utils.query_fixture_networks( + arista_utils.fixture_ip_addr()) + + # NOTE(lourot): the assertion name is misleading as it's not only + # checking the item count but also that all items are present in + # both lists, without checking the order. + self.assertCountEqual(actual_network_names, expected_network_names) diff --git a/zaza/openstack/charm_tests/neutron_arista/utils.py b/zaza/openstack/charm_tests/neutron_arista/utils.py new file mode 100644 index 0000000..3c68f33 --- /dev/null +++ b/zaza/openstack/charm_tests/neutron_arista/utils.py @@ -0,0 +1,67 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common Arista-related utils.""" + +import json +import requests +import urllib3 +import zaza + +FIXTURE_APP_NAME = 'arista-virt-test-fixture' + + +def fixture_ip_addr(): + """Return the public IP address of the Arista test fixture.""" + return zaza.model.get_units(FIXTURE_APP_NAME)[0].public_address + + +_FIXTURE_LOGIN = 'admin' +_FIXTURE_PASSWORD = 'password123' + + +def query_fixture_networks(ip_addr): + """Query the Arista test fixture's list of networks.""" + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + session = requests.Session() + session.headers['Content-Type'] = 'application/json' + session.headers['Accept'] = 'application/json' + session.verify = False + session.auth = (_FIXTURE_LOGIN, _FIXTURE_PASSWORD) + + data = { + 'id': 'Zaza neutron-arista tests', + 'method': 'runCmds', + 'jsonrpc': '2.0', + 'params': { + 'timestamps': False, + 'format': 'json', + 'version': 1, + 'cmds': ['show openstack networks'] + } + } + + response = session.post( + 'https://{}/command-api/'.format(ip_addr), + data=json.dumps(data), + timeout=10 # seconds + ) + + result = [] + for _, region in response.json()['result'][0]['regions'].items(): + for _, tenant in region['tenants'].items(): + for _, network in tenant['tenantNetworks'].items(): + result.append(network['networkName']) + return result From 13b590ea12376ad5d2fc3568db2ada0d490beb3c Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 23 Jun 2020 13:03:04 +0200 Subject: [PATCH 563/898] Refactor overriding mechanism --- zaza/openstack/charm_tests/neutron/tests.py | 61 ++++++++----------- .../charm_tests/neutron_arista/tests.py | 20 +++--- 2 files changed, 35 insertions(+), 46 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index ba014e2..f270604 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -292,55 +292,44 @@ class NeutronCreateNetworkTest(test_utils.OpenStackBaseTest): # set up clients cls.neutron_client = ( openstack_utils.get_neutron_session_client(cls.keystone_session)) + cls.neutron_client.format = 'json' - def _test_400_additional_validation(self, expected_network_names): - """Additional assertions for test_400_create_network. - - Can be overridden in derived classes. - - :type expected_network_names: List[str] - """ - pass + _TEST_NET_NAME = 'test_net' def test_400_create_network(self): - """Create a network, verify that it exists, and then delete it. - - Additional verifications on the created network can be performed by - deriving this class and overriding _test_400_additional_validation(). - """ - self._test_400_additional_validation([]) + """Create a network, verify that it exists, and then delete it.""" + self._assert_test_network_doesnt_exist() + self._create_test_network() + net_id = self._assert_test_network_exists_and_return_id() + self._delete_test_network(net_id) + self._assert_test_network_doesnt_exist() + def _create_test_network(self): logging.debug('Creating neutron network...') - self.neutron_client.format = 'json' - net_name = 'test_net' - - # Verify that the network doesn't exist - networks = self.neutron_client.list_networks(name=net_name) - net_count = len(networks['networks']) - assert net_count == 0, ( - "Expected zero networks, found {}".format(net_count)) - - # Create a network and verify that it exists - network = {'name': net_name} + network = {'name': self._TEST_NET_NAME} self.neutron_client.create_network({'network': network}) - networks = self.neutron_client.list_networks(name=net_name) + def _delete_test_network(self, net_id): + logging.debug('Deleting neutron network...') + self.neutron_client.delete_network(net_id) + + def _assert_test_network_exists_and_return_id(self): + logging.debug('Confirming new neutron network...') + networks = self.neutron_client.list_networks(name=self._TEST_NET_NAME) logging.debug('Networks: {}'.format(networks)) net_len = len(networks['networks']) assert net_len == 1, ( "Expected 1 network, found {}".format(net_len)) - - logging.debug('Confirming new neutron network...') network = networks['networks'][0] - assert network['name'] == net_name, "network ext_net not found" + assert network['name'] == self._TEST_NET_NAME, \ + "network {} not found".format(self._TEST_NET_NAME) + return network['id'] - self._test_400_additional_validation([net_name]) - - # Cleanup - logging.debug('Deleting neutron network...') - self.neutron_client.delete_network(network['id']) - - self._test_400_additional_validation([]) + def _assert_test_network_doesnt_exist(self): + networks = self.neutron_client.list_networks(name=self._TEST_NET_NAME) + net_count = len(networks['networks']) + assert net_count == 0, ( + "Expected zero networks, found {}".format(net_count)) class NeutronApiTest(NeutronCreateNetworkTest): diff --git a/zaza/openstack/charm_tests/neutron_arista/tests.py b/zaza/openstack/charm_tests/neutron_arista/tests.py index c47c9c8..4f6b1de 100644 --- a/zaza/openstack/charm_tests/neutron_arista/tests.py +++ b/zaza/openstack/charm_tests/neutron_arista/tests.py @@ -38,16 +38,16 @@ class NeutronCreateAristaNetworkTest(neutron_tests.NeutronCreateNetworkTest): with attempt: cls.neutron_client.list_networks() - def _test_400_additional_validation(self, expected_network_names): - """Arista-specific assertions for test_400_create_network. - - :type expected_network_names: List[str] - """ - logging.info("Querying Arista CVX's networks...") + def _assert_test_network_exists_and_return_id(self): actual_network_names = arista_utils.query_fixture_networks( arista_utils.fixture_ip_addr()) + self.assertEqual(actual_network_names, [self._TEST_NET_NAME]) + return super(NeutronCreateAristaNetworkTest, + self)._assert_test_network_exists_and_return_id() - # NOTE(lourot): the assertion name is misleading as it's not only - # checking the item count but also that all items are present in - # both lists, without checking the order. - self.assertCountEqual(actual_network_names, expected_network_names) + def _assert_test_network_doesnt_exist(self): + actual_network_names = arista_utils.query_fixture_networks( + arista_utils.fixture_ip_addr()) + self.assertEqual(actual_network_names, []) + super(NeutronCreateAristaNetworkTest, + self)._assert_test_network_doesnt_exist() From a04cc1f077abe1a96859744ca6039960e0356c9a Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 23 Jun 2020 13:11:50 +0200 Subject: [PATCH 564/898] Cosmetic improvements --- zaza/openstack/charm_tests/neutron_arista/setup.py | 4 ++-- zaza/openstack/charm_tests/neutron_arista/utils.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron_arista/setup.py b/zaza/openstack/charm_tests/neutron_arista/setup.py index 50878e6..b990751 100644 --- a/zaza/openstack/charm_tests/neutron_arista/setup.py +++ b/zaza/openstack/charm_tests/neutron_arista/setup.py @@ -24,8 +24,8 @@ def test_fixture(): """Pass arista-virt-test-fixture's IP address to neutron-arista.""" fixture_ip_addr = arista_utils.fixture_ip_addr() logging.info( - "{}'s IP address is '{}'. Passing it to neutron-arista...".format( - arista_utils.FIXTURE_APP_NAME, fixture_ip_addr)) + "{}'s IP address is '{}'. Passing it to neutron-arista..." + .format(arista_utils.FIXTURE_APP_NAME, fixture_ip_addr)) zaza.model.set_application_config('neutron-arista', {'eapi-host': fixture_ip_addr}) diff --git a/zaza/openstack/charm_tests/neutron_arista/utils.py b/zaza/openstack/charm_tests/neutron_arista/utils.py index 3c68f33..78732d6 100644 --- a/zaza/openstack/charm_tests/neutron_arista/utils.py +++ b/zaza/openstack/charm_tests/neutron_arista/utils.py @@ -60,8 +60,8 @@ def query_fixture_networks(ip_addr): ) result = [] - for _, region in response.json()['result'][0]['regions'].items(): - for _, tenant in region['tenants'].items(): - for _, network in tenant['tenantNetworks'].items(): + for region in response.json()['result'][0]['regions'].values(): + for tenant in region['tenants'].values(): + for network in tenant['tenantNetworks'].values(): result.append(network['networkName']) return result From b1ccc4bd24522c11b138c3a713b8ab09d11417bc Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 23 Jun 2020 15:50:34 +0200 Subject: [PATCH 565/898] Rename neutron arista charm --- zaza/openstack/charm_tests/neutron_arista/__init__.py | 2 +- zaza/openstack/charm_tests/neutron_arista/setup.py | 8 ++++---- zaza/openstack/charm_tests/neutron_arista/tests.py | 2 +- zaza/openstack/charm_tests/neutron_arista/utils.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron_arista/__init__.py b/zaza/openstack/charm_tests/neutron_arista/__init__.py index 8d3f5c9..c0eae4e 100644 --- a/zaza/openstack/charm_tests/neutron_arista/__init__.py +++ b/zaza/openstack/charm_tests/neutron_arista/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Collection of code for setting up and testing neutron-arista.""" +"""Collection of code for setting up and testing neutron-api-plugin-arista.""" diff --git a/zaza/openstack/charm_tests/neutron_arista/setup.py b/zaza/openstack/charm_tests/neutron_arista/setup.py index b990751..2b15d5a 100644 --- a/zaza/openstack/charm_tests/neutron_arista/setup.py +++ b/zaza/openstack/charm_tests/neutron_arista/setup.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Code for setting up neutron-arista.""" +"""Code for setting up neutron-api-plugin-arista.""" import logging import tenacity @@ -21,12 +21,12 @@ import zaza.openstack.charm_tests.neutron_arista.utils as arista_utils def test_fixture(): - """Pass arista-virt-test-fixture's IP address to neutron-arista.""" + """Pass arista-virt-test-fixture's IP address to Neutron.""" fixture_ip_addr = arista_utils.fixture_ip_addr() logging.info( - "{}'s IP address is '{}'. Passing it to neutron-arista..." + "{}'s IP address is '{}'. Passing it to neutron-api-plugin-arista..." .format(arista_utils.FIXTURE_APP_NAME, fixture_ip_addr)) - zaza.model.set_application_config('neutron-arista', + zaza.model.set_application_config('neutron-api-plugin-arista', {'eapi-host': fixture_ip_addr}) logging.info('Waiting for {} to become ready...'.format( diff --git a/zaza/openstack/charm_tests/neutron_arista/tests.py b/zaza/openstack/charm_tests/neutron_arista/tests.py index 4f6b1de..3f78bf7 100644 --- a/zaza/openstack/charm_tests/neutron_arista/tests.py +++ b/zaza/openstack/charm_tests/neutron_arista/tests.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Encapsulating `neutron-arista` testing.""" +"""Encapsulating `neutron-api-plugin-arista` testing.""" import logging import tenacity diff --git a/zaza/openstack/charm_tests/neutron_arista/utils.py b/zaza/openstack/charm_tests/neutron_arista/utils.py index 78732d6..bc86e4b 100644 --- a/zaza/openstack/charm_tests/neutron_arista/utils.py +++ b/zaza/openstack/charm_tests/neutron_arista/utils.py @@ -42,7 +42,7 @@ def query_fixture_networks(ip_addr): session.auth = (_FIXTURE_LOGIN, _FIXTURE_PASSWORD) data = { - 'id': 'Zaza neutron-arista tests', + 'id': 'Zaza neutron-api-plugin-arista tests', 'method': 'runCmds', 'jsonrpc': '2.0', 'params': { From 87d5667990c2eab293bb6e97716dcfb0b27ec4a4 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 24 Jun 2020 10:38:02 +0200 Subject: [PATCH 566/898] Download Arista image --- .../charm_tests/neutron_arista/setup.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron_arista/setup.py b/zaza/openstack/charm_tests/neutron_arista/setup.py index 2b15d5a..a1ade9e 100644 --- a/zaza/openstack/charm_tests/neutron_arista/setup.py +++ b/zaza/openstack/charm_tests/neutron_arista/setup.py @@ -15,9 +15,41 @@ """Code for setting up neutron-api-plugin-arista.""" import logging +import os import tenacity import zaza import zaza.openstack.charm_tests.neutron_arista.utils as arista_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +def download_arista_image(): + """Download arista-cvx-virt-test.qcow2 from a web server. + + If TEST_ARISTA_IMAGE_LOCAL isn't set, set it to + `/tmp/arista-cvx-virt-test.qcow2`. If TEST_ARISTA_IMAGE_REMOTE is set (e.g. + to `http://example.com/swift/v1/images/arista-cvx-virt-test.qcow2`), + download it to TEST_ARISTA_IMAGE_LOCAL. + """ + try: + os.environ['TEST_ARISTA_IMAGE_LOCAL'] + except KeyError: + os.environ['TEST_ARISTA_IMAGE_LOCAL'] = '' + if not os.environ['TEST_ARISTA_IMAGE_LOCAL']: + os.environ['TEST_ARISTA_IMAGE_LOCAL'] \ + = '/tmp/arista-cvx-virt-test.qcow2' + + try: + if os.environ['TEST_ARISTA_IMAGE_REMOTE']: + logging.info('Downloading Arista image from {}' + .format(os.environ['TEST_ARISTA_IMAGE_REMOTE'])) + openstack_utils.download_image( + os.environ['TEST_ARISTA_IMAGE_REMOTE'], + os.environ['TEST_ARISTA_IMAGE_LOCAL']) + except KeyError: + pass + + logging.info('Arista image can be found at {}' + .format(os.environ['TEST_ARISTA_IMAGE_LOCAL'])) def test_fixture(): From 68af0d2079089fbbc4f3d1c6cc4bb1fd73ca5ce9 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 24 Jun 2020 16:27:35 +0200 Subject: [PATCH 567/898] Clarification --- zaza/openstack/charm_tests/neutron_arista/setup.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron_arista/setup.py b/zaza/openstack/charm_tests/neutron_arista/setup.py index a1ade9e..4b8dfe7 100644 --- a/zaza/openstack/charm_tests/neutron_arista/setup.py +++ b/zaza/openstack/charm_tests/neutron_arista/setup.py @@ -25,10 +25,13 @@ import zaza.openstack.utilities.openstack as openstack_utils def download_arista_image(): """Download arista-cvx-virt-test.qcow2 from a web server. - If TEST_ARISTA_IMAGE_LOCAL isn't set, set it to - `/tmp/arista-cvx-virt-test.qcow2`. If TEST_ARISTA_IMAGE_REMOTE is set (e.g. - to `http://example.com/swift/v1/images/arista-cvx-virt-test.qcow2`), - download it to TEST_ARISTA_IMAGE_LOCAL. + The download will happen only if the env var TEST_ARISTA_IMAGE_REMOTE has + been set, so you don't have to set it if you already have the image + locally. + + If the env var TEST_ARISTA_IMAGE_LOCAL isn't set, it will be set to + `/tmp/arista-cvx-virt-test.qcow2`. This is where the image will be + downloaded to if TEST_ARISTA_IMAGE_REMOTE has been set. """ try: os.environ['TEST_ARISTA_IMAGE_LOCAL'] @@ -46,6 +49,8 @@ def download_arista_image(): os.environ['TEST_ARISTA_IMAGE_REMOTE'], os.environ['TEST_ARISTA_IMAGE_LOCAL']) except KeyError: + # TEST_ARISTA_IMAGE_REMOTE isn't set, which means the image is already + # available at TEST_ARISTA_IMAGE_LOCAL pass logging.info('Arista image can be found at {}' From b69455d3e83e4c10755d7fc2a17a9ad7f8d286f0 Mon Sep 17 00:00:00 2001 From: Alvaro Uria Date: Tue, 30 Jun 2020 12:10:26 +0200 Subject: [PATCH 568/898] CinderTests: Support enabled-services option charm-cinder supports units with a limited number of services running (api/scheduler on a unit, volume service on another one). Functional tests wait for all services to be restarted when a subset of them should be observed. Partial-Bug: #1779310 --- zaza/openstack/charm_tests/cinder/tests.py | 28 ++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 4cbf7c0..552d8a2 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -217,12 +217,22 @@ class CinderTests(test_utils.OpenStackBaseTest): @property def services(self): """Return a list services for the selected OpenStack release.""" - services = ['cinder-scheduler', 'cinder-volume'] - if (openstack_utils.get_os_release() >= - openstack_utils.get_os_release('xenial_ocata')): - services.append('apache2') + current_value = zaza.model.get_application_config( + self.application_name)['enabled-services']['value'] + + if current_value == "all": + services = ['cinder-scheduler', 'cinder-volume', 'cinder-api'] else: - services.append('cinder-api') + services = ['cinder-{}'.format(svc) + for svc in ('api', 'scheduler', 'volume') + if svc in current_value] + + if ('cinder-api' in services and + (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('xenial_ocata'))): + services.remove('cinder-api') + services.append('apache2') + return services def test_900_restart_on_config_change(self): @@ -246,13 +256,7 @@ class CinderTests(test_utils.OpenStackBaseTest): Pause service and check services are stopped then resume and check they are started """ - services = ['cinder-scheduler', 'cinder-volume'] - if (openstack_utils.get_os_release() >= - openstack_utils.get_os_release('xenial_ocata')): - services.append('apache2') - else: - services.append('cinder-api') - with self.pause_resume(services): + with self.pause_resume(self.services): logging.info("Testing pause resume") From e184ad74985b0b543b42f2626ef5d370137df428 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 30 Jun 2020 16:28:15 +0200 Subject: [PATCH 569/898] Wait for neutron to be connected to Arista before querying it --- zaza/openstack/charm_tests/neutron_arista/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron_arista/setup.py b/zaza/openstack/charm_tests/neutron_arista/setup.py index 4b8dfe7..286a23e 100644 --- a/zaza/openstack/charm_tests/neutron_arista/setup.py +++ b/zaza/openstack/charm_tests/neutron_arista/setup.py @@ -68,6 +68,8 @@ def test_fixture(): logging.info('Waiting for {} to become ready...'.format( arista_utils.FIXTURE_APP_NAME)) + zaza.model.wait_for_agent_status() + zaza.model.wait_for_application_states() for attempt in tenacity.Retrying( wait=tenacity.wait_fixed(10), # seconds stop=tenacity.stop_after_attempt(30), From e75902ac219c012284125e26ea809569bd1cc2ae Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 30 Jun 2020 14:42:25 +0000 Subject: [PATCH 570/898] Set application_name for masakari tests Set application_name for masakari tests so that the tests can be run without relying on getting the application name from the tests.yaml. --- zaza/openstack/charm_tests/masakari/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/masakari/tests.py b/zaza/openstack/charm_tests/masakari/tests.py index c2a3d85..d9490d2 100644 --- a/zaza/openstack/charm_tests/masakari/tests.py +++ b/zaza/openstack/charm_tests/masakari/tests.py @@ -38,7 +38,7 @@ class MasakariTest(test_utils.OpenStackBaseTest): @classmethod def setUpClass(cls): """Run class setup for running tests.""" - super(MasakariTest, cls).setUpClass() + super(MasakariTest, cls).setUpClass(application_name="masakari") cls.current_release = openstack_utils.get_os_release() cls.keystone_session = openstack_utils.get_overcloud_keystone_session() cls.model_name = zaza.model.get_juju_model() From 2331907134fefc9f2922df1ae86ce0f69c08f201 Mon Sep 17 00:00:00 2001 From: Camille Rodriguez <42066843+camille-rodriguez@users.noreply.github.com> Date: Tue, 30 Jun 2020 10:30:39 -0500 Subject: [PATCH 571/898] Add Gnocchi S3 storage backend functional test (#334) Add tests for Gnocchi S3 storage backend support. --- zaza/openstack/charm_tests/gnocchi/setup.py | 69 +++++++++++++++++++++ zaza/openstack/charm_tests/gnocchi/tests.py | 54 +++++++++++++++- 2 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 zaza/openstack/charm_tests/gnocchi/setup.py diff --git a/zaza/openstack/charm_tests/gnocchi/setup.py b/zaza/openstack/charm_tests/gnocchi/setup.py new file mode 100644 index 0000000..4995f22 --- /dev/null +++ b/zaza/openstack/charm_tests/gnocchi/setup.py @@ -0,0 +1,69 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup for Gnocchi tests.""" + +import logging + +import zaza.model as model +import zaza.openstack.utilities.openstack as openstack_utils + + +def configure_s3_backend(): + """Inject S3 parameters from Swift for Gnocchi config.""" + session = openstack_utils.get_overcloud_keystone_session() + ks_client = openstack_utils.get_keystone_session_client(session) + + logging.info('Retrieving S3 connection data from Swift') + token_data = ks_client.tokens.get_token_data(session.get_token()) + project_id = token_data['token']['project']['id'] + user_id = token_data['token']['user']['id'] + + # Store URL to service providing S3 compatible API + for entry in token_data['token']['catalog']: + if entry['type'] == 's3': + for endpoint in entry['endpoints']: + if endpoint['interface'] == 'public': + s3_region = endpoint['region'] + s3_endpoint = endpoint['url'] + + # Create AWS compatible application credentials in Keystone + ec2_creds = ks_client.ec2.create(user_id, project_id) + + logging.info('Changing Gnocchi charm config to connect to S3') + model.set_application_config( + 'gnocchi', + {'s3-endpoint-url': s3_endpoint, + 's3-region-name': s3_region, + 's3-access-key-id': ec2_creds.access, + 's3-secret-access-key': ec2_creds.secret} + ) + logging.info('Waiting for units to execute config-changed hook') + model.wait_for_agent_status() + logging.info('Waiting for units to reach target states') + model.wait_for_application_states( + states={ + 'gnocchi': { + 'workload-status-': 'active', + 'workload-status-message': 'Unit is ready' + }, + 'ceilometer': { + 'workload-status': 'blocked', + 'workload-status-message': 'Run the ' + + 'ceilometer-upgrade action on the leader ' + + 'to initialize ceilometer and gnocchi' + } + } + ) + model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index f789cff..680bf3c 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -16,9 +16,11 @@ """Encapsulate Gnocchi testing.""" +import boto3 import logging - +import pprint from gnocchiclient.v1 import client as gnocchi_client + import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -58,3 +60,53 @@ class GnocchiTest(test_utils.OpenStackBaseTest): """ with self.pause_resume(self.services): logging.info("Testing pause and resume") + + +class GnocchiS3Test(test_utils.OpenStackBaseTest): + """Test Gnocchi for S3 storage backend.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(GnocchiS3Test, cls).setUpClass() + + session = openstack_utils.get_overcloud_keystone_session() + ks_client = openstack_utils.get_keystone_session_client(session) + + # Get token data so we can glean our user_id and project_id + token_data = ks_client.tokens.get_token_data(session.get_token()) + project_id = token_data['token']['project']['id'] + user_id = token_data['token']['user']['id'] + + # Store URL to service providing S3 compatible API + for entry in token_data['token']['catalog']: + if entry['type'] == 's3': + for endpoint in entry['endpoints']: + if endpoint['interface'] == 'public': + cls.s3_region = endpoint['region'] + cls.s3_endpoint = endpoint['url'] + + # Create AWS compatible application credentials in Keystone + cls.ec2_creds = ks_client.ec2.create(user_id, project_id) + + def test_s3_list_gnocchi_buckets(self): + """Verify that the gnocchi buckets were created in the S3 backend.""" + kwargs = { + 'region_name': self.s3_region, + 'aws_access_key_id': self.ec2_creds.access, + 'aws_secret_access_key': self.ec2_creds.secret, + 'endpoint_url': self.s3_endpoint, + 'verify': self.cacert, + } + s3_client = boto3.client('s3', **kwargs) + + bucket_names = ['gnocchi-measure', 'gnocchi-aggregates'] + # Validate their presence + bucket_list = s3_client.list_buckets() + logging.info(pprint.pformat(bucket_list)) + for bkt in bucket_list['Buckets']: + for gnocchi_bkt in bucket_names: + if bkt['Name'] == gnocchi_bkt: + break + else: + AssertionError('Bucket "{}" not found'.format(gnocchi_bkt)) From 1e682e94801927ed05f3a8074a33b2c8a8c5cc1b Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 1 Jul 2020 11:07:49 +0200 Subject: [PATCH 572/898] Fix application name in trace --- zaza/openstack/charm_tests/neutron_arista/setup.py | 9 +++++---- zaza/openstack/charm_tests/neutron_arista/utils.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron_arista/setup.py b/zaza/openstack/charm_tests/neutron_arista/setup.py index 286a23e..4e35d33 100644 --- a/zaza/openstack/charm_tests/neutron_arista/setup.py +++ b/zaza/openstack/charm_tests/neutron_arista/setup.py @@ -61,13 +61,14 @@ def test_fixture(): """Pass arista-virt-test-fixture's IP address to Neutron.""" fixture_ip_addr = arista_utils.fixture_ip_addr() logging.info( - "{}'s IP address is '{}'. Passing it to neutron-api-plugin-arista..." - .format(arista_utils.FIXTURE_APP_NAME, fixture_ip_addr)) - zaza.model.set_application_config('neutron-api-plugin-arista', + "{}'s IP address is '{}'. Passing it to {}..." + .format(arista_utils.FIXTURE_APP_NAME, fixture_ip_addr, + arista_utils.PLUGIN_APP_NAME)) + zaza.model.set_application_config(arista_utils.PLUGIN_APP_NAME, {'eapi-host': fixture_ip_addr}) logging.info('Waiting for {} to become ready...'.format( - arista_utils.FIXTURE_APP_NAME)) + arista_utils.PLUGIN_APP_NAME)) zaza.model.wait_for_agent_status() zaza.model.wait_for_application_states() for attempt in tenacity.Retrying( diff --git a/zaza/openstack/charm_tests/neutron_arista/utils.py b/zaza/openstack/charm_tests/neutron_arista/utils.py index bc86e4b..6fb77b3 100644 --- a/zaza/openstack/charm_tests/neutron_arista/utils.py +++ b/zaza/openstack/charm_tests/neutron_arista/utils.py @@ -20,6 +20,7 @@ import urllib3 import zaza FIXTURE_APP_NAME = 'arista-virt-test-fixture' +PLUGIN_APP_NAME = 'neutron-api-plugin-arista' def fixture_ip_addr(): @@ -42,7 +43,7 @@ def query_fixture_networks(ip_addr): session.auth = (_FIXTURE_LOGIN, _FIXTURE_PASSWORD) data = { - 'id': 'Zaza neutron-api-plugin-arista tests', + 'id': 'Zaza {} tests'.format(PLUGIN_APP_NAME), 'method': 'runCmds', 'jsonrpc': '2.0', 'params': { From 58182b86b893ed09fe7b397ebd11aeab4c1673da Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 1 Jul 2020 13:38:19 +0100 Subject: [PATCH 573/898] Deprecate zaza.openstack.utilities.juju (#340) This is the 3rd change in the effort to deprecate zaza.openstack.utilities.juju in favour of zaza.utilities.juju. All functions now just wrap their equivalents in zaza.utilities.juju and a decorator has been added which logs a warning if the function is used. --- .../utilities/test_zaza_utilities_juju.py | 312 ------------------ zaza/openstack/utilities/juju.py | 237 +++++-------- 2 files changed, 82 insertions(+), 467 deletions(-) delete mode 100644 unit_tests/utilities/test_zaza_utilities_juju.py diff --git a/unit_tests/utilities/test_zaza_utilities_juju.py b/unit_tests/utilities/test_zaza_utilities_juju.py deleted file mode 100644 index 4255f9e..0000000 --- a/unit_tests/utilities/test_zaza_utilities_juju.py +++ /dev/null @@ -1,312 +0,0 @@ -# Copyright 2018 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock -import unit_tests.utils as ut_utils -from zaza.openstack.utilities import juju as juju_utils - - -class TestJujuUtils(ut_utils.BaseTestCase): - - def setUp(self): - super(TestJujuUtils, self).setUp() - - # Juju Status Object and data - self.key = "instance-id" - self.key_data = "machine-uuid" - self.machine = "1" - self.machine_data = {self.key: self.key_data} - self.unit = "app/1" - self.unit_data = {"machine": self.machine} - self.application = "app" - self.application_data = {"units": {self.unit: self.unit_data}} - self.subordinate_application = "subordinate_application" - self.subordinate_application_data = { - "subordinate-to": [self.application]} - self.juju_status = mock.MagicMock() - self.juju_status.name = "juju_status_object" - self.juju_status.applications.get.return_value = self.application_data - self.juju_status.machines.get.return_value = self.machine_data - - # Model - self.patch_object(juju_utils, "model") - self.model_name = "model-name" - self.model.get_juju_model.return_value = self.model_name - self.model.get_status.return_value = self.juju_status - self.run_output = {"Code": "0", "Stderr": "", "Stdout": "RESULT"} - self.error_run_output = {"Code": "1", "Stderr": "ERROR", "Stdout": ""} - self.model.run_on_unit.return_value = self.run_output - - # Clouds - self.cloud_name = "FakeCloudName" - self.cloud_type = "FakeCloudType" - self.clouds = { - "clouds": - {self.cloud_name: - {"type": self.cloud_type}}} - - # Controller - self.patch_object(juju_utils, "controller") - self.controller.get_cloud.return_value = self.cloud_name - - def test_get_application_status(self): - self.patch_object(juju_utils, "get_full_juju_status") - self.get_full_juju_status.return_value = self.juju_status - - # Full status juju object return - self.assertEqual( - juju_utils.get_application_status(), self.juju_status) - self.get_full_juju_status.assert_called_once() - - # Application only dictionary return - self.assertEqual( - juju_utils.get_application_status(application=self.application), - self.application_data) - - # Unit no application dictionary return - self.assertEqual( - juju_utils.get_application_status(unit=self.unit), - self.unit_data) - - def test_get_cloud_configs(self): - self.patch_object(juju_utils.Path, "home") - self.patch_object(juju_utils.generic_utils, "get_yaml_config") - self.get_yaml_config.return_value = self.clouds - - # All the cloud configs - self.assertEqual(juju_utils.get_cloud_configs(), self.clouds) - - # With cloud specified - self.assertEqual(juju_utils.get_cloud_configs(self.cloud_name), - self.clouds["clouds"][self.cloud_name]) - - def test_get_full_juju_status(self): - self.assertEqual(juju_utils.get_full_juju_status(), self.juju_status) - self.model.get_status.assert_called_once_with(model_name=None) - - def test_get_machines_for_application(self): - self.patch_object(juju_utils, "get_application_status") - self.get_application_status.return_value = self.application_data - - # Machine data - self.assertEqual( - next(juju_utils.get_machines_for_application(self.application)), - self.machine) - self.get_application_status.assert_called_once() - - # Subordinate application has no units - def _get_application_status(application, model_name=None): - _apps = { - self.application: self.application_data, - self.subordinate_application: - self.subordinate_application_data} - return _apps[application] - self.get_application_status.side_effect = _get_application_status - - self.assertEqual( - next(juju_utils.get_machines_for_application( - self.subordinate_application)), - self.machine) - - def test_get_unit_name_from_host_name(self): - unit_mock1 = mock.MagicMock() - unit_mock1.data = {'machine-id': 12} - unit_mock1.entity_id = 'myapp/2' - unit_mock2 = mock.MagicMock() - unit_mock2.data = {'machine-id': 15} - unit_mock2.entity_id = 'myapp/5' - self.model.get_units.return_value = [unit_mock1, unit_mock2] - self.assertEqual( - juju_utils.get_unit_name_from_host_name('juju-model-12', 'myapp'), - 'myapp/2') - - def test_get_machine_status(self): - self.patch_object(juju_utils, "get_full_juju_status") - self.get_full_juju_status.return_value = self.juju_status - - # All machine data - self.assertEqual( - juju_utils.get_machine_status(self.machine), - self.machine_data) - self.get_full_juju_status.assert_called_once() - - # Request a specific key - self.assertEqual( - juju_utils.get_machine_status(self.machine, self.key), - self.key_data) - - def test_get_machine_uuids_for_application(self): - self.patch_object(juju_utils, "get_machines_for_application") - self.get_machines_for_application.return_value = [self.machine] - - self.assertEqual( - next(juju_utils.get_machine_uuids_for_application( - self.application)), - self.machine_data.get("instance-id")) - self.get_machines_for_application.assert_called_once_with( - self.application, model_name=None) - - def test_get_provider_type(self): - self.patch_object(juju_utils, "get_cloud_configs") - self.get_cloud_configs.return_value = {"type": self.cloud_type} - self.assertEqual(juju_utils.get_provider_type(), - self.cloud_type) - self.get_cloud_configs.assert_called_once_with(self.cloud_name) - - def test_remote_run(self): - _cmd = "do the thing" - - # Success - self.assertEqual(juju_utils.remote_run(self.unit, _cmd), - self.run_output["Stdout"]) - self.model.run_on_unit.assert_called_once_with( - self.unit, _cmd, timeout=None, model_name=None) - - # Non-fatal failure - self.model.run_on_unit.return_value = self.error_run_output - self.assertEqual( - juju_utils.remote_run( - self.unit, - _cmd, - fatal=False, - model_name=None), - self.error_run_output["Stderr"]) - - # Fatal failure - with self.assertRaises(Exception): - juju_utils.remote_run(self.unit, _cmd, fatal=True) - - def test_get_unit_names(self): - self.patch('zaza.model.get_first_unit_name', new_callable=mock.Mock(), - name='_get_first_unit_name') - juju_utils._get_unit_names(['aunit/0', 'otherunit/0']) - self.assertFalse(self._get_first_unit_name.called) - - def test_get_unit_names_called_with_application_name(self): - self.patch_object(juju_utils, 'model') - juju_utils._get_unit_names(['aunit', 'otherunit/0']) - self.model.get_first_unit_name.assert_called() - - def test_get_relation_from_unit(self): - self.patch_object(juju_utils, '_get_unit_names') - self.patch_object(juju_utils, 'yaml') - self.patch_object(juju_utils, 'model') - self._get_unit_names.return_value = ['aunit/0', 'otherunit/0'] - data = {'foo': 'bar'} - self.model.get_relation_id.return_value = 42 - self.model.run_on_unit.return_value = {'Code': 0, 'Stdout': str(data)} - juju_utils.get_relation_from_unit('aunit/0', 'otherunit/0', - 'arelation') - self.model.run_on_unit.assert_called_with( - 'aunit/0', - 'relation-get --format=yaml -r "42" - "otherunit/0"', - model_name=None) - self.yaml.safe_load.assert_called_with(str(data)) - - def test_get_relation_from_unit_fails(self): - self.patch_object(juju_utils, '_get_unit_names') - self.patch_object(juju_utils, 'yaml') - self.patch_object(juju_utils, 'model') - self._get_unit_names.return_value = ['aunit/0', 'otherunit/0'] - self.model.get_relation_id.return_value = 42 - self.model.run_on_unit.return_value = {'Code': 1, 'Stderr': 'ERROR'} - with self.assertRaises(Exception): - juju_utils.get_relation_from_unit('aunit/0', 'otherunit/0', - 'arelation') - self.model.run_on_unit.assert_called_with( - 'aunit/0', - 'relation-get --format=yaml -r "42" - "otherunit/0"', - model_name=None) - self.assertFalse(self.yaml.safe_load.called) - - def test_leader_get(self): - self.patch_object(juju_utils, 'yaml') - self.patch_object(juju_utils, 'model') - data = {'foo': 'bar'} - self.model.run_on_leader.return_value = { - 'Code': 0, 'Stdout': str(data)} - juju_utils.leader_get('application') - self.model.run_on_leader.assert_called_with( - 'application', 'leader-get --format=yaml ', model_name=None) - self.yaml.safe_load.assert_called_with(str(data)) - - def test_leader_get_key(self): - self.patch_object(juju_utils, 'yaml') - self.patch_object(juju_utils, 'model') - data = {'foo': 'bar'} - self.model.run_on_leader.return_value = { - 'Code': 0, 'Stdout': data['foo']} - juju_utils.leader_get('application', 'foo') - self.model.run_on_leader.assert_called_with( - 'application', 'leader-get --format=yaml foo', model_name=None) - self.yaml.safe_load.assert_called_with(data['foo']) - - def test_leader_get_fails(self): - self.patch_object(juju_utils, 'yaml') - self.patch_object(juju_utils, 'model') - self.model.run_on_leader.return_value = { - 'Code': 1, 'Stderr': 'ERROR'} - with self.assertRaises(Exception): - juju_utils.leader_get('application') - self.model.run_on_leader.assert_called_with( - 'application', 'leader-get --format=yaml ', - model_name=None) - self.assertFalse(self.yaml.safe_load.called) - - def test_get_machine_series(self): - self.patch( - 'zaza.openstack.utilities.juju.get_machine_status', - new_callable=mock.MagicMock(), - name='_get_machine_status' - ) - self._get_machine_status.return_value = 'xenial' - expected = 'xenial' - actual = juju_utils.get_machine_series('6') - self._get_machine_status.assert_called_with( - machine='6', - key='series', - model_name=None - ) - self.assertEqual(expected, actual) - - def test_get_subordinate_units(self): - juju_status = mock.MagicMock() - juju_status.applications = { - 'nova-compute': { - 'units': { - 'nova-compute/0': { - 'subordinates': { - 'neutron-openvswitch/2': { - 'charm': 'cs:neutron-openvswitch-22'}}}}}, - 'cinder': { - 'units': { - 'cinder/1': { - 'subordinates': { - 'cinder-hacluster/0': { - 'charm': 'cs:hacluster-42'}, - 'cinder-ceph/3': { - 'charm': 'cs:cinder-ceph-2'}}}}}, - } - self.assertEqual( - sorted(juju_utils.get_subordinate_units( - ['nova-compute/0', 'cinder/1'], - status=juju_status)), - sorted(['neutron-openvswitch/2', 'cinder-hacluster/0', - 'cinder-ceph/3'])) - self.assertEqual( - juju_utils.get_subordinate_units( - ['nova-compute/0', 'cinder/1'], - charm_name='ceph', - status=juju_status), - ['cinder-ceph/3']) diff --git a/zaza/openstack/utilities/juju.py b/zaza/openstack/utilities/juju.py index 3bdf4b8..073a26f 100644 --- a/zaza/openstack/utilities/juju.py +++ b/zaza/openstack/utilities/juju.py @@ -13,18 +13,31 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Module for interacting with juju.""" -import os -from pathlib import Path -import yaml +"""Deprecated, please use zaza.utilities.juju.""" -from zaza import ( - model, - controller, -) -from zaza.openstack.utilities import generic as generic_utils +import logging +import functools + +import zaza.utilities.juju +def deprecate(): + """Add a deprecation warning to wrapped function.""" + def wrap(f): + + @functools.wraps(f) + def wrapped_f(*args, **kwargs): + msg = ( + "{} from zaza.openstack.utilities.juju is deprecated. " + "Please use the equivalent from zaza.utilities.juju".format( + f.__name__)) + logging.warning(msg) + return f(*args, **kwargs) + return wrapped_f + return wrap + + +@deprecate() def get_application_status(application=None, unit=None, model_name=None): """Return the juju status for an application. @@ -37,39 +50,29 @@ def get_application_status(application=None, unit=None, model_name=None): :returns: Juju status output for an application :rtype: dict """ - status = get_full_juju_status() - - if unit and not application: - application = unit.split("/")[0] - - if application: - status = status.applications.get(application) - if unit: - status = status.get("units").get(unit) - return status + return zaza.utilities.juju.get_application_status( + application=application, + unit=unit, + model_name=model_name) -def get_application_ip(application): +@deprecate() +def get_application_ip(application, model_name=None): """Get the application's IP address. :param application: Application name :type application: str + :param model_name: Name of model to query. + :type model_name: str :returns: Application's IP address :rtype: str """ - try: - app_config = model.get_application_config(application) - except KeyError: - return '' - vip = app_config.get("vip").get("value") - if vip: - ip = vip - else: - unit = model.get_units(application)[0] - ip = unit.public_address - return ip + return zaza.utilities.juju.get_application_ip( + application, + model_name=model_name) +@deprecate() def get_cloud_configs(cloud=None): """Get cloud configuration from local clouds.yaml. @@ -81,14 +84,11 @@ def get_cloud_configs(cloud=None): :returns: Dictionary of cloud configuration :rtype: dict """ - home = str(Path.home()) - cloud_config = os.path.join(home, ".local", "share", "juju", "clouds.yaml") - if cloud: - return generic_utils.get_yaml_config(cloud_config)["clouds"].get(cloud) - else: - return generic_utils.get_yaml_config(cloud_config) + return zaza.utilities.juju.get_cloud_configs( + cloud=cloud) +@deprecate() def get_full_juju_status(model_name=None): """Return the full juju status output. @@ -97,10 +97,11 @@ def get_full_juju_status(model_name=None): :returns: Full juju status output :rtype: dict """ - status = model.get_status(model_name=model_name) - return status + return zaza.utilities.juju.get_full_juju_status( + model_name=model_name) +@deprecate() def get_machines_for_application(application, model_name=None): """Return machines for a given application. @@ -111,20 +112,12 @@ def get_machines_for_application(application, model_name=None): :returns: machines for an application :rtype: Iterator[str] """ - status = get_application_status(application, model_name=model_name) - if not status: - return - - # libjuju juju status no longer has units for subordinate charms - # Use the application it is subordinate-to to find machines - if not status.get("units") and status.get("subordinate-to"): - status = get_application_status(status.get("subordinate-to")[0], - model_name=model_name) - - for unit in status.get("units").keys(): - yield status.get("units").get(unit).get("machine") + return zaza.utilities.juju.get_machines_for_application( + application, + model_name=model_name) +@deprecate() def get_unit_name_from_host_name(host_name, application, model_name=None): """Return the juju unit name corresponding to a hostname. @@ -135,17 +128,13 @@ def get_unit_name_from_host_name(host_name, application, model_name=None): :param model_name: Name of model to query. :type model_name: str """ - # Assume that a juju managed hostname always ends in the machine number and - # remove the domain name if it present. - machine_number = host_name.split('-')[-1].split('.')[0] - unit_names = [ - u.entity_id - for u in model.get_units(application_name=application, - model_name=model_name) - if int(u.data['machine-id']) == int(machine_number)] - return unit_names[0] + return zaza.utilities.juju.get_unit_name_from_host_name( + host_name, + application, + model_name=model_name) +@deprecate() def get_machine_status(machine, key=None, model_name=None): """Return the juju status for a machine. @@ -158,17 +147,13 @@ def get_machine_status(machine, key=None, model_name=None): :returns: Juju status output for a machine :rtype: dict """ - status = get_full_juju_status(model_name=model_name) - if "lxd" in machine: - host = machine.split('/')[0] - status = status.machines.get(host)['containers'][machine] - else: - status = status.machines.get(machine) - if key: - status = status.get(key) - return status + return zaza.utilities.juju.get_machine_status( + machine, + key=key, + model_name=model_name) +@deprecate() def get_machine_series(machine, model_name=None): """Return the juju series for a machine. @@ -179,13 +164,12 @@ def get_machine_series(machine, model_name=None): :returns: Juju series :rtype: string """ - return get_machine_status( - machine=machine, - key='series', - model_name=model_name - ) + return zaza.utilities.juju.get_machine_series( + machine, + model_name=model_name) +@deprecate() def get_machine_uuids_for_application(application, model_name=None): """Return machine uuids for a given application. @@ -196,30 +180,22 @@ def get_machine_uuids_for_application(application, model_name=None): :returns: machine uuuids for an application :rtype: Iterator[str] """ - for machine in get_machines_for_application(application, - model_name=model_name): - yield get_machine_status(machine, key="instance-id", - model_name=model_name) + return zaza.utilities.juju.get_machine_uuids_for_application( + application, + model_name=model_name) +@deprecate() def get_provider_type(): """Get the type of the undercloud. :returns: Name of the undercloud type :rtype: string """ - cloud = controller.get_cloud() - if cloud: - # If the controller was deployed from this system with - # the cloud configured in ~/.local/share/juju/clouds.yaml - # Determine the cloud type directly - return get_cloud_configs(cloud)["type"] - else: - # If the controller was deployed elsewhere - # For now assume openstack - return "openstack" + return zaza.utilities.juju.get_provider_type() +@deprecate() def remote_run(unit, remote_cmd, timeout=None, fatal=None, model_name=None): """Run command on unit and return the output. @@ -238,46 +214,15 @@ def remote_run(unit, remote_cmd, timeout=None, fatal=None, model_name=None): :rtype: string :raises: model.CommandRunFailed """ - if fatal is None: - fatal = True - result = model.run_on_unit( + return zaza.utilities.juju.remote_run( unit, remote_cmd, timeout=timeout, + fatal=fatal, model_name=model_name) - if result: - if int(result.get("Code")) == 0: - return result.get("Stdout") - else: - if fatal: - raise model.CommandRunFailed(remote_cmd, result) - return result.get("Stderr") - - -def _get_unit_names(names, model_name=None): - """Resolve given application names to first unit name of said application. - - Helper function that resolves application names to first unit name of - said application. Any already resolved unit names are returned as-is. - - :param names: List of units/applications to translate - :type names: list(str) - :param model_name: Name of model to query. - :type model_name: str - :returns: List of units - :rtype: list(str) - """ - result = [] - for name in names: - if '/' in name: - result.append(name) - else: - result.append(model.get_first_unit_name( - name, - model_name=model_name)) - return result +@deprecate() def get_relation_from_unit(entity, remote_entity, remote_interface_name, model_name=None): """Get relation data passed between two units. @@ -302,22 +247,14 @@ def get_relation_from_unit(entity, remote_entity, remote_interface_name, :rtype: dict :raises: model.CommandRunFailed """ - application = entity.split('/')[0] - remote_application = remote_entity.split('/')[0] - rid = model.get_relation_id(application, remote_application, - remote_interface_name=remote_interface_name, - model_name=model_name) - (unit, remote_unit) = _get_unit_names( - [entity, remote_entity], + return zaza.utilities.juju.get_relation_from_unit( + entity, + remote_entity, + remote_interface_name, model_name=model_name) - cmd = 'relation-get --format=yaml -r "{}" - "{}"' .format(rid, remote_unit) - result = model.run_on_unit(unit, cmd, model_name=model_name) - if result and int(result.get('Code')) == 0: - return yaml.safe_load(result.get('Stdout')) - else: - raise model.CommandRunFailed(cmd, result) +@deprecate() def leader_get(application, key='', model_name=None): """Get leader settings from leader unit of named application. @@ -331,14 +268,13 @@ def leader_get(application, key='', model_name=None): :rtype: dict :raises: model.CommandRunFailed """ - cmd = 'leader-get --format=yaml {}'.format(key) - result = model.run_on_leader(application, cmd, model_name=model_name) - if result and int(result.get('Code')) == 0: - return yaml.safe_load(result.get('Stdout')) - else: - raise model.CommandRunFailed(cmd, result) + return zaza.utilities.juju.leader_get( + application, + key=key, + model_name=model_name) +@deprecate() def get_subordinate_units(unit_list, charm_name=None, status=None, model_name=None): """Get a list of all subordinate units associated with units in unit_list. @@ -369,17 +305,8 @@ def get_subordinate_units(unit_list, charm_name=None, status=None, :returns: List of matching unit names. :rtype: [] """ - if not status: - status = model.get_status(model_name=model_name) - sub_units = [] - for unit_name in unit_list: - app_name = unit_name.split('/')[0] - subs = status.applications[app_name]['units'][unit_name].get( - 'subordinates') or {} - if charm_name: - for unit_name, unit_data in subs.items(): - if charm_name in unit_data['charm']: - sub_units.append(unit_name) - else: - sub_units.extend([n for n in subs.keys()]) - return sub_units + return zaza.utilities.juju.get_subordinate_units( + unit_list, + charm_name=charm_name, + status=status, + model_name=model_name) From fe54ebeb985d94882c788656ba4398e9d2e79224 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 1 Jul 2020 16:20:29 +0200 Subject: [PATCH 574/898] Minor improvements to keystone tests --- zaza/openstack/charm_tests/keystone/tests.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 9f1d31d..ed5f42a 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -21,7 +21,7 @@ import keystoneauth1 import zaza.model import zaza.openstack.utilities.exceptions as zaza_exceptions -import zaza.openstack.utilities.juju as juju_utils +import zaza.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.openstack.charm_tests.test_utils as test_utils @@ -262,6 +262,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT openrc['OS_AUTH_URL'] = ( openrc['OS_AUTH_URL'].replace('http', 'https')) + logging.info('keystone IP {}'.format(ip)) keystone_session = openstack_utils.get_keystone_session( openrc) keystone_client = openstack_utils.get_keystone_session_client( @@ -319,10 +320,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): 'OS_PROJECT_DOMAIN_NAME': DEMO_DOMAIN, 'OS_PROJECT_NAME': DEMO_PROJECT, } - with self.config_change( - {'preferred-api-version': self.default_api_version}, - {'preferred-api-version': self.api_v3}, - application_name="keystone"): + with self.v3_keystone_preferred(): for ip in self.keystone_ips: openrc.update( {'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip)}) From 670292c6830d765b779d98ac4111b0a94446fb17 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 1 Jul 2020 18:05:06 +0200 Subject: [PATCH 575/898] Remove more deprecation warnings --- unit_tests/utilities/test_zaza_utilities_openstack.py | 4 ++-- zaza/openstack/utilities/generic.py | 2 +- zaza/openstack/utilities/openstack.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index a6aa190..7271708 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -809,12 +809,12 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): name='_get_os_version' ) self.patch( - 'zaza.openstack.utilities.juju.get_machines_for_application', + 'zaza.utilities.juju.get_machines_for_application', new_callable=mock.MagicMock(), name='_get_machines' ) self.patch( - 'zaza.openstack.utilities.juju.get_machine_series', + 'zaza.utilities.juju.get_machine_series', new_callable=mock.MagicMock(), name='_get_machine_series' ) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 0874cc6..dd8b1e0 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -23,9 +23,9 @@ import telnetlib import yaml from zaza import model -from zaza.openstack.utilities import juju as juju_utils from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE +from zaza.utilities import juju as juju_utils def assertActionRanOK(action): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index bb834f0..4c79a7d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -40,6 +40,7 @@ from keystoneauth1.identity import ( ) import zaza.openstack.utilities.cert as cert import zaza.utilities.deployment_env as deployment_env +import zaza.utilities.juju as juju_utils from novaclient import client as novaclient_client from neutronclient.v2_0 import client as neutronclient from neutronclient.common import exceptions as neutronexceptions @@ -66,7 +67,6 @@ from zaza import model from zaza.openstack.utilities import ( exceptions, generic as generic_utils, - juju as juju_utils, ) CIRROS_RELEASE_URL = 'http://download.cirros-cloud.net/version/released' From 520830905bb4ed45705c5f5818fdb9e18474161d Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 2 Jul 2020 13:03:42 +0200 Subject: [PATCH 576/898] Increase wait time for cloud_init_complete We often see test runs being killed prematurely due to slow to complete cloud-init step on a loaded CI cloud. Related issue #311 --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index bb834f0..28ba29a 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2283,7 +2283,7 @@ def get_ports_from_device_id(neutron_client, device_id): @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=120), - reraise=True, stop=tenacity.stop_after_attempt(12)) + reraise=True, stop=tenacity.stop_after_delay(1800)) def cloud_init_complete(nova_client, vm_id, bootstring): """Wait for cloud init to complete on the given vm. From 0f3d9bf7c481486b4ac6ab4e8919b3d7a3faeb6c Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 2 Jul 2020 13:33:03 +0200 Subject: [PATCH 577/898] Move useful helpers from Neutron tests to OpenStackBaseTest The dual-instance launch and retrieve is a common pattern in tests. Let's share them. --- zaza/openstack/charm_tests/neutron/tests.py | 54 ++---------- zaza/openstack/charm_tests/test_utils.py | 94 +++++++++++++++++++++ 2 files changed, 99 insertions(+), 49 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index f270604..2ddfda4 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -25,10 +25,7 @@ import logging import tenacity import unittest -import novaclient - import zaza -import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.nova.utils as nova_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.guest as guest @@ -608,7 +605,7 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): logging.info('Testing pause resume') -class NeutronNetworkingBase(unittest.TestCase): +class NeutronNetworkingBase(test_utils.OpenStackBaseTest): """Base for checking openstack instances have valid networking.""" RESOURCE_PREFIX = 'zaza-neutrontests' @@ -616,10 +613,7 @@ class NeutronNetworkingBase(unittest.TestCase): @classmethod def setUpClass(cls): """Run class setup for running Neutron API Networking tests.""" - cls.keystone_session = ( - openstack_utils.get_overcloud_keystone_session()) - cls.nova_client = ( - openstack_utils.get_nova_session_client(cls.keystone_session)) + super(NeutronNetworkingBase, cls).setUpClass() cls.neutron_client = ( openstack_utils.get_neutron_session_client(cls.keystone_session)) # NOTE(fnordahl): in the event of a test failure we do not want to run @@ -737,44 +731,6 @@ class NeutronNetworkingBase(unittest.TestCase): assert agent['admin_state_up'] assert agent['alive'] - def launch_guests(self): - """Launch two guests to use in tests.""" - guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX)) - guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX)) - - def retrieve_guest(self, nova_client, guest_name): - """Return guest matching name. - - :param nova_client: Nova client to use when checking status - :type nova_client: Nova client - :returns: the matching guest - :rtype: Union[novaclient.Server, None] - """ - try: - return nova_client.servers.find(name=guest_name) - except novaclient.exceptions.NotFound: - return None - - def retrieve_guests(self, nova_client): - """Return test guests. - - :param nova_client: Nova client to use when checking status - :type nova_client: Nova client - :returns: the matching guest - :rtype: Union[novaclient.Server, None] - """ - instance_1 = self.retrieve_guest( - nova_client, - '{}-ins-1'.format(self.RESOURCE_PREFIX)) - instance_2 = self.retrieve_guest( - nova_client, - '{}-ins-1'.format(self.RESOURCE_PREFIX)) - return instance_1, instance_2 - def check_connectivity(self, instance_1, instance_2): """Run North/South and East/West connectivity tests.""" def verify(stdin, stdout, stderr): @@ -845,7 +801,7 @@ class NeutronNetworkingTest(NeutronNetworkingBase): def test_instances_have_networking(self): """Validate North/South and East/West networking.""" self.launch_guests() - instance_1, instance_2 = self.retrieve_guests(self.nova_client) + instance_1, instance_2 = self.retrieve_guests() self.check_connectivity(instance_1, instance_2) self.run_tearDown = True @@ -855,10 +811,10 @@ class NeutronNetworkingVRRPTests(NeutronNetworkingBase): def test_gateway_failure(self): """Validate networking in the case of a gateway failure.""" - instance_1, instance_2 = self.retrieve_guests(self.nova_client) + instance_1, instance_2 = self.retrieve_guests() if not all([instance_1, instance_2]): self.launch_guests() - instance_1, instance_2 = self.retrieve_guests(self.nova_client) + instance_1, instance_2 = self.retrieve_guests() self.check_connectivity(instance_1, instance_2) routers = self.neutron_client.list_routers( diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 47c12d7..4b4e1d0 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -16,12 +16,17 @@ import contextlib import logging import ipaddress import subprocess +import tenacity import unittest +import novaclient + import zaza.model as model import zaza.charm_lifecycle.utils as lifecycle_utils +import zaza.openstack.configure.guest as configure_guest import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.charm_tests.glance.setup as glance_setup def skipIfNotHA(service_name): @@ -432,6 +437,95 @@ class OpenStackBaseTest(BaseCharmTest): cls.keystone_session = openstack_utils.get_overcloud_keystone_session( model_name=cls.model_name) cls.cacert = openstack_utils.get_cacert() + cls.nova_client = ( + openstack_utils.get_nova_session_client(cls.keystone_session)) + + def launch_guest(self, guest_name, userdata=None): + """Launch two guests to use in tests. + + Note that it is up to the caller to have set the RESOURCE_PREFIX class + variable prior to calling this method. + + Also note that this method will remove any already existing instance + with same name as what is requested. + + :param guest_name: Name of instance + :type guest_name: str + :param userdata: Userdata to attach to instance + :type userdata: Optional[str] + :returns: Nova instance objects + :rtype: Server + """ + instance_name = '{}-{}'.format(self.RESOURCE_PREFIX, guest_name) + + instance = self.retrieve_guest(instance_name) + if instance: + logging.info('Removing already existing instance ({}) with ' + 'requested name ({})' + .format(instance.id, instance_name)) + openstack_utils.delete_resource( + self.nova_client.servers, + instance.id, + msg="server") + + return configure_guest.launch_instance( + glance_setup.LTS_IMAGE_NAME, + vm_name=instance_name, + userdata=userdata) + + def launch_guests(self, userdata=None): + """Launch two guests to use in tests. + + Note that it is up to the caller to have set the RESOURCE_PREFIX class + variable prior to calling this method. + + :param userdata: Userdata to attach to instance + :type userdata: Optional[str] + :returns: List of launched Nova instance objects + :rtype: List[Server] + """ + launched_instances = [] + for guest_number in range(1, 2+1): + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)): + with attempt: + launched_instances.append( + self.launch_guest( + guest_name='ins-{}'.format(guest_number), + userdata=userdata)) + return launched_instances + + def retrieve_guest(self, guest_name): + """Return guest matching name. + + :param nova_client: Nova client to use when checking status + :type nova_client: Nova client + :returns: the matching guest + :rtype: Union[novaclient.Server, None] + """ + try: + return self.nova_client.servers.find(name=guest_name) + except novaclient.exceptions.NotFound: + return None + + def retrieve_guests(self): + """Return test guests. + + Note that it is up to the caller to have set the RESOURCE_PREFIX class + variable prior to calling this method. + + :param nova_client: Nova client to use when checking status + :type nova_client: Nova client + :returns: the matching guest + :rtype: Union[novaclient.Server, None] + """ + instance_1 = self.retrieve_guest( + '{}-ins-1'.format(self.RESOURCE_PREFIX)) + instance_2 = self.retrieve_guest( + '{}-ins-1'.format(self.RESOURCE_PREFIX)) + return instance_1, instance_2 def format_addr(addr): From 2472a7b5052156e03f3005e97e94f7a20a5a59a6 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 2 Jul 2020 13:47:55 +0200 Subject: [PATCH 578/898] ceph/fs: Use common helpers for launching test instances Fixes #311 --- zaza/openstack/charm_tests/ceph/fs/tests.py | 26 ++++----------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/fs/tests.py b/zaza/openstack/charm_tests/ceph/fs/tests.py index 4b0842b..28ac259 100644 --- a/zaza/openstack/charm_tests/ceph/fs/tests.py +++ b/zaza/openstack/charm_tests/ceph/fs/tests.py @@ -18,7 +18,6 @@ import logging from tenacity import Retrying, stop_after_attempt, wait_exponential import zaza.model as model -import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.neutron.tests as neutron_tests import zaza.openstack.charm_tests.nova.utils as nova_utils import zaza.openstack.charm_tests.test_utils as test_utils @@ -64,27 +63,10 @@ write_files: conf = model.run_on_leader( 'ceph-mon', 'cat /etc/ceph/ceph.conf')['Stdout'] # Spawn Servers - for attempt in Retrying( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10)): - with attempt: - instance_1 = guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX), - userdata=self.INSTANCE_USERDATA.format( - _indent(conf, 8), - _indent(keyring, 8))) - - for attempt in Retrying( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10)): - with attempt: - instance_2 = guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX), - userdata=self.INSTANCE_USERDATA.format( - _indent(conf, 8), - _indent(keyring, 8))) + instance_1, instance_2 = self.launch_guests( + userdata=self.INSTANCE_USERDATA.format( + _indent(conf, 8), + _indent(keyring, 8))) # Write a file on instance_1 def verify_setup(stdin, stdout, stderr): From 39b6065879710772ae94e00f8c18036e9d22009b Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 2 Jul 2020 16:10:39 -0700 Subject: [PATCH 579/898] Do not run API check on Ocata Since commit c98aa001f997d2cc149e0c64e81f0339a83adc50, there are some missing changes. This stops the API check on Ocata when gnocchi is in play. Gnocchi must be in play for Ocata but heretofore has not been. --- zaza/openstack/charm_tests/ceilometer/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceilometer/tests.py b/zaza/openstack/charm_tests/ceilometer/tests.py index eb50cb0..c460cdb 100644 --- a/zaza/openstack/charm_tests/ceilometer/tests.py +++ b/zaza/openstack/charm_tests/ceilometer/tests.py @@ -101,7 +101,7 @@ class CeilometerTest(test_utils.OpenStackBaseTest): def test_400_api_connection(self): """Simple api calls to check service is up and responding.""" - if self.current_release >= CeilometerTest.XENIAL_PIKE: + if self.current_release >= CeilometerTest.XENIAL_OCATA: logging.info('Skipping API checks as ceilometer api has been ' 'removed') return From 0df72bf47c88a69a20010255e60bd18a43e4d23c Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 3 Jul 2020 08:04:09 +0200 Subject: [PATCH 580/898] Make shared resource cleanup method class instance method At present the shared resource cleanup method is a class method. This will prohibit descendents of the class to influence whether cleanup should be run. Remove the @classmethod decorator so that it will make its descisions based on class instance variables set by the descendent. --- zaza/openstack/charm_tests/test_utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 4b4e1d0..6abb381 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -100,8 +100,7 @@ class BaseCharmTest(unittest.TestCase): run_resource_cleanup = False - @classmethod - def resource_cleanup(cls): + def resource_cleanup(self): """Cleanup any resources created during the test run. Override this method with a method which removes any resources @@ -111,12 +110,13 @@ class BaseCharmTest(unittest.TestCase): """ pass - @classmethod - def tearDown(cls): + # this must be a class instance method otherwise descentents will not be + # able to influence if cleanup should be run. + def tearDown(self): """Run teardown for test class.""" - if cls.run_resource_cleanup: + if self.run_resource_cleanup: logging.info('Running resource cleanup') - cls.resource_cleanup() + self.resource_cleanup() @classmethod def setUpClass(cls, application_name=None, model_alias=None): @@ -440,6 +440,7 @@ class OpenStackBaseTest(BaseCharmTest): cls.nova_client = ( openstack_utils.get_nova_session_client(cls.keystone_session)) + def launch_guest(self, guest_name, userdata=None): """Launch two guests to use in tests. From 29356f6419ef76dd182c63322bdba5edcfcefac9 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 3 Jul 2020 08:14:50 +0200 Subject: [PATCH 581/898] Move cleanup of launched instances to common class Use the nomenclature for resource cleanup as defined in the `BaseCharmTest`. --- zaza/openstack/charm_tests/neutron/tests.py | 20 +------------------- zaza/openstack/charm_tests/test_utils.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 2ddfda4..620767e 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -616,24 +616,6 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): super(NeutronNetworkingBase, cls).setUpClass() cls.neutron_client = ( openstack_utils.get_neutron_session_client(cls.keystone_session)) - # NOTE(fnordahl): in the event of a test failure we do not want to run - # tear down code as it will make debugging a problem virtually - # impossible. To alleviate each test method will set the - # `run_tearDown` instance variable at the end which will let us run - # tear down only when there were no failure. - cls.run_tearDown = False - - @classmethod - def tearDown(cls): - """Remove test resources.""" - if cls.run_tearDown: - logging.info('Running teardown') - for server in cls.nova_client.servers.list(): - if server.name.startswith(cls.RESOURCE_PREFIX): - openstack_utils.delete_resource( - cls.nova_client.servers, - server.id, - msg="server") @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(8)) @@ -803,7 +785,7 @@ class NeutronNetworkingTest(NeutronNetworkingBase): self.launch_guests() instance_1, instance_2 = self.retrieve_guests() self.check_connectivity(instance_1, instance_2) - self.run_tearDown = True + self.run_resource_cleanup = True class NeutronNetworkingVRRPTests(NeutronNetworkingBase): diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 6abb381..83595b0 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -440,6 +440,20 @@ class OpenStackBaseTest(BaseCharmTest): cls.nova_client = ( openstack_utils.get_nova_session_client(cls.keystone_session)) + def resource_cleanup(self): + """Remove test resources.""" + try: + logging.info('Removing instances launched by test ({}*)' + .format(self.RESOURCE_PREFIX)) + for server in self.nova_client.servers.list(): + if server.name.startswith(self.RESOURCE_PREFIX): + openstack_utils.delete_resource( + self.nova_client.servers, + server.id, + msg="server") + except AttributeError: + # Test did not define self.RESOURCE_PREFIX, ignore. + pass def launch_guest(self, guest_name, userdata=None): """Launch two guests to use in tests. From b4f155d5e32ff45c50500bbedd073ea0485db223 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 3 Jul 2020 08:28:38 +0200 Subject: [PATCH 582/898] octavia: Consume common helpers for test instance lifecycle --- zaza/openstack/charm_tests/octavia/setup.py | 19 ------- zaza/openstack/charm_tests/octavia/tests.py | 56 +++++++++++++-------- 2 files changed, 34 insertions(+), 41 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/setup.py b/zaza/openstack/charm_tests/octavia/setup.py index 55f8903..729fb16 100644 --- a/zaza/openstack/charm_tests/octavia/setup.py +++ b/zaza/openstack/charm_tests/octavia/setup.py @@ -98,25 +98,6 @@ def configure_octavia(): pass -def prepare_payload_instance(): - """Prepare a instance we can use as payload test.""" - session = openstack.get_overcloud_keystone_session() - keystone = openstack.get_keystone_session_client(session) - neutron = openstack.get_neutron_session_client(session) - project_id = openstack.get_project_id( - keystone, 'admin', domain_name='admin_domain') - openstack.add_neutron_secgroup_rules( - neutron, - project_id, - [{'protocol': 'tcp', - 'port_range_min': '80', - 'port_range_max': '80', - 'direction': 'ingress'}]) - zaza.openstack.configure.guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - userdata='#cloud-config\npackages:\n - apache2\n') - - def centralized_fip_network(): """Create network with centralized router for connecting lb and fips. diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index b189484..7049942 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -48,12 +48,13 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run class setup for running LBaaSv2 service tests.""" super(LBAASv2Test, cls).setUpClass() - - cls.keystone_session = openstack_utils.get_overcloud_keystone_session() + cls.keystone_client = openstack_utils.get_keystone_session_client( + cls.keystone_session) cls.neutron_client = openstack_utils.get_neutron_session_client( cls.keystone_session) cls.octavia_client = openstack_utils.get_octavia_session_client( cls.keystone_session) + cls.RESOURCE_PREFIX = 'zaza-octavia' # NOTE(fnordahl): in the event of a test failure we do not want to run # tear down code as it will make debugging a problem virtually @@ -63,28 +64,24 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): cls.run_tearDown = False # List of load balancers created by this test cls.loadbalancers = [] - # LIst of floating IPs created by this test + # List of floating IPs created by this test cls.fips = [] - @classmethod - def tearDown(cls): - """Remove resources created during test execution. - - Note that resources created in the configure step prior to executing - the test should not be touched here. - """ - if not cls.run_tearDown: - return - for lb in cls.loadbalancers: - cls.octavia_client.load_balancer_delete(lb['id'], cascade=True) + def resource_cleanup(self): + """Remove resources created during test execution.""" + for lb in self.loadbalancers: + self.octavia_client.load_balancer_delete(lb['id'], cascade=True) try: - cls.wait_for_lb_resource( - cls.octavia_client.load_balancer_show, lb['id'], + self.wait_for_lb_resource( + self.octavia_client.load_balancer_show, lb['id'], provisioning_status='DELETED') except osc_lib.exceptions.NotFound: pass - for fip in cls.fips: - cls.neutron_client.delete_floatingip(fip) + for fip in self.fips: + self.neutron_client.delete_floatingip(fip) + # we run the parent resource_cleanup last as it will remove instances + # referenced as members in the above cleaned up load balancers + super(LBAASv2Test, self).resource_cleanup() @staticmethod @tenacity.retry(retry=tenacity.retry_if_exception_type(AssertionError), @@ -238,12 +235,27 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): def test_create_loadbalancer(self): """Create load balancer.""" - nova_client = openstack_utils.get_nova_session_client( - self.keystone_session) + # Prepare payload instances + # First we allow communication to port 80 by adding a security group + # rule + project_id = openstack_utils.get_project_id( + self.keystone_client, 'admin', domain_name='admin_domain') + openstack_utils.add_neutron_secgroup_rules( + self.neutron_client, + project_id, + [{'protocol': 'tcp', + 'port_range_min': '80', + 'port_range_max': '80', + 'direction': 'ingress'}]) + + # Then we request two Ubuntu instances with the Apache web server + # installed + instance_1, instance_2 = self.launch_guests( + userdata='#cloud-config\npackages:\n - apache2\n') # Get IP of the prepared payload instances payload_ips = [] - for server in nova_client.servers.list(): + for server in (instance_1, instance_2): payload_ips.append(server.networks['private'][0]) self.assertTrue(len(payload_ips) > 0) @@ -274,4 +286,4 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): lb_fp['floating_ip_address'])) # If we get here, it means the tests passed - self.run_tearDown = True + self.run_resource_cleanup = True From 29849225c2a8b3b71fd456f4860ade23266d2c28 Mon Sep 17 00:00:00 2001 From: James Page Date: Mon, 22 Jun 2020 13:02:15 +0100 Subject: [PATCH 583/898] Pass deployment CA cert to requests Ensure that the CA cert for the deployment is passed to requests when retrieving stream data from swift. Refresh the product catalog entry on each attempt to retrieve stream data as the URL will update once data was been written to the swift endpoint in the deployment. Retry on KeyError and drop number of retry attempts Add a log message to detail URL being used for product-stream data Log return data from streams endpoint Fixup endpoint resolution --- .../glance_simplestreams_sync/tests.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py index adbc87b..b5947a3 100644 --- a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py @@ -42,7 +42,7 @@ def get_product_streams(url): # There is a race between the images being available in glance and any # metadata being written. Use tenacity to avoid this race. client = requests.session() - json_data = client.get(url).text + json_data = client.get(url, verify=openstack_utils.get_cacert()).text return json.loads(json_data) @@ -102,24 +102,31 @@ class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): if openstack_utils.get_os_release() <= xenial_pike: key = "publicURL" - catalog = self.keystone_client.service_catalog.get_endpoints() - ps_interface = catalog["product-streams"][0][key] - url = "{}/{}".format(ps_interface, uri) - # There is a race between the images being available in glance and the # metadata being written for each image. Use tenacity to avoid this # race and make the test idempotent. @tenacity.retry( - retry=tenacity.retry_if_exception_type(AssertionError), + retry=tenacity.retry_if_exception_type( + (AssertionError, KeyError) + ), wait=tenacity.wait_fixed(10), reraise=True, - stop=tenacity.stop_after_attempt(10)) - def _check_local_product_streams(url, expected_images): + stop=tenacity.stop_after_attempt(25)) + def _check_local_product_streams(expected_images): + # Refresh from catalog as URL may change if swift in use. + catalog = self.keystone_client.service_catalog.get_endpoints() + ps_interface = self.keystone_client.service_catalog.url_for( + service_type='product-streams', interface='publicURL' + ) + url = "{}/{}".format(ps_interface, uri) + logging.info('Retrieving product stream information' + ' from {}'.format(url)) product_streams = get_product_streams(url) + logging.debug(product_streams) images = product_streams["products"] for image in expected_images: self.assertIn(image, images) - _check_local_product_streams(url, expected_images) + _check_local_product_streams(expected_images) logging.debug("Local product stream successful") From ecb47bfa4fa223b811511be94e748a5728a19cd9 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 3 Jul 2020 10:59:39 +0100 Subject: [PATCH 584/898] Fix lint --- .../openstack/charm_tests/glance_simplestreams_sync/tests.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py index b5947a3..0c0d472 100644 --- a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py @@ -97,10 +97,6 @@ class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): 'com.ubuntu.cloud:server:20.04:amd64', ] uri = "streams/v1/auto.sync.json" - key = "url" - xenial_pike = openstack_utils.get_os_release('xenial_pike') - if openstack_utils.get_os_release() <= xenial_pike: - key = "publicURL" # There is a race between the images being available in glance and the # metadata being written for each image. Use tenacity to avoid this @@ -113,7 +109,6 @@ class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): stop=tenacity.stop_after_attempt(25)) def _check_local_product_streams(expected_images): # Refresh from catalog as URL may change if swift in use. - catalog = self.keystone_client.service_catalog.get_endpoints() ps_interface = self.keystone_client.service_catalog.url_for( service_type='product-streams', interface='publicURL' ) From 0fe56cbc33e5232a0e9b76c8c8089f51fcc9380b Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 3 Jul 2020 10:48:12 +0000 Subject: [PATCH 585/898] Switch pre_deploy_certs from OS_ to TEST_ vars --- zaza/openstack/configure/pre_deploy_certs.py | 34 ++++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/zaza/openstack/configure/pre_deploy_certs.py b/zaza/openstack/configure/pre_deploy_certs.py index 11ec709..34af066 100644 --- a/zaza/openstack/configure/pre_deploy_certs.py +++ b/zaza/openstack/configure/pre_deploy_certs.py @@ -14,19 +14,19 @@ def set_cidr_certs(): """Create certs and keys for deploy using IP SANS from CIDR. Create a certificate authority certificate and key. The CA cert and key - are then base 64 encoded and assigned to the OS_TEST_CAKEY and - OS_TEST_CACERT environment variables. + are then base 64 encoded and assigned to the TEST_CAKEY and + TEST_CACERT environment variables. Using the CA key a second certificate and key are generated. The new certificate has a SAN entry for the first 2^11 IPs in the CIDR. - The cert and key are then base 64 encoded and assigned to the OS_TEST_KEY - and OS_TEST_CERT environment variables. + The cert and key are then base 64 encoded and assigned to the TEST_KEY + and TEST_CERT environment variables. """ (cakey, cacert) = zaza.openstack.utilities.cert.generate_cert( ISSUER_NAME, generate_ca=True) - os.environ['OS_TEST_CAKEY'] = base64.b64encode(cakey).decode() - os.environ['OS_TEST_CACERT'] = base64.b64encode(cacert).decode() + os.environ['TEST_CAKEY'] = base64.b64encode(cakey).decode() + os.environ['TEST_CACERT'] = base64.b64encode(cacert).decode() # We need to restrain the number of SubjectAlternativeNames we attempt to # put # in the certificate. There is a hard limit for what length the sum # of all extensions in the certificate can have. @@ -34,37 +34,37 @@ def set_cidr_certs(): # - 2^11 ought to be enough for anybody alt_names = [] for addr in itertools.islice( - ipaddress.IPv4Network(os.environ.get('OS_CIDR_EXT')), 2**11): + ipaddress.IPv4Network(os.environ.get('TEST_CIDR_EXT')), 2**11): alt_names.append(str(addr)) (key, cert) = zaza.openstack.utilities.cert.generate_cert( '*.serverstack', alternative_names=alt_names, issuer_name=ISSUER_NAME, signing_key=cakey) - os.environ['OS_TEST_KEY'] = base64.b64encode(key).decode() - os.environ['OS_TEST_CERT'] = base64.b64encode(cert).decode() + os.environ['TEST_KEY'] = base64.b64encode(key).decode() + os.environ['TEST_CERT'] = base64.b64encode(cert).decode() def set_certs_per_vips(): """Create certs and keys for deploy using VIPS. Create a certificate authority certificate and key. The CA cert and key - are then base 64 encoded and assigned to the OS_TEST_CAKEY and - OS_TEST_CACERT environment variables. + are then base 64 encoded and assigned to the TEST_CAKEY and + TEST_CACERT environment variables. Using the CA key a certificate and key is generated for each VIP specified - via environment variables. eg if OS_VIP06=172.20.0.107 is set in the + via environment variables. eg if TEST_VIP06=172.20.0.107 is set in the environment then a cert with a SAN entry for 172.20.0.107 is generated. - The cert and key are then base 64 encoded and assigned to the OS_VIP06_KEY - and OS_VIP06_CERT environment variables. + The cert and key are then base 64 encoded and assigned to the + TEST_VIP06_KEY and TEST_VIP06_CERT environment variables. """ (cakey, cacert) = zaza.openstack.utilities.cert.generate_cert( ISSUER_NAME, generate_ca=True) - os.environ['OS_TEST_CAKEY'] = base64.b64encode(cakey).decode() - os.environ['OS_TEST_CACERT'] = base64.b64encode(cacert).decode() + os.environ['TEST_CAKEY'] = base64.b64encode(cakey).decode() + os.environ['TEST_CACERT'] = base64.b64encode(cacert).decode() for vip_name, vip_ip in os.environ.items(): - if vip_name.startswith('OS_VIP'): + if vip_name.startswith('TEST_VIP'): (key, cert) = zaza.openstack.utilities.cert.generate_cert( '*.serverstack', alternative_names=[vip_ip], From 7fc0aa8bd96eb062ec49bb8909ddd2fa928e1a95 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 3 Jul 2020 15:26:52 +0200 Subject: [PATCH 586/898] Remove some deprecation warnings --- unit_tests/utilities/test_zaza_utilities_openstack.py | 4 ++-- zaza/openstack/utilities/generic.py | 2 +- zaza/openstack/utilities/openstack.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index a6aa190..7271708 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -809,12 +809,12 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): name='_get_os_version' ) self.patch( - 'zaza.openstack.utilities.juju.get_machines_for_application', + 'zaza.utilities.juju.get_machines_for_application', new_callable=mock.MagicMock(), name='_get_machines' ) self.patch( - 'zaza.openstack.utilities.juju.get_machine_series', + 'zaza.utilities.juju.get_machine_series', new_callable=mock.MagicMock(), name='_get_machine_series' ) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 0874cc6..dd8b1e0 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -23,9 +23,9 @@ import telnetlib import yaml from zaza import model -from zaza.openstack.utilities import juju as juju_utils from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE +from zaza.utilities import juju as juju_utils def assertActionRanOK(action): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 28ba29a..227d041 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -40,6 +40,7 @@ from keystoneauth1.identity import ( ) import zaza.openstack.utilities.cert as cert import zaza.utilities.deployment_env as deployment_env +import zaza.utilities.juju as juju_utils from novaclient import client as novaclient_client from neutronclient.v2_0 import client as neutronclient from neutronclient.common import exceptions as neutronexceptions @@ -66,7 +67,6 @@ from zaza import model from zaza.openstack.utilities import ( exceptions, generic as generic_utils, - juju as juju_utils, ) CIRROS_RELEASE_URL = 'http://download.cirros-cloud.net/version/released' From 16cee0193fa2dff1834bf8838f34a63add5fddca Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 3 Jul 2020 15:43:00 +0200 Subject: [PATCH 587/898] Make linter happy --- zaza/openstack/charm_tests/neutron/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 4b1ff1b..c8c302a 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -23,7 +23,6 @@ import copy import logging import tenacity -import unittest import zaza import zaza.openstack.charm_tests.nova.utils as nova_utils From 37dfa53bafec129d167331dc839e083a0fdd8d66 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 3 Jul 2020 14:44:08 +0200 Subject: [PATCH 588/898] ensure that we add the bare cinder endpoint When there is a cinderv2 or cinderv3 endpoint, we should enable the bare cinder bits in the tempest config. --- zaza/openstack/charm_tests/tempest/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 1cdaf1e..af1ee4c 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -235,6 +235,9 @@ def get_tempest_context(): 'cinder': add_cinder_config, 'keystone': add_keystone_config} ctxt['enabled_services'] = get_service_list(keystone_session) + if set(['cinderv2', 'cinderv3']) \ + .intersection(set(ctxt['enabled_services'])): + ctxt['enabled_services'].append('cinder') ctxt['disabled_services'] = list( set(TEMPEST_SVC_LIST) - set(ctxt['enabled_services'])) add_application_ips(ctxt) From 849e08528b4e57a23f4c342c501f6927a5b98ebe Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 3 Jul 2020 10:27:24 +0200 Subject: [PATCH 589/898] octavia: fix pause/resume test and temporarily disable it Re-enable when LP: #1886202 is fixed. --- zaza/openstack/charm_tests/octavia/tests.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 7049942..e84ac3e 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -38,7 +38,20 @@ class CharmOperationTest(test_utils.OpenStackBaseTest): Pause service and check services are stopped, then resume and check they are started. """ - self.pause_resume(['apache2']) + services = [ + 'apache2', + 'octavia-health-manager', + 'octavia-housekeeping', + 'octavia-worker', + ] + if openstack_utils.ovn_present(): + services.append('octavia-driver-agent') + logging.info('Skipping pause resume test LP: #1886202...') + return + logging.info('Testing pause resume (services="{}")' + .format(services)) + with self.pause_resume(services, pgrep_full=True): + pass class LBAASv2Test(test_utils.OpenStackBaseTest): From d0981d64af57b062376de6676bc4f846ba74f5d8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 7 Jul 2020 08:57:40 +0000 Subject: [PATCH 590/898] Add auto_initialize_no_validation_no_wait When vault is in its own model with no clients then vault needs to be initialised without waiting for clients to start executing and without validating a client has recieved the cert. To achieve this, this PR adds auto_initialize_no_validation_no_wait. --- zaza/openstack/charm_tests/vault/setup.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 0709162..4db90b5 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -16,6 +16,7 @@ import base64 import functools +import logging import requests import tempfile @@ -99,7 +100,7 @@ async def async_mojo_unseal_by_unit(): unit_name, './hooks/update-status') -def auto_initialize(cacert=None, validation_application='keystone'): +def auto_initialize(cacert=None, validation_application='keystone', wait=True): """Auto initialize vault for testing. Generate a csr and uploading a signed certificate. @@ -114,6 +115,7 @@ def auto_initialize(cacert=None, validation_application='keystone'): :returns: None :rtype: None """ + logging.info('Running auto_initialize') basic_setup(cacert=cacert, unseal_and_authorize=True) action = vault_utils.run_get_csr() @@ -131,10 +133,11 @@ def auto_initialize(cacert=None, validation_application='keystone'): root_ca=cacertificate, allowed_domains='openstack.local') - zaza.model.wait_for_agent_status() - test_config = lifecycle_utils.get_charm_config(fatal=False) - zaza.model.wait_for_application_states( - states=test_config.get('target_deploy_status', {})) + if wait: + zaza.model.wait_for_agent_status() + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get('target_deploy_status', {})) if validation_application: validate_ca(cacertificate, application=validation_application) @@ -163,6 +166,12 @@ auto_initialize_no_validation = functools.partial( validation_application=None) +auto_initialize_no_validation_no_wait = functools.partial( + auto_initialize, + validation_application=None, + wait=False) + + def validate_ca(cacertificate, application="keystone", port=5000): """Validate Certificate Authority against application. From 8f44ab681ae6ed44939038726c914d77e0d4d4f4 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 7 Jul 2020 09:03:36 +0000 Subject: [PATCH 591/898] Add wait_for_cacert wait_for_cacert will wait for keystone to recieve and install a cacert. This is particularly useful when the certificate issuer is in a different model. --- zaza/openstack/charm_tests/keystone/setup.py | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index 6dbb7c1..748439f 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -14,8 +14,12 @@ """Code for setting up keystone.""" +import logging + import keystoneauth1 +import zaza.charm_lifecycle.utils as lifecycle_utils +import zaza.model import zaza.openstack.utilities.openstack as openstack_utils from zaza.openstack.charm_tests.keystone import ( BaseKeystoneTest, @@ -30,6 +34,25 @@ from zaza.openstack.charm_tests.keystone import ( ) +def wait_for_cacert(model_name=None): + """Wait for keystone to install a cacert. + + :param model_name: Name of model to query. + :type model_name: str + """ + logging.info("Waiting for cacert") + zaza.model.block_until_file_has_contents( + 'keystone', + openstack_utils.KEYSTONE_REMOTE_CACERT, + 'CERTIFICATE', + model_name=model_name) + zaza.model.block_until_all_units_idle(model_name=model_name) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get('target_deploy_status', {}), + model_name=model_name) + + def add_demo_user(): """Add a demo user to the current deployment.""" def _v2(): From 9c52b3390d4c6d391b002fba15ff3ce5c3324d83 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 7 Jul 2020 09:06:35 +0000 Subject: [PATCH 592/898] Add LTSGuestCreateVolumeBackedTest Add a test class which launches a volume backed instance. --- zaza/openstack/charm_tests/nova/tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 1b12240..c4dd5b4 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -56,6 +56,16 @@ class LTSGuestCreateTest(BaseGuestCreateTest): glance_setup.LTS_IMAGE_NAME) +class LTSGuestCreateVolumeBackedTest(BaseGuestCreateTest): + """Tests to launch a LTS image.""" + + def test_launch_small_instance(self): + """Launch a Bionic instance and test connectivity.""" + zaza.openstack.configure.guest.launch_instance( + glance_setup.LTS_IMAGE_NAME, + use_boot_volume=True) + + class NovaCompute(test_utils.OpenStackBaseTest): """Run nova-compute specific tests.""" From a829b372e0d6c5bb3e759a40b84e83ac56e409ee Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 7 Jul 2020 09:56:25 +0000 Subject: [PATCH 593/898] Use model tmpdor for key storage Use the model specific tmp dir to store the test ssh private key. This avoids the key being overwritten in CMR tests. Depends-On: https://github.com/openstack-charmers/zaza/pull/371 --- zaza/openstack/utilities/openstack.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 28ba29a..034cae3 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2202,7 +2202,8 @@ def get_private_key_file(keypair_name): :returns: Path to file containing key :rtype: str """ - return 'tests/id_rsa_{}'.format(keypair_name) + tmp_dir = deployment_env.get_tmpdir() + return '{}/id_rsa_{}'.format(tmp_dir, keypair_name) def write_private_key(keypair_name, key): From 76fdb61e0f1b364713a4ecbc474fd3f8b900deb9 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 7 Jul 2020 12:18:01 +0000 Subject: [PATCH 594/898] Fix up unit tests --- .../.test_zaza_utilities_openstack.py.swp | Bin 0 -> 73728 bytes .../utilities/test_zaza_utilities_openstack.py | 10 ++++++++-- zaza/openstack/utilities/.openstack.py.swp | Bin 0 -> 16384 bytes 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 unit_tests/utilities/.test_zaza_utilities_openstack.py.swp create mode 100644 zaza/openstack/utilities/.openstack.py.swp diff --git a/unit_tests/utilities/.test_zaza_utilities_openstack.py.swp b/unit_tests/utilities/.test_zaza_utilities_openstack.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..2ef70f74efe45a7a7db302b8b0e7266b3d9ec4d8 GIT binary patch literal 73728 zcmeI537lkAb^nXSEi6h%-2M`Znqc*q?&@u(XOY%4LjwcM40bmlv}0La^{VGN)m2R` z-P4TxT_U2mCmQ3fQImf#q7qcZC1TvNs2Ig1sA&G8Ni+!x<{u51|M%Rt-23i(we-vY zLF@JB*H!i2UCurC-23i1_uP9IMlRmDE4?*0lHz()D)p7cWAnpDJuCIO*;J~v(5g0D z@}TpzSZ&qTh87Bqrdz9ywGaEhy#$9>YxVuZ3-v;^xX)<}Us1TCFjQN1s*Pr$xPPeW zG@6a!R@H6h{jW{8;x=7E%I~}c&$ohm`7mN`d98iU{U?16{EBm+)hrKfJs~wdHmaKX zhRkW{lYjl9UvZ}Ni{Ix!p96gk^f}PyK%WDB4)i(D=fJNP2b#-|NfaB_ce(ZaVgB=Z|MN4}^M_l{PdmoD5#;{}|9O!9bFAmT z=0Cs0|NN5m{5b!9z@GuXKtcM)`_G@|e?G^0?kAUewEz6`)^q=M|D(@=J_q_7=yRaY zfj$TN9O!eP&w)M%`W)zU;FpvGg;FXdI^>(>A4>SM{BnK=h5g&$P2f^642HlxDEdDK zH-KeuI=CIX*PR;H}^-;3eQvunC+5{t1Qm zv*0b@#b5z!21kQ0pe$bxo(G-^P5?KfU|$3FgK=;-3i)Ti>%e7TJNPL|{N3O-@Lq5b zJQX|~+>8O>`QUk=493A2csw{7+=+qVI74+#(fWHK<22TSgfk%N) zp?n_zt6&RwG`JP}!3V$%pb7>+7TiHSsJf7PNc&gAsg!e7r`4?2s>O=yRGYbqi!a2& z;&Q&R>=gDp^?a?GFVzc+@~rMOTlH#wrBG=(>B;mnx~8L|HSBmQSMO0rQi^q_&~)-u zrzwsdiK+O{x)S1tOC}|FsEHBwcvO~3b*ItDH;VX!Z0wYcPi(=AV3Rx$CXk##qhk)L zW+LgwiZh^#kWjTgiRDVI&~&Sd?y^mvilZNg)Fz~*h|xh~lvqhMdLeANR*%V;N=F`a zB}M`zlcAn;h>s{Q)r$Lby9$eLaTnhPHfnOErsTSE@0BU_#rl_ZOWEz|?8uhF$d+we zOGCx6ZN;HYg>7R)Tg%0Tq2j1h+&r;ueAAY(ZP~$C3I`*1GDf>=WVWa0!>5W3=2K`s zxDFy0+-jk|CWQK}eTc+{{C1r{mlV=3*_QHnpHVLJC&NyyhM+lXpK}noQ6rC5kV9svDROo@DkKHDZQn#8Z>vM6b+py|&_( zoceY@qkMC1*&(w`xl(9mu1v0MoraFTyl<`H77LZ2pOQ4>9_fx|;x(?5kZ+lKt<{7G z=K%k!g$j}9>Mf^hw5C%nRGWFXLah_+QT@^Wc^%%K59}N|+(@1EqW}_Ume`PwbrhBcr%lQ?j-oV`udN=8! z+PQeYGmU+Px+8VaQ>B@VkCZDfPokMwe9<)#jToM=7|`6V)EQH56t@z@MeZAAQ^DfG zA-5CCQ+HSVfff(x*}|We6!^V`1oFRH0q9iyUdT(x`y~}(1q{k8!GiBrkg%jGDG?II z2Q8PaWv4zchwJVhe0*oc$M+(GMXWN)-IL(Wed6SE*m7Fqn<+n)c#_g$8u~KL> z((%{>Ed)rC$(>nfIP!c4pAsWguGf~*;w7BZzQLitaN1o`%ykYFon>)XZuAyDOe$Sy zq{AN`2`N8K?X#Ryq)E_#Oq3umW9jyaZgVa8-DoXP8$|}M!5tan1mE3S@U2A6H{B&C z_{~5Uf9DP9io4`C87ESbbgRmXGTq$gq%Uxb_)|MM0?ay&c`IG6c|p({if-90rYnW& zVvF~Cda*{IT4ltUrpA`shIFrubfH=zl}mQcoxA6vx%4GdGc!}W=XOrdruWRGckJ1H z*3P+|dv^2d?DW*`OVj7?+70`JpOoZ__G zhyCxxD4lv!@r<(uHMmC>9nf4kcQZJPV6; zhiA=N+O4Kn>-dmY7m>PJxw*>gAhm&X<#hNe%Pk1N23e(I)gjaq&nv9xKGzL!w(-(X1!8G)x2c+&Zex%ICCo6(?KwsTm z+}BKxjf`$h?N&H=}QucG6>1vJ1gxSO;;;pg#3l28BYbD+P%dtrWa@ufcR=8ca{bSUP#GSgV%Z#a7+PFQPfG7S{4&3K893&+aI5 z*dnH=Ac$B%8o37c=DbrbW2eQYF`q3nzcwU~vU^hjRU6X2VHyQc9*LTqMt3akM};di z)2mLpRI6s2>3YFM^_yWKHOl*RyKMyP2FnB-tH z2?*(lA9-O%-dS31uH~6+bn1mBWeU@fwANgwQi|M>yWXs1-vMt0SA!RU z0yqOG+ReVB|Gm$FJ_q_7=yRaYfj$TN9B9vh$hfbE;nQ+!ftjKFerL__rIE2^UPhit zLrcj)Mzw15B3$y^lEg+EVH&Iv&T|HI$Y*wGjcg$TO>$QV>%EFbcM_vAb4v>_M}~Cx5aWQU*fqMA<8p>dW+@oK)j@9lGmS zJ6CJu5T<>D>5SWu89(0}Ct4;=QeML3)s$;>F;OOjFyRn!D1s#z@-R_2bPl@ES|lWz z!97bX482eSX4}oE8?)>}Lv}E|G??~W7?rk`lGt)7T~z63s-0;8Tngku9;yw!2Jk2? z<(cv>D6wc;XfI*;AE_>-B&LFgN4&OT6|@iKzvAR%irL9|Z7k0OD=HpdxkV>0B_GIA zZx|(*|8LSTL+ZS^!U5k0LdaffpUQA^Y1!K{ZK+mrDx<^nC-ny1kJsX9`>RIkl9(+S zicFML&wIpd5JXNU@<_-+R2N>4Flc6}s+9f!CY0Q|-9Ke*=44aCfaEJ8G3##NC1<%( zTU(+s(dJitz1*Y9e=&q@Jxiq%Yz&~}}wK2q1H>Q9oIe7w1f{=WhL z_F;VBMgI@h&wmSD|7IXQ0Q7Sunn95jtBpY?*A$9O0Wnn1lz%n(f#iPp8{_MC2$`2EpQC@C-ncX zfY*aR0wwTNa4h&HHjPh%_k!1e8Sn`3aBw^2eI0Otl-&Wc79ay20X_*&J_MG5tOGa& zd>23e?|_ej=YkoK1`h)_8Mc%lvy|*rrKkU8=2Cb>`i(RFOO*o~?#P zHC>_4qs``!FikT+h3ibH1W9CG8>v!!)ZLpRxs#1EC-p6+DC}0eEBxBeXl8azfr!a< zxXwhPwXvSa#;De-9V4>AyI#9#PZXDF0C^e$Q}VU?TCBOKs!M1BtM0-=Wi2LeZL3kS zH<^Fk1yT_a$x+vvdX_9(qW_}~rw;h*|AYPMzKD+hYLExN1%8Z<|Ht4`@Obd|==EO$ zqVGQ!oC!_>Uq+Yz6X1ZYK-TfU6+8>%!5DZr_zt@Jd%?>=15AQb!9&5nqPzbHd=R`0 zh)yqS`cDG?4`Ocy?*P|;E5I7K1Uw%60Nwp#-~c!e>;$6I|10|Y-+@^Fhm z11|uwrvD7^U+C+f2G0ZMfz!cngRi2m-vHhYUJRPxJa97jIr{o7K-TjA1^5H-Y_JjB z1z)cN!rxWlv42UGBwrE3RJKuDa&TK>(!qR-cWsE6pX-GscYlpx_f)G{n<|wq^b(2o zI8hQJSqCYoj#|soy0V^~J$HxNufXW&BI$N*hcZZGk<1b%*2H$l!WtIj$^bOp0HxWo zB|&L2i>6nptg#BTB=@*w&?79!j*jHyZ*;_X5Fu0?%Br;g-Wp=3oHZ*|&P!`6OGc(` zsZkH2uME4}7hcJ0S9kf*A~*+}V!%xMr8SUkG)u0S@A+i0GjhWhr(Wl+m6E*W@*YuV zZY?*oO zVPS5Env1y5TV|+y5G{kgSy?@OFD}ujme%+%1~sZ@sJE(?17$}-nCX$dN{Y;7Ol}&} zeN*>dMkkA?lB+?86d_n=nOiMzk_=1U9AnJdk;~FD>T4<(Z6yDIaC$!FIC0UwYcBm--N7dI$!mqk>+RN0MIGHMS*y5IO-XEy9 zA1QY%d9r*Ra#bWVEsoicdf^4rMEqToCU zvZ|9+JSw=|v+5l-%n0pXe3EUL z8G#N88$7CITM&MX`97y1d)t)~sn0KoH)%4}9)&YD5To2v=WG}#3 z@C0xS_y)EB@dx-Q_hTT*Z@8NUIktWrogG-cyJeX0GapyIJg>A!A0NTw~&0ro}22KOFVh^|$ybA0Fj|D%&4)CAg zBj9=94DbYS3p^LT2lM=aZ6Nj;NT_^<1KUQkwwy1N*^nw}6YxT*+~k z<%@cqMt4B;iW+u{oysS6>!6IW7qb*jb#Z0^H6P`%Tq;PUob)PZPn|!VpE_rH_gsGG zS@ux%P->vpf$9A$1U+%NBZoaEcKe@2Pa9HZTO6w_g{~btmE`7qQ67kjksdg-QIhpG zK)KzX3{S}pn!9B4o9d#aB_Af!BZRhVYEk+)jXaq-|R~^mY`X7EPkr zzTtRu#G6dF+NM+S8&qVMLrdf9t=bryd<7^xlTDEt%s>4>3QL3Yv#%KS%hL z7R)s2DKE(-Hh}P(vp*lkSgug2xK%b**f>A3Hz(U0vZJLgs|+;)4h-a(8*r<7noWK) zHDGp&_e*Q^XYEA)mt8YIY3ctTLC;?U=Yn&1JQjQ# zJ^x$adQbxwfMdYd(eb|q{tT4C4sa~^273M1!FAvb-~w8}d{f~gxg4cjWa2k;P{=WiV4CL&= zqktT`D15(Jcz?*3NT^&&O2yFT?}RCPf->7H+n}(hN`Bch6PfXh7W9~q+wU4lT8y$y zjI8A020JhhY#dCFX<=VK8Dh0fsUn!AD7NZ#Szc4a3hZrh8CQWa0X=M~o2j@97?0O7 zS6XH2G~n|bo!WEEnATS1>LINJy8To_jiW6mB@eSBFzQ5^JnQ(+vdyi`;V)~Bn|FDk z9B`^G+nitCa@fE${gmLhY!kb`5c1g%<=rJ#QfL78`5VMN^Q8Q zNf6pSCnCE#bxJCXv#K0mW&0j)F5htTwnipBtd|whF}e%0AjUXHRl4eUJnLYj8_^v1 zW5>pZM@nYe5Nb@F8jZw~s+2HKE=5ehW4~LsTg*Z6tygauneg^HC}W$Ym?0^cj`fy>;%Hz1Vv{ zC$!C;>RWqM=-^9i#Lb9W6du=!z4(3c(v4GuBE4re+zi_@#N1yTm{mk}EPm(m#797? zjrfsb#h=HmL$OD@{woG}%%ZmwmnyUHF}~WH+(ihuFNI~w+ofGbqrLDeUUaL-J|hy! z9Po%&f^x$$dTQM}=r${bHeLbI`qF}?f*pysz_VM=ZmFFqteo^)L{*y6?@QL*=xy8& zl;TWPW6DuiWQTWEN1FA>{yv~O?Clhjjh9!TOmv_+@`eqgvbjY=POk{Y#A>(~d)-Qj z<%&+^#ODfADwO{(B-TIEqSIt%IL#KO(7lWH=eEu~=fYXU*RO&zz>|TT zHSjR-ee`=-?=N%yF9f^6CLn7AZbHYegNKqF8|ly9pFkJy1ks` zC+h%S4K4#u0Z#qmj`_xZx(*yYQBu|p+iqKKK#+lR*< z9(m8COA7@+-@^7(Xtw11rR>y>xt$kJ>y~8%YAwrYBC`KvmLnmy4ULQsjcl149pAob zbo=J5mnFqX>P6#(cM2t;4{-)nx$sJ-P8$E!p3XI@=Bd&Awj-S zY`QB>+qRqK0Fa@5oHJHmaak;2XsJPw+u8HQfyv*fHWh@GPrHHJw1p+o4gfv0YBD>g zg}?u=4Q66WOXGcak7om`|Xi!)5BcFK_0=D3zz{J0;a6m%i1p4_k+@wQa;Y zt4?zCV3oGKG`4(^eWezYd$`WMTT;)aS)oU#htQ*KNbi)xjyU~_ld-4->A}6-vO_l< zIsSvRGc3DO&st=ZM_b`lQjkfK1HN*9o!642pHsw6dA zw#8Bsq}kizk${$5q-<6%IPX2w8m;#`$8PeI0oAVFN8Nkcsmt<0z+?htydP@c`b;X| zksm_JFJYF}yc;?|NVx1_-c)nchA{{uy&O{JMbcVy7|DAOtx0+RdFzpkgu)o>5+bou zMC)DGO7wo08Vs`6BH&3x$S0+H$6UsQi$PRN{Ozjs3Db4!-^k_NYPkmQ4+h6Y21W3M z)jP*ExaXFx3`z3ub*ld$nyy~|j;`Mzh$kcdFH1;*B6iA3F-Y1)V5>D}1Z|aNEG&e! zX|&zpUUnWxOx2i+Q`RnD3Z+uQF~8MP?Y=8ktx0l{R3e$<9M>Wimfo;}be?LB7}&z- zp>3hpIZkg9EH=eyO6od`B>F$#3AbP*}`Tsc(-G3O|iJmWe z`+XF=3Y5S$uoWB!?m*Z71b72j2AjYm!M~vEe*(M^Q zvfwU?=|C-#3Xayvt}Mx`!(7Oq3SxFIDM?+f!x@sVLyNnwO$L!yMAz$b+#c1$C9%E29_n_})-(r+jORopra0aCoRZAt(s37D z+%wiBwonk(KuQIRgw(cIjyP5){1}$hp!glT89&fiuLN(3XE2uQmEbYM(Bf66;WrVK zf26Z?jzBF`tY3}rM zx1YXi`{}cnWeq>jNKni2KB6q|THJ%M!MWeL@C)vxc=Ihe+Eg5bO|sp{GhOUx`gWtX zjR0|rc`C?i9oFd?UNePy^8v9<2R7;4gMG>)Ru$6i9s1Nq&!%aq@57L4Z$2q1b&_N# z%;A^5RBN-e(b2^%m3@&G4&-AEA`t2PlH)WB{`2-ze7*J5_WWp4FCB_-_$67ElX~Z* zp7}6(Pz}59s47d`(Q;Qc_({gb)> zM+4FQuLH6N;5?A||D(Yd(ewWlTn=V|?By@}{=FZ(9;^Y;`%eQWgRi6OzX!-3fU`i> z^?x5d|3lyA52I04*^o_`nkFmS;DxC7n(ec+{FKbQs^z%S78WqrS_?YH;-6BbJ^ z7+h-i2WI3M_Ls1`%JOBc7vKNSg|WnHp_=_#z^1!b!&4zi3UJ3KySrz~~^ zv2!%zgO1DqjF^i>FrAjcEjyS)hwr9x*>gb#&ps%D0CkYIIV?O09*n0c>55X&@)s zpyvqWFHeXUu@|F($B;1Y%2;A&2$`}#D{3}U1!dm_c2&G^X3u%kJLarPmH^qQ3wP!( zo}QWAxo0<3G=63LHIE}e-cKNZ)}CEcJ9jG+l2oX#Fy=igZc_45vy&00(I#||lUSE} z2rh=GL!#nkw;lzOm|- z@wG@v4VjlzMPOBvp#E;Kem670Kn<(f@Cx>Ulo`!xJ4uZruzGltJjP)Hk%ZJ~McyP_ zT-G>J7q4Zvxi;jBg++ySvW`-hRiz!G&pKe9$JVKDXlF_diM3z9Nkxt}Ew;`Sru$?> z%^tBv66PaZ?$AEGGkz0WhZ!r7brsy^8daVoUSbo!B%oI-Hm6xRvZqec3pPn#M;&? zF14%6Z0vqzZQ54eR+`vaTG;Ar-85b<7q*U;Mz)P?+cL7H=!{Jij3mP56%IPo?iu|t znXF?`GkVsGLl|aH>m*H!I=Ev!eeGp^v^?=A@bbf0`(#Twg=qe>b=q z$Qb~e!Li`$==pN)Ulp7KMEAcNJ^!!4yFmf`Hh2X1F}l92`M&{({=XmOfEXNZMc=;` z90Zqwjo^47Hh`aizXN{=E&(Tl&!h8S1 zK-U-je*x?UvabKL@KX5sGU4fZE*WmfrR1%%l9{coR8iLP(u(Yq)=SLNN}WS1y$LF- zPwA2;r(SXfhO8an+#)s}tg_A=%K|g(`+1KVe)1de@Jz2d>BXA%;oP7+JJs%P=>jgG zl}dVNb}5a`gShyC;`-UF;kwOK9e!Tv1#uItmPjQ_NJ24TTG#soY)GHI^Xa>$w|fzN zx6y)HJuYWVrRAmLCC0)50y*h&tBTt{E~k(t8I*kAx^T+J+bf|}ck+uAakU_aG~@Wk zkSV4)D>i$oMDi}WM(vn8fv9+*Q5cRx)^_Rh=R0d^kM_ZI@J&yd8u=#ffqF!;75(gf z$x!sLdZ51L7JG+ag%RMPx!zLc7!> zmmZ0=-8>aJx~)Wsm0wCiJ2JqQv`+PJng&ynd?@MBlG$}FK|@)WVc-fD3pR$g^OjUP z5*yT3ds|PuCX$AowUtIt3COfmru8D~PKi|&Dv^{FH2J!JDJ1|rEW{E~N$rc_pms<``BD7*7>(QJR1P}Zz#a6T9 z{Bs%?FODr63VF~JB;|G3i4AYh4#uZ|hW0kN@d@9mJeISqP1?k!WL27b;NKbO%H8`K zp`dS9mdTef%M97OB-w`9X_vi-4%GH7?<@H2G&Tfv`!MX(wC5MBRYz+ZtEgQtND!4@F< z{@)0iU>wNVfWHYoM0sDdY06Yxbg1-Mn@Ii0@hC z_+K){s9wnO0~zNF_!0fJOIWEjxqceh@$5mTPql5e&_0xYR4KkeOK!QgJRobbVtGhF z@<}|UK$S?xkYWVIYhX4tw`-3V&)f6eCnvW5jec)#NS{@6yw$R6PBX3SDwh28QgMmP z^ByF1wjn)Lk-b;fywW$U;>(}{J}O1Sp5hB+Pd!XY4aueM_R#8rRPCnLPRb-*+xFo5 z`s5qZB71Fhkw}No_9F59h7`FspNafDgw~jps+#xL4I+|7`^Fjxrdw*{w%1-Gw=#PZ zT5Tjm`?ebirU@3=h%%dpE-j=*WIm>l^bK{e2w`-d({;E*&hZsIiexdYvcV z<}>o!Oi8ETzO@Ho^6Z_Rk_IQ2#ib~v3C%o#d14662Z_7O@iu6b-fq>0`i|Av1vS-M zlhpB!OeM+u#eG{r|5F=az)6uN5eoC|n;;~93lq5wYAn{aR5S^p-wW!NIkuI@e$~m> zcuNR{Ix;phqNrD8@S77|{5GcZlyuv-B6(^xACr)EG#QDO&{))MYY=;=%|y})*xg$L zkr)AU`1(Ne_kXJN3!7qMABvuN&{<0K|3P%pH=&=({C}YT-++$4A6x>C2cJa0m%aa1 zfb98yH25I;{Q)or{tbQpR`5!&0~`%Li!T3iPy`F$N#JMb^s>L-)nEba1mfp^Joqbg z`fI@=I14-pJQn;K_zAlGmw~MJe+QTYli*?C)9ClIuYVn!4ITsJ9KRocZvdJ5uYson z(fz-Jo_`|{{eLg`9U%MteGa?=RKPRAG`JW2{^Q`y;7yOl$!QaEzHv{4C8->T~xunYEQvHlN+TGhbIpSMB&}dZh;vb|Ah4wlMYhK87 zMb8JpV*=7Smz6Qu;=?<5I*kKK=y=~_wq}%kf-`#O+|AgBkl~@1wL3r8U6!s%#;EnA z!1(X2)YdPB*o(-ve~2y|HJP%dNynyBTD?HnB?NQHo!z^%V`zhpaRFg=rGg$}~5(@{Ezwtk`x zV}>WJRLkHV5Yx>tFX5~!hj$ETgby_9g=)ilyZZ0grtIOV+*U#>f|+2%wCyzqBS(FN+u%qt!NsaOp_pv4 zEMrIox1CDES*IW#`MJ3xQFA>`s|4jXJ|CQw?Ct!|UPnerIB`i=vdk)m59xZmH&nE) zL%9~QQ0v*k7?P~WI#0wMKD-|n7zw#Id@jF&OL;J*f?wvRIvaM!%-mU1b5jY;HI#kB z5iIJNH0cIM-Wxf2{clnt;I+~N|bnceY1a< zrdJQ5qqNs)u-Obo29sV~l>b26QI5BRx_PNOHH|T0U*YenFLE8SQBAwZwtg9Zd})YH z*rTj_U=n+0ZQuZ|U~)HL%%KwMjWv%5o%*aa(f^-0x9)My4 z_zm!7bpGE5Vgr~5r-M7u{r??^EkNe{)8JGf^Zx$_l)-u6H^9l@8`uC|3gn!>5|DHL zzJ%^C_JCdBap0@y`(g*U9880wfcW`;9=sh~0!{>*A z-vu{;KLB-bK9K$WQb6YT-vH*oHgGEV8an-F!DqnRKpl*MAEV2E0K5^%-hL;8Uk5*e zZ=&N1|I``^pMQt;5r~L<+TvSw*}1;8(5g0Dd3TA^st2+QxU;(~A53Qpg?e${#HQ|B zw}qC0cyc0_rkzxX2-XRgYO7TyD@wVdTd&s`I>^#?nY?G@KkVNd9UJn=7-D~>{W*8Z zToTZ>NM7uqNIe%s(reW#NV3`hknw_YNfxdRY~+0ft=4OQAv-#z2RUIQk0fHpi9A#} zq$FdxR02Kp|B)n&)*HF6l4c~T%3vNsWOZne&!3ch2FA`P>s0JCj7mJt(rIK8%%3r z$MmQ^l2*Hlkg^)FbN)uVw8Kh8OO_F$w}eJ@LKHnExyYr{X}hUeC>aTLYKw-F*9&E- z{JE^FsW|n(^d6h(IOu6g_>}s)1T-$kb$=~;rDbqhXt|Y=-}>4nX6C3QL?@CYn2}at zuZ=L%vRkV8EZR_GPde3#$9nU>DNB>Aif9hZP0!Bd&z+mQP;66AzjV@KQ=1*%yfoFg zmvvN?+M1lyCZn+?({#purd|7$oDm&Yer+}D9@FD4tiyuRprL2XDm410Lr6*Z;S1K` zC1h5YpHf83BN8I})f`xh8B7)Z21RY9QwyVtVzQ%c93$Ps6=)+VTb_+aQE|2(9BFA5 zWw$TFmSA*x&*x&TYF#3nEWA-=rOQ6h>)isH15|@4A*~R3dC=vig5FOx2_eJ?FU!+y zQBB@WIdk^Z`P0f$tW5-6`xRejwlZ8#hvKJTmn*R+>A(h|)E*26eCh2oacN1&Qk%ZE zy^UYDN?)DyyajcSpY#UR(>!(_iFK=kcdyz6>cx98CJ&0(P>ejH>83(eTR)OK<;h{l zO0)onB`Q*w!%DzwLrJoxsf^W%EozF(Z7n0dB}MFohA~8bdRwMu$_s#{8@3 z|ETe)D}4Pw*#Ga7==-k#F9**CXMlSM_z_S8PXymU&wn+@gCQ^o{u>?ttKb^&6fg;n z1OJGgFZ=z8&M*7@KN19M{Xd4z{~92^0FMFppznVV{2lmP@M<9E{yhQw2)$qY{a*(T zf;@O87z01V2Jn8c4{QW-=HJh-1Nh3wS4Z2RH!sfQNuvu?5K4f3go?89Wi(1y8RB!dtbzB4QK>dJ(U> zS;=2()zQJ~&Jwj&YTF-#ZO3l=b_xS5iS*J`OPfbVOXZ2p1;bd^CS!BFOVf-Prd5(V5X%op=e`>{2`xoIa)2+0)BQhH^!c^v*T1 zxG`@j=x%A!$XKhBk(Anv(88t~E1cTQG{%=B!jlMPJ(qEzuyt%?)53_k^L!+w^_MYL zJgEV-1+~l865gJUON9e@+3R|8Y(x*p!f?zhGgf8eIJTi!;s9`$)5V=-ZINI}-sX+}hP!NFT+zmxo)kTSMobUP+T=+X>cKXW4q$>G~Q zGAltaB1s)KuiDFG!pK2bC|868%@%pIe%9Pya@VTfhmu!Dremz^nHN!h8ckkzypbU5 z5TN_6%(3E1^#?{@Z#rhUsHVFZhI$*wjasu`s<*~YB<~>CS#i3L6jThiN?*bmJHp=% z2K%T`a%{TF{qZpmW|8ZNGQRwxk*71a*ys* zQZq=6oHuElMj-B9bK=ufD0=NU)z~6xD!5hHq!2z%xoD6QIkZ%Rg1&iPu za5nf~;8yGe*MWmT_6x`WIXB>w*a@x&{~Hv*6nGN&AvS_9f@{H5;2Gc)a1{6vHiB=1 zH-Q5{>;_}t5#ZtAHf#kq0NF!eJNPMfg1-R22lj%qz-iz_a0hiF_3~b+8~>78mP^TO zREm1+Q`AcF*vcN|O_Ned`~R|P9yWR_v9pK-@#j&MP98q6`yhY0l@d=&iJ%>ej#hH( zb_33=x(f@HwRISibCOcotzan2=yZ*S+e^c5F;!P6dA()6S<8C`TxTNDids)(y!>)> zO&su;@hZ-UZ1B2khhwrl&ra6Z7ZsNi-)$lR>~Aoc-6?bGINcVSE%tLPxaK}x5=bu3 zyPGW`!(lJmEUqN(Y`?8EIy$zwP)=}&HOk`8;ufOs$u}&bOM_{z z#GYeeN}|i92I)$nQzHVM-%`-VXxUYuC`s)Lm4=O7}fL{RF)Cj*%V1_K+scJtM&3GTEZsW^RP8B$jGpkS0&D zGE`loZ7HnO=zCakF?`nZOhXt$H4Bf}w#?TC22E<4dI-|(#gwl|?=nUY(nCVj9Eu@# z;Ug(RyV?we-Kws{cchq=CmV%=M?m%KtJSK01GrYusyT~)Rj zu}-oVL|ev&!6n*y1wx0nvJG2Pz_i}-Pco|O`W6b?Tk z2@Nt`7o}7Ot-+~V3=P`Ik)%&JRURnR7e@!v{2x>Q$92ZiDFoyUuqGiEa=Ya))1{M& fNt*pKuqCyJ>5CwWVP}?N>8@Cdl)4{=A58s!W(kc5 literal 0 HcmV?d00001 diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index a6aa190..b0d76b0 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -581,17 +581,23 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): nova_mock.keypairs.create.assert_called_once_with(name='mykeys') def test_get_private_key_file(self): + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir', + return_value=False) + self.get_tmpdir.return_value = '/tmp/zaza-model1' self.assertEqual( openstack_utils.get_private_key_file('mykeys'), - 'tests/id_rsa_mykeys') + '/tmp/zaza-model1/id_rsa_mykeys') def test_write_private_key(self): + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir', + return_value=False) + self.get_tmpdir.return_value = '/tmp/zaza-model1' m = mock.mock_open() with mock.patch( 'zaza.openstack.utilities.openstack.open', m, create=False ): openstack_utils.write_private_key('mykeys', 'keycontents') - m.assert_called_once_with('tests/id_rsa_mykeys', 'w') + m.assert_called_once_with('/tmp/zaza-model1/id_rsa_mykeys', 'w') handle = m() handle.write.assert_called_once_with('keycontents') diff --git a/zaza/openstack/utilities/.openstack.py.swp b/zaza/openstack/utilities/.openstack.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..8928f8e097d100e8cc7cc6552fcb436d622ccf75 GIT binary patch literal 16384 zcmeHOZHy#E8E*OT1ov%

9!J($7W~T3Y zx|{CmnH$JqjENBv4M8*}@&ghO)R^$YANoUo_(6_nh(?VVqiBQ(ii-FJCxTCX^z=;c z?uocR)O07$PItW@RrS_eRqtEf+unV+dXVfhcN@6gVi+Iae@<)do|}zR4a11qk;@}N zIP=wWBX3-2(~#SqyR<#;?Mh*B8-Tj`OFPDwT>4!3%M#RP6~?= zw;h|?;GUG3{&@SEGD>EF%mSM&aG|kxX^))h^K+Mzx4m^^GfP&ISs=4OW`WECnFTTn zWERLQkXazJ!2hiUxc_$JL+IUEx`%O6{RaIU|Gug(s>^Rp(O=Q@t}g#+ivABxw{&?) zD~|QQtm%QKKaisTUDLUyU#1%z%b(Wt8#VpT6#XSlKdk9br|2(gdffi2&sL1F{=aE@ zU(4U0qW@LXJDPspIkWPA(R8ZmgB1PGnjW|R@f7_}nr>_Pg>z^1zo6-krr({S|54N1 zn*K_P{=BC9ntt;eXZ8O<(_{O5FGc^orr)6D-@R*A{&$*wOw;d5(Vx@wxc#S7^xtaw zNXvKMG^_tNntqd}|13p+R@3ADFP}Fn|4d5$-V|LAw;_{fh`yAf>uR%~%mSGOG7DrD z$SjaqAhSSbfy@G#1u_f#*IIxgVcmz;?GyPh-v7t>|1T~!jGqJd0!M)&Pyk-O$S_U= zUj}^O3gF2L4dd&;*MLs~cLUb}Wgrh+44l5eFn$Gm8~7&h4d8y@KEMaA0WJZadb43X z1bhxS2J8kd0-inJFn$UA1o$!V1t0{j0nP(XAs_n$a5K;XYCr|J3YZ5j0nP_bBQJa& zcpCTta0>Ve@G$Th;7;HUAOH>kR|D?=&IQf}&H}!Q-1H;B0Jsu(0lDxaz!(?-L!b+6 z09OIK00VddIrk5Nhk*Nly8s_31MdSCfeU~qk^4UZJPLdfxE&Y)Hvm@wy8r`t4oT@L z;IqIzz=wek0(*cfflGlGF|I!X9tXY)dFD#CA>6?Asxsz8`qh>XSYV!NFV-Mm`HHCb+2EG3H6lk($KEA0!IdtacKz zU}#%RYGO2iX35~#At&4;_P}@GxF-#Q>*S*R5w|%nuzR-K3K{3N+siGJT(GVb9q1=2;L?o&a8X#ZAwAaySbJQ3+V2s!9kuR#(#R4h{Qlj|QEI5hvgEB&3}(=5aO-x#zODuF3OGRbbgJoHA`cS-4}i z>pA!gD(w!caL4L!y&+BYLKN+seire-OV1xymC9Pm*O%%CTg{_~Dh;xNxumD6n4{{- z-4-y1(U6@ih`BDT?s!qh9`p*g(ZK4TG;y0-&{MPPS#nb5bA8VJacvdUK;yJy^P~(%Pu6DPxqY^?I$5 zl)^%BzdGK?bv)WJEjtLjP#MK^8P6G7s$N;Elp0doYmqC$v?2OX)aLHw3}e9%rZ^TI ziwKry@!JyL+l9*5J?65&wggGvPE*+li}~WSKX7*gZ$QM?eCA|9Fee}Gi8a~_ILRk$ zanZo*Fvl=7LliiU-Hx9)JHbZ*^^<3pS~edi&jY_}J1l+<$Km8QYGc>IrjeA`Ct_kS zk??6i2ljC{EmijR?i}aLnGxH?}%PGTswId#=kYvGBy}SM5+`z0oX{ z547}>*k~OpRqGAq0f#^ki*C8LT3Ih0tmxv#k?Q_tx-ii+#VrTG^um^2K*P8td1G#F z&X8d)7P81;#PVEDZ5P2Raic*S0kYR6-N=>A4N2Dv5WiyWpeS&AWMuO`sUc2ki0_|G zL+CrGH2q7~dE(dun^U<0lSqUsGB934u8DngDKAyVM zb|;jBbF4XCkL#2MofDfC3?p_;(JZXdXtoFel^=+anHoHOJ8h_vY-7x{NDiwFouGXr zFU04x(A>h+vcYAv4{1=Jlcn8z_K`AmMF>M3vc@qUBL9C3x#@$*V@3WSzu$Zex&Eht zPXYG;6u2CC0{Q-ZfCKCWoS)!0wPQa7>@!MIXG{{`2o%~WssJ)5Ceq>Z4ipE75R6OAeOVc;}!L@ zU>HJ&xFqk5IUG0~s@L9MDK}fIwS%SVdP{`Gxtnl^fTIB!P~Mo+$F8l8H$beDyUCCm zW|V9+D)s+N!@pU*-9@Np$6+{GSs{CuWxJA&H4);kH_y}|&Rptp-JwJ#?(`Mq$1))P|Grz@-B=Er$^jP9TDHdF<5^$!uO8qO=6>N^+}C zc4o{aQjdH*!X^~ub5r!(B3W9Pa+XBZCauC1Vm-26m1v3LIInIYCQnLLewUQR!st(O z<|JCwiYYCTbhRSa=6OTt6^WMkT@LuzXUmfuTV?ZlGMEiDDG;C^Lx!jYdbHntt9F4GC`uEE$!XuvZ-mlJTRg`DuU5`nJbdyc z1Wo{6yP{Z9tZys1c!!2OChnG-wp!fc`t!-SE+SG9EloCj%6X7iuSFIKcJ4yu1I Date: Tue, 7 Jul 2020 12:24:32 +0000 Subject: [PATCH 595/898] Remove swap files --- .../.test_zaza_utilities_openstack.py.swp | Bin 73728 -> 0 bytes zaza/openstack/utilities/.openstack.py.swp | Bin 16384 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 unit_tests/utilities/.test_zaza_utilities_openstack.py.swp delete mode 100644 zaza/openstack/utilities/.openstack.py.swp diff --git a/unit_tests/utilities/.test_zaza_utilities_openstack.py.swp b/unit_tests/utilities/.test_zaza_utilities_openstack.py.swp deleted file mode 100644 index 2ef70f74efe45a7a7db302b8b0e7266b3d9ec4d8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 73728 zcmeI537lkAb^nXSEi6h%-2M`Znqc*q?&@u(XOY%4LjwcM40bmlv}0La^{VGN)m2R` z-P4TxT_U2mCmQ3fQImf#q7qcZC1TvNs2Ig1sA&G8Ni+!x<{u51|M%Rt-23i(we-vY zLF@JB*H!i2UCurC-23i1_uP9IMlRmDE4?*0lHz()D)p7cWAnpDJuCIO*;J~v(5g0D z@}TpzSZ&qTh87Bqrdz9ywGaEhy#$9>YxVuZ3-v;^xX)<}Us1TCFjQN1s*Pr$xPPeW zG@6a!R@H6h{jW{8;x=7E%I~}c&$ohm`7mN`d98iU{U?16{EBm+)hrKfJs~wdHmaKX zhRkW{lYjl9UvZ}Ni{Ix!p96gk^f}PyK%WDB4)i(D=fJNP2b#-|NfaB_ce(ZaVgB=Z|MN4}^M_l{PdmoD5#;{}|9O!9bFAmT z=0Cs0|NN5m{5b!9z@GuXKtcM)`_G@|e?G^0?kAUewEz6`)^q=M|D(@=J_q_7=yRaY zfj$TN9O!eP&w)M%`W)zU;FpvGg;FXdI^>(>A4>SM{BnK=h5g&$P2f^642HlxDEdDK zH-KeuI=CIX*PR;H}^-;3eQvunC+5{t1Qm zv*0b@#b5z!21kQ0pe$bxo(G-^P5?KfU|$3FgK=;-3i)Ti>%e7TJNPL|{N3O-@Lq5b zJQX|~+>8O>`QUk=493A2csw{7+=+qVI74+#(fWHK<22TSgfk%N) zp?n_zt6&RwG`JP}!3V$%pb7>+7TiHSsJf7PNc&gAsg!e7r`4?2s>O=yRGYbqi!a2& z;&Q&R>=gDp^?a?GFVzc+@~rMOTlH#wrBG=(>B;mnx~8L|HSBmQSMO0rQi^q_&~)-u zrzwsdiK+O{x)S1tOC}|FsEHBwcvO~3b*ItDH;VX!Z0wYcPi(=AV3Rx$CXk##qhk)L zW+LgwiZh^#kWjTgiRDVI&~&Sd?y^mvilZNg)Fz~*h|xh~lvqhMdLeANR*%V;N=F`a zB}M`zlcAn;h>s{Q)r$Lby9$eLaTnhPHfnOErsTSE@0BU_#rl_ZOWEz|?8uhF$d+we zOGCx6ZN;HYg>7R)Tg%0Tq2j1h+&r;ueAAY(ZP~$C3I`*1GDf>=WVWa0!>5W3=2K`s zxDFy0+-jk|CWQK}eTc+{{C1r{mlV=3*_QHnpHVLJC&NyyhM+lXpK}noQ6rC5kV9svDROo@DkKHDZQn#8Z>vM6b+py|&_( zoceY@qkMC1*&(w`xl(9mu1v0MoraFTyl<`H77LZ2pOQ4>9_fx|;x(?5kZ+lKt<{7G z=K%k!g$j}9>Mf^hw5C%nRGWFXLah_+QT@^Wc^%%K59}N|+(@1EqW}_Ume`PwbrhBcr%lQ?j-oV`udN=8! z+PQeYGmU+Px+8VaQ>B@VkCZDfPokMwe9<)#jToM=7|`6V)EQH56t@z@MeZAAQ^DfG zA-5CCQ+HSVfff(x*}|We6!^V`1oFRH0q9iyUdT(x`y~}(1q{k8!GiBrkg%jGDG?II z2Q8PaWv4zchwJVhe0*oc$M+(GMXWN)-IL(Wed6SE*m7Fqn<+n)c#_g$8u~KL> z((%{>Ed)rC$(>nfIP!c4pAsWguGf~*;w7BZzQLitaN1o`%ykYFon>)XZuAyDOe$Sy zq{AN`2`N8K?X#Ryq)E_#Oq3umW9jyaZgVa8-DoXP8$|}M!5tan1mE3S@U2A6H{B&C z_{~5Uf9DP9io4`C87ESbbgRmXGTq$gq%Uxb_)|MM0?ay&c`IG6c|p({if-90rYnW& zVvF~Cda*{IT4ltUrpA`shIFrubfH=zl}mQcoxA6vx%4GdGc!}W=XOrdruWRGckJ1H z*3P+|dv^2d?DW*`OVj7?+70`JpOoZ__G zhyCxxD4lv!@r<(uHMmC>9nf4kcQZJPV6; zhiA=N+O4Kn>-dmY7m>PJxw*>gAhm&X<#hNe%Pk1N23e(I)gjaq&nv9xKGzL!w(-(X1!8G)x2c+&Zex%ICCo6(?KwsTm z+}BKxjf`$h?N&H=}QucG6>1vJ1gxSO;;;pg#3l28BYbD+P%dtrWa@ufcR=8ca{bSUP#GSgV%Z#a7+PFQPfG7S{4&3K893&+aI5 z*dnH=Ac$B%8o37c=DbrbW2eQYF`q3nzcwU~vU^hjRU6X2VHyQc9*LTqMt3akM};di z)2mLpRI6s2>3YFM^_yWKHOl*RyKMyP2FnB-tH z2?*(lA9-O%-dS31uH~6+bn1mBWeU@fwANgwQi|M>yWXs1-vMt0SA!RU z0yqOG+ReVB|Gm$FJ_q_7=yRaYfj$TN9B9vh$hfbE;nQ+!ftjKFerL__rIE2^UPhit zLrcj)Mzw15B3$y^lEg+EVH&Iv&T|HI$Y*wGjcg$TO>$QV>%EFbcM_vAb4v>_M}~Cx5aWQU*fqMA<8p>dW+@oK)j@9lGmS zJ6CJu5T<>D>5SWu89(0}Ct4;=QeML3)s$;>F;OOjFyRn!D1s#z@-R_2bPl@ES|lWz z!97bX482eSX4}oE8?)>}Lv}E|G??~W7?rk`lGt)7T~z63s-0;8Tngku9;yw!2Jk2? z<(cv>D6wc;XfI*;AE_>-B&LFgN4&OT6|@iKzvAR%irL9|Z7k0OD=HpdxkV>0B_GIA zZx|(*|8LSTL+ZS^!U5k0LdaffpUQA^Y1!K{ZK+mrDx<^nC-ny1kJsX9`>RIkl9(+S zicFML&wIpd5JXNU@<_-+R2N>4Flc6}s+9f!CY0Q|-9Ke*=44aCfaEJ8G3##NC1<%( zTU(+s(dJitz1*Y9e=&q@Jxiq%Yz&~}}wK2q1H>Q9oIe7w1f{=WhL z_F;VBMgI@h&wmSD|7IXQ0Q7Sunn95jtBpY?*A$9O0Wnn1lz%n(f#iPp8{_MC2$`2EpQC@C-ncX zfY*aR0wwTNa4h&HHjPh%_k!1e8Sn`3aBw^2eI0Otl-&Wc79ay20X_*&J_MG5tOGa& zd>23e?|_ej=YkoK1`h)_8Mc%lvy|*rrKkU8=2Cb>`i(RFOO*o~?#P zHC>_4qs``!FikT+h3ibH1W9CG8>v!!)ZLpRxs#1EC-p6+DC}0eEBxBeXl8azfr!a< zxXwhPwXvSa#;De-9V4>AyI#9#PZXDF0C^e$Q}VU?TCBOKs!M1BtM0-=Wi2LeZL3kS zH<^Fk1yT_a$x+vvdX_9(qW_}~rw;h*|AYPMzKD+hYLExN1%8Z<|Ht4`@Obd|==EO$ zqVGQ!oC!_>Uq+Yz6X1ZYK-TfU6+8>%!5DZr_zt@Jd%?>=15AQb!9&5nqPzbHd=R`0 zh)yqS`cDG?4`Ocy?*P|;E5I7K1Uw%60Nwp#-~c!e>;$6I|10|Y-+@^Fhm z11|uwrvD7^U+C+f2G0ZMfz!cngRi2m-vHhYUJRPxJa97jIr{o7K-TjA1^5H-Y_JjB z1z)cN!rxWlv42UGBwrE3RJKuDa&TK>(!qR-cWsE6pX-GscYlpx_f)G{n<|wq^b(2o zI8hQJSqCYoj#|soy0V^~J$HxNufXW&BI$N*hcZZGk<1b%*2H$l!WtIj$^bOp0HxWo zB|&L2i>6nptg#BTB=@*w&?79!j*jHyZ*;_X5Fu0?%Br;g-Wp=3oHZ*|&P!`6OGc(` zsZkH2uME4}7hcJ0S9kf*A~*+}V!%xMr8SUkG)u0S@A+i0GjhWhr(Wl+m6E*W@*YuV zZY?*oO zVPS5Env1y5TV|+y5G{kgSy?@OFD}ujme%+%1~sZ@sJE(?17$}-nCX$dN{Y;7Ol}&} zeN*>dMkkA?lB+?86d_n=nOiMzk_=1U9AnJdk;~FD>T4<(Z6yDIaC$!FIC0UwYcBm--N7dI$!mqk>+RN0MIGHMS*y5IO-XEy9 zA1QY%d9r*Ra#bWVEsoicdf^4rMEqToCU zvZ|9+JSw=|v+5l-%n0pXe3EUL z8G#N88$7CITM&MX`97y1d)t)~sn0KoH)%4}9)&YD5To2v=WG}#3 z@C0xS_y)EB@dx-Q_hTT*Z@8NUIktWrogG-cyJeX0GapyIJg>A!A0NTw~&0ro}22KOFVh^|$ybA0Fj|D%&4)CAg zBj9=94DbYS3p^LT2lM=aZ6Nj;NT_^<1KUQkwwy1N*^nw}6YxT*+~k z<%@cqMt4B;iW+u{oysS6>!6IW7qb*jb#Z0^H6P`%Tq;PUob)PZPn|!VpE_rH_gsGG zS@ux%P->vpf$9A$1U+%NBZoaEcKe@2Pa9HZTO6w_g{~btmE`7qQ67kjksdg-QIhpG zK)KzX3{S}pn!9B4o9d#aB_Af!BZRhVYEk+)jXaq-|R~^mY`X7EPkr zzTtRu#G6dF+NM+S8&qVMLrdf9t=bryd<7^xlTDEt%s>4>3QL3Yv#%KS%hL z7R)s2DKE(-Hh}P(vp*lkSgug2xK%b**f>A3Hz(U0vZJLgs|+;)4h-a(8*r<7noWK) zHDGp&_e*Q^XYEA)mt8YIY3ctTLC;?U=Yn&1JQjQ# zJ^x$adQbxwfMdYd(eb|q{tT4C4sa~^273M1!FAvb-~w8}d{f~gxg4cjWa2k;P{=WiV4CL&= zqktT`D15(Jcz?*3NT^&&O2yFT?}RCPf->7H+n}(hN`Bch6PfXh7W9~q+wU4lT8y$y zjI8A020JhhY#dCFX<=VK8Dh0fsUn!AD7NZ#Szc4a3hZrh8CQWa0X=M~o2j@97?0O7 zS6XH2G~n|bo!WEEnATS1>LINJy8To_jiW6mB@eSBFzQ5^JnQ(+vdyi`;V)~Bn|FDk z9B`^G+nitCa@fE${gmLhY!kb`5c1g%<=rJ#QfL78`5VMN^Q8Q zNf6pSCnCE#bxJCXv#K0mW&0j)F5htTwnipBtd|whF}e%0AjUXHRl4eUJnLYj8_^v1 zW5>pZM@nYe5Nb@F8jZw~s+2HKE=5ehW4~LsTg*Z6tygauneg^HC}W$Ym?0^cj`fy>;%Hz1Vv{ zC$!C;>RWqM=-^9i#Lb9W6du=!z4(3c(v4GuBE4re+zi_@#N1yTm{mk}EPm(m#797? zjrfsb#h=HmL$OD@{woG}%%ZmwmnyUHF}~WH+(ihuFNI~w+ofGbqrLDeUUaL-J|hy! z9Po%&f^x$$dTQM}=r${bHeLbI`qF}?f*pysz_VM=ZmFFqteo^)L{*y6?@QL*=xy8& zl;TWPW6DuiWQTWEN1FA>{yv~O?Clhjjh9!TOmv_+@`eqgvbjY=POk{Y#A>(~d)-Qj z<%&+^#ODfADwO{(B-TIEqSIt%IL#KO(7lWH=eEu~=fYXU*RO&zz>|TT zHSjR-ee`=-?=N%yF9f^6CLn7AZbHYegNKqF8|ly9pFkJy1ks` zC+h%S4K4#u0Z#qmj`_xZx(*yYQBu|p+iqKKK#+lR*< z9(m8COA7@+-@^7(Xtw11rR>y>xt$kJ>y~8%YAwrYBC`KvmLnmy4ULQsjcl149pAob zbo=J5mnFqX>P6#(cM2t;4{-)nx$sJ-P8$E!p3XI@=Bd&Awj-S zY`QB>+qRqK0Fa@5oHJHmaak;2XsJPw+u8HQfyv*fHWh@GPrHHJw1p+o4gfv0YBD>g zg}?u=4Q66WOXGcak7om`|Xi!)5BcFK_0=D3zz{J0;a6m%i1p4_k+@wQa;Y zt4?zCV3oGKG`4(^eWezYd$`WMTT;)aS)oU#htQ*KNbi)xjyU~_ld-4->A}6-vO_l< zIsSvRGc3DO&st=ZM_b`lQjkfK1HN*9o!642pHsw6dA zw#8Bsq}kizk${$5q-<6%IPX2w8m;#`$8PeI0oAVFN8Nkcsmt<0z+?htydP@c`b;X| zksm_JFJYF}yc;?|NVx1_-c)nchA{{uy&O{JMbcVy7|DAOtx0+RdFzpkgu)o>5+bou zMC)DGO7wo08Vs`6BH&3x$S0+H$6UsQi$PRN{Ozjs3Db4!-^k_NYPkmQ4+h6Y21W3M z)jP*ExaXFx3`z3ub*ld$nyy~|j;`Mzh$kcdFH1;*B6iA3F-Y1)V5>D}1Z|aNEG&e! zX|&zpUUnWxOx2i+Q`RnD3Z+uQF~8MP?Y=8ktx0l{R3e$<9M>Wimfo;}be?LB7}&z- zp>3hpIZkg9EH=eyO6od`B>F$#3AbP*}`Tsc(-G3O|iJmWe z`+XF=3Y5S$uoWB!?m*Z71b72j2AjYm!M~vEe*(M^Q zvfwU?=|C-#3Xayvt}Mx`!(7Oq3SxFIDM?+f!x@sVLyNnwO$L!yMAz$b+#c1$C9%E29_n_})-(r+jORopra0aCoRZAt(s37D z+%wiBwonk(KuQIRgw(cIjyP5){1}$hp!glT89&fiuLN(3XE2uQmEbYM(Bf66;WrVK zf26Z?jzBF`tY3}rM zx1YXi`{}cnWeq>jNKni2KB6q|THJ%M!MWeL@C)vxc=Ihe+Eg5bO|sp{GhOUx`gWtX zjR0|rc`C?i9oFd?UNePy^8v9<2R7;4gMG>)Ru$6i9s1Nq&!%aq@57L4Z$2q1b&_N# z%;A^5RBN-e(b2^%m3@&G4&-AEA`t2PlH)WB{`2-ze7*J5_WWp4FCB_-_$67ElX~Z* zp7}6(Pz}59s47d`(Q;Qc_({gb)> zM+4FQuLH6N;5?A||D(Yd(ewWlTn=V|?By@}{=FZ(9;^Y;`%eQWgRi6OzX!-3fU`i> z^?x5d|3lyA52I04*^o_`nkFmS;DxC7n(ec+{FKbQs^z%S78WqrS_?YH;-6BbJ^ z7+h-i2WI3M_Ls1`%JOBc7vKNSg|WnHp_=_#z^1!b!&4zi3UJ3KySrz~~^ zv2!%zgO1DqjF^i>FrAjcEjyS)hwr9x*>gb#&ps%D0CkYIIV?O09*n0c>55X&@)s zpyvqWFHeXUu@|F($B;1Y%2;A&2$`}#D{3}U1!dm_c2&G^X3u%kJLarPmH^qQ3wP!( zo}QWAxo0<3G=63LHIE}e-cKNZ)}CEcJ9jG+l2oX#Fy=igZc_45vy&00(I#||lUSE} z2rh=GL!#nkw;lzOm|- z@wG@v4VjlzMPOBvp#E;Kem670Kn<(f@Cx>Ulo`!xJ4uZruzGltJjP)Hk%ZJ~McyP_ zT-G>J7q4Zvxi;jBg++ySvW`-hRiz!G&pKe9$JVKDXlF_diM3z9Nkxt}Ew;`Sru$?> z%^tBv66PaZ?$AEGGkz0WhZ!r7brsy^8daVoUSbo!B%oI-Hm6xRvZqec3pPn#M;&? zF14%6Z0vqzZQ54eR+`vaTG;Ar-85b<7q*U;Mz)P?+cL7H=!{Jij3mP56%IPo?iu|t znXF?`GkVsGLl|aH>m*H!I=Ev!eeGp^v^?=A@bbf0`(#Twg=qe>b=q z$Qb~e!Li`$==pN)Ulp7KMEAcNJ^!!4yFmf`Hh2X1F}l92`M&{({=XmOfEXNZMc=;` z90Zqwjo^47Hh`aizXN{=E&(Tl&!h8S1 zK-U-je*x?UvabKL@KX5sGU4fZE*WmfrR1%%l9{coR8iLP(u(Yq)=SLNN}WS1y$LF- zPwA2;r(SXfhO8an+#)s}tg_A=%K|g(`+1KVe)1de@Jz2d>BXA%;oP7+JJs%P=>jgG zl}dVNb}5a`gShyC;`-UF;kwOK9e!Tv1#uItmPjQ_NJ24TTG#soY)GHI^Xa>$w|fzN zx6y)HJuYWVrRAmLCC0)50y*h&tBTt{E~k(t8I*kAx^T+J+bf|}ck+uAakU_aG~@Wk zkSV4)D>i$oMDi}WM(vn8fv9+*Q5cRx)^_Rh=R0d^kM_ZI@J&yd8u=#ffqF!;75(gf z$x!sLdZ51L7JG+ag%RMPx!zLc7!> zmmZ0=-8>aJx~)Wsm0wCiJ2JqQv`+PJng&ynd?@MBlG$}FK|@)WVc-fD3pR$g^OjUP z5*yT3ds|PuCX$AowUtIt3COfmru8D~PKi|&Dv^{FH2J!JDJ1|rEW{E~N$rc_pms<``BD7*7>(QJR1P}Zz#a6T9 z{Bs%?FODr63VF~JB;|G3i4AYh4#uZ|hW0kN@d@9mJeISqP1?k!WL27b;NKbO%H8`K zp`dS9mdTef%M97OB-w`9X_vi-4%GH7?<@H2G&Tfv`!MX(wC5MBRYz+ZtEgQtND!4@F< z{@)0iU>wNVfWHYoM0sDdY06Yxbg1-Mn@Ii0@hC z_+K){s9wnO0~zNF_!0fJOIWEjxqceh@$5mTPql5e&_0xYR4KkeOK!QgJRobbVtGhF z@<}|UK$S?xkYWVIYhX4tw`-3V&)f6eCnvW5jec)#NS{@6yw$R6PBX3SDwh28QgMmP z^ByF1wjn)Lk-b;fywW$U;>(}{J}O1Sp5hB+Pd!XY4aueM_R#8rRPCnLPRb-*+xFo5 z`s5qZB71Fhkw}No_9F59h7`FspNafDgw~jps+#xL4I+|7`^Fjxrdw*{w%1-Gw=#PZ zT5Tjm`?ebirU@3=h%%dpE-j=*WIm>l^bK{e2w`-d({;E*&hZsIiexdYvcV z<}>o!Oi8ETzO@Ho^6Z_Rk_IQ2#ib~v3C%o#d14662Z_7O@iu6b-fq>0`i|Av1vS-M zlhpB!OeM+u#eG{r|5F=az)6uN5eoC|n;;~93lq5wYAn{aR5S^p-wW!NIkuI@e$~m> zcuNR{Ix;phqNrD8@S77|{5GcZlyuv-B6(^xACr)EG#QDO&{))MYY=;=%|y})*xg$L zkr)AU`1(Ne_kXJN3!7qMABvuN&{<0K|3P%pH=&=({C}YT-++$4A6x>C2cJa0m%aa1 zfb98yH25I;{Q)or{tbQpR`5!&0~`%Li!T3iPy`F$N#JMb^s>L-)nEba1mfp^Joqbg z`fI@=I14-pJQn;K_zAlGmw~MJe+QTYli*?C)9ClIuYVn!4ITsJ9KRocZvdJ5uYson z(fz-Jo_`|{{eLg`9U%MteGa?=RKPRAG`JW2{^Q`y;7yOl$!QaEzHv{4C8->T~xunYEQvHlN+TGhbIpSMB&}dZh;vb|Ah4wlMYhK87 zMb8JpV*=7Smz6Qu;=?<5I*kKK=y=~_wq}%kf-`#O+|AgBkl~@1wL3r8U6!s%#;EnA z!1(X2)YdPB*o(-ve~2y|HJP%dNynyBTD?HnB?NQHo!z^%V`zhpaRFg=rGg$}~5(@{Ezwtk`x zV}>WJRLkHV5Yx>tFX5~!hj$ETgby_9g=)ilyZZ0grtIOV+*U#>f|+2%wCyzqBS(FN+u%qt!NsaOp_pv4 zEMrIox1CDES*IW#`MJ3xQFA>`s|4jXJ|CQw?Ct!|UPnerIB`i=vdk)m59xZmH&nE) zL%9~QQ0v*k7?P~WI#0wMKD-|n7zw#Id@jF&OL;J*f?wvRIvaM!%-mU1b5jY;HI#kB z5iIJNH0cIM-Wxf2{clnt;I+~N|bnceY1a< zrdJQ5qqNs)u-Obo29sV~l>b26QI5BRx_PNOHH|T0U*YenFLE8SQBAwZwtg9Zd})YH z*rTj_U=n+0ZQuZ|U~)HL%%KwMjWv%5o%*aa(f^-0x9)My4 z_zm!7bpGE5Vgr~5r-M7u{r??^EkNe{)8JGf^Zx$_l)-u6H^9l@8`uC|3gn!>5|DHL zzJ%^C_JCdBap0@y`(g*U9880wfcW`;9=sh~0!{>*A z-vu{;KLB-bK9K$WQb6YT-vH*oHgGEV8an-F!DqnRKpl*MAEV2E0K5^%-hL;8Uk5*e zZ=&N1|I``^pMQt;5r~L<+TvSw*}1;8(5g0Dd3TA^st2+QxU;(~A53Qpg?e${#HQ|B zw}qC0cyc0_rkzxX2-XRgYO7TyD@wVdTd&s`I>^#?nY?G@KkVNd9UJn=7-D~>{W*8Z zToTZ>NM7uqNIe%s(reW#NV3`hknw_YNfxdRY~+0ft=4OQAv-#z2RUIQk0fHpi9A#} zq$FdxR02Kp|B)n&)*HF6l4c~T%3vNsWOZne&!3ch2FA`P>s0JCj7mJt(rIK8%%3r z$MmQ^l2*Hlkg^)FbN)uVw8Kh8OO_F$w}eJ@LKHnExyYr{X}hUeC>aTLYKw-F*9&E- z{JE^FsW|n(^d6h(IOu6g_>}s)1T-$kb$=~;rDbqhXt|Y=-}>4nX6C3QL?@CYn2}at zuZ=L%vRkV8EZR_GPde3#$9nU>DNB>Aif9hZP0!Bd&z+mQP;66AzjV@KQ=1*%yfoFg zmvvN?+M1lyCZn+?({#purd|7$oDm&Yer+}D9@FD4tiyuRprL2XDm410Lr6*Z;S1K` zC1h5YpHf83BN8I})f`xh8B7)Z21RY9QwyVtVzQ%c93$Ps6=)+VTb_+aQE|2(9BFA5 zWw$TFmSA*x&*x&TYF#3nEWA-=rOQ6h>)isH15|@4A*~R3dC=vig5FOx2_eJ?FU!+y zQBB@WIdk^Z`P0f$tW5-6`xRejwlZ8#hvKJTmn*R+>A(h|)E*26eCh2oacN1&Qk%ZE zy^UYDN?)DyyajcSpY#UR(>!(_iFK=kcdyz6>cx98CJ&0(P>ejH>83(eTR)OK<;h{l zO0)onB`Q*w!%DzwLrJoxsf^W%EozF(Z7n0dB}MFohA~8bdRwMu$_s#{8@3 z|ETe)D}4Pw*#Ga7==-k#F9**CXMlSM_z_S8PXymU&wn+@gCQ^o{u>?ttKb^&6fg;n z1OJGgFZ=z8&M*7@KN19M{Xd4z{~92^0FMFppznVV{2lmP@M<9E{yhQw2)$qY{a*(T zf;@O87z01V2Jn8c4{QW-=HJh-1Nh3wS4Z2RH!sfQNuvu?5K4f3go?89Wi(1y8RB!dtbzB4QK>dJ(U> zS;=2()zQJ~&Jwj&YTF-#ZO3l=b_xS5iS*J`OPfbVOXZ2p1;bd^CS!BFOVf-Prd5(V5X%op=e`>{2`xoIa)2+0)BQhH^!c^v*T1 zxG`@j=x%A!$XKhBk(Anv(88t~E1cTQG{%=B!jlMPJ(qEzuyt%?)53_k^L!+w^_MYL zJgEV-1+~l865gJUON9e@+3R|8Y(x*p!f?zhGgf8eIJTi!;s9`$)5V=-ZINI}-sX+}hP!NFT+zmxo)kTSMobUP+T=+X>cKXW4q$>G~Q zGAltaB1s)KuiDFG!pK2bC|868%@%pIe%9Pya@VTfhmu!Dremz^nHN!h8ckkzypbU5 z5TN_6%(3E1^#?{@Z#rhUsHVFZhI$*wjasu`s<*~YB<~>CS#i3L6jThiN?*bmJHp=% z2K%T`a%{TF{qZpmW|8ZNGQRwxk*71a*ys* zQZq=6oHuElMj-B9bK=ufD0=NU)z~6xD!5hHq!2z%xoD6QIkZ%Rg1&iPu za5nf~;8yGe*MWmT_6x`WIXB>w*a@x&{~Hv*6nGN&AvS_9f@{H5;2Gc)a1{6vHiB=1 zH-Q5{>;_}t5#ZtAHf#kq0NF!eJNPMfg1-R22lj%qz-iz_a0hiF_3~b+8~>78mP^TO zREm1+Q`AcF*vcN|O_Ned`~R|P9yWR_v9pK-@#j&MP98q6`yhY0l@d=&iJ%>ej#hH( zb_33=x(f@HwRISibCOcotzan2=yZ*S+e^c5F;!P6dA()6S<8C`TxTNDids)(y!>)> zO&su;@hZ-UZ1B2khhwrl&ra6Z7ZsNi-)$lR>~Aoc-6?bGINcVSE%tLPxaK}x5=bu3 zyPGW`!(lJmEUqN(Y`?8EIy$zwP)=}&HOk`8;ufOs$u}&bOM_{z z#GYeeN}|i92I)$nQzHVM-%`-VXxUYuC`s)Lm4=O7}fL{RF)Cj*%V1_K+scJtM&3GTEZsW^RP8B$jGpkS0&D zGE`loZ7HnO=zCakF?`nZOhXt$H4Bf}w#?TC22E<4dI-|(#gwl|?=nUY(nCVj9Eu@# z;Ug(RyV?we-Kws{cchq=CmV%=M?m%KtJSK01GrYusyT~)Rj zu}-oVL|ev&!6n*y1wx0nvJG2Pz_i}-Pco|O`W6b?Tk z2@Nt`7o}7Ot-+~V3=P`Ik)%&JRURnR7e@!v{2x>Q$92ZiDFoyUuqGiEa=Ya))1{M& fNt*pKuqCyJ>5CwWVP}?N>8@Cdl)4{=A58s!W(kc5 diff --git a/zaza/openstack/utilities/.openstack.py.swp b/zaza/openstack/utilities/.openstack.py.swp deleted file mode 100644 index 8928f8e097d100e8cc7cc6552fcb436d622ccf75..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHOZHy#E8E*OT1ov%

9!J($7W~T3Y zx|{CmnH$JqjENBv4M8*}@&ghO)R^$YANoUo_(6_nh(?VVqiBQ(ii-FJCxTCX^z=;c z?uocR)O07$PItW@RrS_eRqtEf+unV+dXVfhcN@6gVi+Iae@<)do|}zR4a11qk;@}N zIP=wWBX3-2(~#SqyR<#;?Mh*B8-Tj`OFPDwT>4!3%M#RP6~?= zw;h|?;GUG3{&@SEGD>EF%mSM&aG|kxX^))h^K+Mzx4m^^GfP&ISs=4OW`WECnFTTn zWERLQkXazJ!2hiUxc_$JL+IUEx`%O6{RaIU|Gug(s>^Rp(O=Q@t}g#+ivABxw{&?) zD~|QQtm%QKKaisTUDLUyU#1%z%b(Wt8#VpT6#XSlKdk9br|2(gdffi2&sL1F{=aE@ zU(4U0qW@LXJDPspIkWPA(R8ZmgB1PGnjW|R@f7_}nr>_Pg>z^1zo6-krr({S|54N1 zn*K_P{=BC9ntt;eXZ8O<(_{O5FGc^orr)6D-@R*A{&$*wOw;d5(Vx@wxc#S7^xtaw zNXvKMG^_tNntqd}|13p+R@3ADFP}Fn|4d5$-V|LAw;_{fh`yAf>uR%~%mSGOG7DrD z$SjaqAhSSbfy@G#1u_f#*IIxgVcmz;?GyPh-v7t>|1T~!jGqJd0!M)&Pyk-O$S_U= zUj}^O3gF2L4dd&;*MLs~cLUb}Wgrh+44l5eFn$Gm8~7&h4d8y@KEMaA0WJZadb43X z1bhxS2J8kd0-inJFn$UA1o$!V1t0{j0nP(XAs_n$a5K;XYCr|J3YZ5j0nP_bBQJa& zcpCTta0>Ve@G$Th;7;HUAOH>kR|D?=&IQf}&H}!Q-1H;B0Jsu(0lDxaz!(?-L!b+6 z09OIK00VddIrk5Nhk*Nly8s_31MdSCfeU~qk^4UZJPLdfxE&Y)Hvm@wy8r`t4oT@L z;IqIzz=wek0(*cfflGlGF|I!X9tXY)dFD#CA>6?Asxsz8`qh>XSYV!NFV-Mm`HHCb+2EG3H6lk($KEA0!IdtacKz zU}#%RYGO2iX35~#At&4;_P}@GxF-#Q>*S*R5w|%nuzR-K3K{3N+siGJT(GVb9q1=2;L?o&a8X#ZAwAaySbJQ3+V2s!9kuR#(#R4h{Qlj|QEI5hvgEB&3}(=5aO-x#zODuF3OGRbbgJoHA`cS-4}i z>pA!gD(w!caL4L!y&+BYLKN+seire-OV1xymC9Pm*O%%CTg{_~Dh;xNxumD6n4{{- z-4-y1(U6@ih`BDT?s!qh9`p*g(ZK4TG;y0-&{MPPS#nb5bA8VJacvdUK;yJy^P~(%Pu6DPxqY^?I$5 zl)^%BzdGK?bv)WJEjtLjP#MK^8P6G7s$N;Elp0doYmqC$v?2OX)aLHw3}e9%rZ^TI ziwKry@!JyL+l9*5J?65&wggGvPE*+li}~WSKX7*gZ$QM?eCA|9Fee}Gi8a~_ILRk$ zanZo*Fvl=7LliiU-Hx9)JHbZ*^^<3pS~edi&jY_}J1l+<$Km8QYGc>IrjeA`Ct_kS zk??6i2ljC{EmijR?i}aLnGxH?}%PGTswId#=kYvGBy}SM5+`z0oX{ z547}>*k~OpRqGAq0f#^ki*C8LT3Ih0tmxv#k?Q_tx-ii+#VrTG^um^2K*P8td1G#F z&X8d)7P81;#PVEDZ5P2Raic*S0kYR6-N=>A4N2Dv5WiyWpeS&AWMuO`sUc2ki0_|G zL+CrGH2q7~dE(dun^U<0lSqUsGB934u8DngDKAyVM zb|;jBbF4XCkL#2MofDfC3?p_;(JZXdXtoFel^=+anHoHOJ8h_vY-7x{NDiwFouGXr zFU04x(A>h+vcYAv4{1=Jlcn8z_K`AmMF>M3vc@qUBL9C3x#@$*V@3WSzu$Zex&Eht zPXYG;6u2CC0{Q-ZfCKCWoS)!0wPQa7>@!MIXG{{`2o%~WssJ)5Ceq>Z4ipE75R6OAeOVc;}!L@ zU>HJ&xFqk5IUG0~s@L9MDK}fIwS%SVdP{`Gxtnl^fTIB!P~Mo+$F8l8H$beDyUCCm zW|V9+D)s+N!@pU*-9@Np$6+{GSs{CuWxJA&H4);kH_y}|&Rptp-JwJ#?(`Mq$1))P|Grz@-B=Er$^jP9TDHdF<5^$!uO8qO=6>N^+}C zc4o{aQjdH*!X^~ub5r!(B3W9Pa+XBZCauC1Vm-26m1v3LIInIYCQnLLewUQR!st(O z<|JCwiYYCTbhRSa=6OTt6^WMkT@LuzXUmfuTV?ZlGMEiDDG;C^Lx!jYdbHntt9F4GC`uEE$!XuvZ-mlJTRg`DuU5`nJbdyc z1Wo{6yP{Z9tZys1c!!2OChnG-wp!fc`t!-SE+SG9EloCj%6X7iuSFIKcJ4yu1I Date: Tue, 7 Jul 2020 12:32:54 +0000 Subject: [PATCH 596/898] Fix another unit test --- unit_tests/utilities/test_zaza_utilities_openstack.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index b0d76b0..0b504b1 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -582,16 +582,14 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): def test_get_private_key_file(self): self.patch_object(openstack_utils.deployment_env, 'get_tmpdir', - return_value=False) - self.get_tmpdir.return_value = '/tmp/zaza-model1' + return_value='/tmp/zaza-model1') self.assertEqual( openstack_utils.get_private_key_file('mykeys'), '/tmp/zaza-model1/id_rsa_mykeys') def test_write_private_key(self): self.patch_object(openstack_utils.deployment_env, 'get_tmpdir', - return_value=False) - self.get_tmpdir.return_value = '/tmp/zaza-model1' + return_value='/tmp/zaza-model1') m = mock.mock_open() with mock.patch( 'zaza.openstack.utilities.openstack.open', m, create=False @@ -602,6 +600,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): handle.write.assert_called_once_with('keycontents') def test_get_private_key(self): + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir', + return_value='/tmp/zaza-model1') self.patch_object(openstack_utils.os.path, "isfile", return_value=True) m = mock.mock_open(read_data='myprivkey') From 7f122b8611198fc8b36f9a8ce6942e7224b1197c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 7 Jul 2020 12:38:18 +0000 Subject: [PATCH 597/898] Last unit test fix (passing locally, was failing in travis) --- unit_tests/utilities/test_zaza_utilities_openstack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 0b504b1..549ed71 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -613,6 +613,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'myprivkey') def test_get_private_key_file_missing(self): + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir', + return_value='/tmp/zaza-model1') self.patch_object(openstack_utils.os.path, "isfile", return_value=False) self.assertIsNone(openstack_utils.get_private_key('mykeys')) From 7265b79769c05abc28373da0e5ecd60df8c87b17 Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 8 Jul 2020 10:27:01 -0700 Subject: [PATCH 598/898] Fixes for MySQL testing * Better handle workload state checking during cold start * Switch from debug to info to see log messages --- zaza/openstack/charm_tests/mysql/tests.py | 53 +++++++++++------------ 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 7bf0979..bf61932 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -149,7 +149,7 @@ class MySQLCommonTests(MySQLBaseTest): set_alternate = {"max-connections": "1000"} # Make config change, check for service restarts - logging.debug("Setting max connections ...") + logging.info("Setting max connections ...") self.restart_on_changed( self.conf_file, set_default, @@ -198,7 +198,7 @@ class PerconaClusterBaseTest(MySQLBaseTest): output = zaza.model.run_on_leader( self.application, cmd)["Stdout"].strip() value = re.search(r"^.+?\s+(.+)", output).group(1) - logging.debug("%s = %s" % (attr, value)) + logging.info("%s = %s" % (attr, value)) return value def is_pxc_bootstrapped(self): @@ -236,7 +236,7 @@ class PerconaClusterBaseTest(MySQLBaseTest): cmd = "ip -br addr" result = zaza.model.run_on_unit(unit.entity_id, cmd) output = result.get("Stdout").strip() - logging.debug(output) + logging.info(output) if self.vip in output: logging.info("vip ({}) running in {}".format( self.vip, @@ -333,12 +333,12 @@ class PerconaClusterColdStartTest(PerconaClusterBaseTest): juju_utils.get_machine_uuids_for_application(self.application)) # Stop Nodes # Avoid hitting an update-status hook - logging.debug("Wait till model is idle ...") + logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() logging.info("Stopping instances: {}".format(_machines)) for uuid in _machines: self.nova_client.servers.stop(uuid) - logging.debug("Wait till all machines are shutoff ...") + logging.info("Wait till all machines are shutoff ...") for uuid in _machines: openstack_utils.resource_reaches_status(self.nova_client.servers, uuid, @@ -357,7 +357,7 @@ class PerconaClusterColdStartTest(PerconaClusterBaseTest): 'unknown', negate_match=True) - logging.debug("Wait till model is idle ...") + logging.info("Wait till model is idle ...") # XXX If a hook was executing on a unit when it was powered off # it comes back in an error state. try: @@ -366,7 +366,7 @@ class PerconaClusterColdStartTest(PerconaClusterBaseTest): self.resolve_update_status_errors() zaza.model.block_until_all_units_idle() - logging.debug("Wait for application states ...") + logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): try: zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") @@ -389,7 +389,7 @@ class PerconaClusterColdStartTest(PerconaClusterBaseTest): _non_leaders[0], "bootstrap-pxc", action_params={}) - logging.debug("Wait for application states ...") + logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") states = {"percona-cluster": { @@ -403,7 +403,7 @@ class PerconaClusterColdStartTest(PerconaClusterBaseTest): self.application, "notify-bootstrapped", action_params={}) - logging.debug("Wait for application states ...") + logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") test_config = lifecycle_utils.get_charm_config(fatal=False) @@ -532,12 +532,12 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): juju_utils.get_machine_uuids_for_application(self.application)) # Stop Nodes # Avoid hitting an update-status hook - logging.debug("Wait till model is idle ...") + logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() logging.info("Stopping instances: {}".format(_machines)) for uuid in _machines: self.nova_client.servers.stop(uuid) - logging.debug("Wait till all machines are shutoff ...") + logging.info("Wait till all machines are shutoff ...") for uuid in _machines: openstack_utils.resource_reaches_status(self.nova_client.servers, uuid, @@ -550,38 +550,37 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): for uuid in _machines: self.nova_client.servers.start(uuid) + logging.info( + "Wait till all {} units are in state 'unkown' ..." + .format(self.application)) for unit in zaza.model.get_units(self.application): zaza.model.block_until_unit_wl_status( unit.entity_id, 'unknown', negate_match=True) - logging.debug("Wait till model is idle ...") + logging.info("Wait till model is idle ...") try: zaza.model.block_until_all_units_idle() except zaza.model.UnitError: self.resolve_update_status_errors() zaza.model.block_until_all_units_idle() - logging.debug("Clear error hooks after reboot ...") + logging.info("Clear error hooks after reboot ...") for unit in zaza.model.get_units(self.application): try: zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") except zaza.model.UnitError: self.resolve_update_status_errors() zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") - logging.debug("Wait for application states blocked ...") - states = { - self.application: { - "workload-status": "blocked", - "workload-status-message": - "MySQL InnoDB Cluster not healthy: None"}, - "mysql-router": { - "workload-status": "blocked", - "workload-status-message": - "Failed to connect to MySQL"}} - zaza.model.wait_for_application_states(states=states) + logging.info( + "Wait till all {} units are in state 'blocked' ..." + .format(self.application)) + for unit in zaza.model.get_units(self.application): + zaza.model.block_until_unit_wl_status( + unit.entity_id, + 'blocked') logging.info("Execute reboot-cluster-from-complete-outage " "action after cold boot ...") @@ -592,15 +591,15 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): unit.entity_id, "reboot-cluster-from-complete-outage", action_params={}) - if "Success" in action.data["results"].get("outcome"): + if "Success" in action.data.get("results", {}).get("outcome", ""): break else: - logging.info(action.data["results"].get("output")) + logging.info(action.data.get("results", {}).get("output", "")) assert "Success" in action.data["results"]["outcome"], ( "Reboot cluster from complete outage action failed: {}" .format(action.data)) - logging.debug("Wait for application states ...") + logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") test_config = lifecycle_utils.get_charm_config(fatal=False) From 701e751398e4fe9077b5cc20f9f5d59875ad5e00 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 9 Jul 2020 10:15:56 +0000 Subject: [PATCH 599/898] Add check that guest is up On a recent mosci run the guest used for a restart test had a pending power task which stopped the restart test working properly. This PR adds an assertion that the guest is ready. --- zaza/openstack/charm_tests/masakari/tests.py | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/zaza/openstack/charm_tests/masakari/tests.py b/zaza/openstack/charm_tests/masakari/tests.py index d9490d2..6f27d17 100644 --- a/zaza/openstack/charm_tests/masakari/tests.py +++ b/zaza/openstack/charm_tests/masakari/tests.py @@ -134,6 +134,26 @@ class MasakariTest(test_utils.OpenStackBaseTest): vm_uuid, model_name=self.model_name) + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), + reraise=True, stop=tenacity.stop_after_attempt(5), + retry=tenacity.retry_if_exception_type(AssertionError)) + def wait_for_guest_ready(self, vm_name): + """Wait for the guest to be ready. + + :param vm_name: Name of guest to check. + :type vm_name: str + """ + guest_ready_attr_checks = [ + ('OS-EXT-STS:task_state', None), + ('status', 'ACTIVE'), + ('OS-EXT-STS:power_state', 1), + ('OS-EXT-STS:vm_state', 'active')] + guest = self.nova_client.servers.find(name=vm_name) + logging.info('Checking guest {} attributes'.format(vm_name)) + for (attr, required_state) in guest_ready_attr_checks: + logging.info('Checking {} is {}'.format(attr, required_state)) + assert getattr(guest, attr) == required_state + def test_instance_failover(self): """Test masakari managed guest migration.""" # Workaround for Bug #1874719 @@ -168,6 +188,7 @@ class MasakariTest(test_utils.OpenStackBaseTest): model_name=self.model_name) openstack_utils.enable_all_nova_services(self.nova_client) zaza.openstack.configure.masakari.enable_hosts() + self.wait_for_guest_ready(vm_name) def test_instance_restart_on_fail(self): """Test single guest crash and recovery.""" @@ -178,6 +199,7 @@ class MasakariTest(test_utils.OpenStackBaseTest): self.current_release)) vm_name = 'zaza-test-instance-failover' vm = self.ensure_guest(vm_name) + self.wait_for_guest_ready(vm_name) _, unit_name = self.get_guests_compute_info(vm_name) logging.info('{} is running on {}'.format(vm_name, unit_name)) guest_pid = self.get_guest_qemu_pid( From 0287664d9243efbc26994739f095f910bf12a047 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 9 Jul 2020 16:35:00 +0200 Subject: [PATCH 600/898] Fix paramiko.ssh_exception.BadAuthenticationType (#355) Fix paramiko.ssh_exception.BadAuthenticationType when SSHing to a new Ubuntu instance. Note that paramiko still has a few issues around authentication: https://github.com/paramiko/paramiko/pull/1106/files This paramiko PR also shows that password='' isn't the same as password=None --- unit_tests/utilities/test_zaza_utilities_openstack.py | 2 +- zaza/openstack/utilities/openstack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 549ed71..a3ad423 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -773,7 +773,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): privkey='myprivkey') paramiko_mock.connect.assert_called_once_with( '10.0.0.10', - password='', + password=None, pkey='akey', username='bob') diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 034cae3..bf7a041 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2401,7 +2401,7 @@ def ssh_command(username, ssh.connect(ip, username=username, password=password) else: key = paramiko.RSAKey.from_private_key(io.StringIO(privkey)) - ssh.connect(ip, username=username, password='', pkey=key) + ssh.connect(ip, username=username, password=None, pkey=key) logging.info("Running {} on {}".format(command, vm_name)) stdin, stdout, stderr = ssh.exec_command(command) if verify and callable(verify): From 4546e821c1bf1bf868c2a6c852a97df937b7b660 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 9 Jul 2020 09:27:25 -0700 Subject: [PATCH 601/898] Wait for resolving update-status hook errors The cold boot restart takes too long on update-status. Make the timeout longer. --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index bf61932..c9a767d 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -521,7 +521,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.resolve_units( application_name=self.application, erred_hook='update-status', - wait=True) + wait=True, timeout=180) def test_100_reboot_cluster_from_complete_outage(self): """Reboot cluster from complete outage. From f6e2a40166caefcdd9b8ce0bc9e2389791551a11 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 13 Jul 2020 14:33:35 +0000 Subject: [PATCH 602/898] Set application_name for ceph-radosgw tests Set application_name for ceph-radosgw tests so that the tests can be run without relying on getting the application name from the tests.yaml. --- zaza/openstack/charm_tests/ceph/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index ff79820..c25df2a 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -544,7 +544,7 @@ class CephRGWTest(test_utils.OpenStackBaseTest): @classmethod def setUpClass(cls): """Run class setup for running ceph low level tests.""" - super(CephRGWTest, cls).setUpClass() + super(CephRGWTest, cls).setUpClass(application_name='ceph-radosgw') @property def expected_apps(self): From 65cc6aa604912291dc9772db1b59473414f83d49 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 13 Jul 2020 14:36:59 +0000 Subject: [PATCH 603/898] Stop assuming RegionOne in ceph-radosgw tests Stop assuming RegionOne in ceph-radosgw tests and get the region from the app config instead. --- zaza/openstack/charm_tests/ceph/tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index ff79820..b4ae613 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -622,7 +622,9 @@ class CephRGWTest(test_utils.OpenStackBaseTest): 'multisite configuration') logging.info('Checking Swift REST API') keystone_session = zaza_openstack.get_overcloud_keystone_session() - region_name = 'RegionOne' + region_name = zaza_model.get_application_config( + self.application_name, + model_name=self.model_name)['region']['value'] swift_client = zaza_openstack.get_swift_session_client( keystone_session, region_name, From 9e395f69b76e6a70f053c2a35cf4741a7dfca7f0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 13 Jul 2020 14:42:43 +0000 Subject: [PATCH 604/898] Add ceph_ready config step Add a ceph_ready config step which just checks that the ceph units are in a 'active' 'Unit is Ready.*' state. This is useful when encryption at rest is being used and the tests.yaml assumes that the units are going to come up blocked. Once vault is ready the tests then need to wait for the ceph units to become unblocked. --- zaza/openstack/charm_tests/ceph/setup.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/setup.py b/zaza/openstack/charm_tests/ceph/setup.py index c53ff3c..c93fec8 100644 --- a/zaza/openstack/charm_tests/ceph/setup.py +++ b/zaza/openstack/charm_tests/ceph/setup.py @@ -14,7 +14,23 @@ """Setup for ceph-osd deployments.""" +import logging +import zaza.model + def basic_setup(): """Run basic setup for ceph-osd.""" pass + + +def ceph_ready(): + """Wait for ceph to be ready. + + Wait for ceph to be ready. This is useful if the target_deploy_status in + the tests.yaml is expecting ceph to be in a blocked state. After ceph + has been unblocked the deploy may need to wait to be ceph to be ready. + """ + logging.info("Waiting for ceph units to settle") + zaza.model.wait_for_application_states() + zaza.model.block_until_all_units_idle() + logging.info("Ceph units settled") From c4176881d00b1191b290f882e245f9ed1b4f4e31 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 13 Jul 2020 14:56:04 +0000 Subject: [PATCH 605/898] Fix spelling typo --- zaza/openstack/charm_tests/ceph/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/setup.py b/zaza/openstack/charm_tests/ceph/setup.py index c93fec8..87f213d 100644 --- a/zaza/openstack/charm_tests/ceph/setup.py +++ b/zaza/openstack/charm_tests/ceph/setup.py @@ -28,7 +28,7 @@ def ceph_ready(): Wait for ceph to be ready. This is useful if the target_deploy_status in the tests.yaml is expecting ceph to be in a blocked state. After ceph - has been unblocked the deploy may need to wait to be ceph to be ready. + has been unblocked the deploy may need to wait for ceph to be ready. """ logging.info("Waiting for ceph units to settle") zaza.model.wait_for_application_states() From e409b68639f75bc5b333b6d4074ae92602bc4216 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 14 Jul 2020 12:39:58 +0000 Subject: [PATCH 606/898] Add Groovy Victoria support --- zaza/openstack/utilities/os_versions.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index b2e430e..648a73b 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -35,6 +35,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('disco', 'stein'), ('eoan', 'train'), ('focal', 'ussuri'), + ('groovy', 'victoria'), ]) @@ -57,6 +58,7 @@ OPENSTACK_CODENAMES = OrderedDict([ ('2019.1', 'stein'), ('2019.2', 'train'), ('2020.1', 'ussuri'), + ('2020.2', 'victoria'), ]) OPENSTACK_RELEASES_PAIRS = [ @@ -66,7 +68,8 @@ OPENSTACK_RELEASES_PAIRS = [ 'xenial_pike', 'artful_pike', 'xenial_queens', 'bionic_queens', 'bionic_rocky', 'cosmic_rocky', 'bionic_stein', 'disco_stein', 'bionic_train', - 'eoan_train', 'bionic_ussuri', 'focal_ussuri'] + 'eoan_train', 'bionic_ussuri', 'focal_ussuri', + 'focal-victoria', 'groovy-victoria'] # The ugly duckling - must list releases oldest to newest SWIFT_CODENAMES = OrderedDict([ @@ -106,6 +109,8 @@ SWIFT_CODENAMES = OrderedDict([ ['2.22.0']), ('ussuri', ['2.24.0']), + ('victoria', + ['2.25.0']), ]) # >= Liberty version->codename mapping @@ -121,6 +126,7 @@ PACKAGE_CODENAMES = { ('19', 'stein'), ('20', 'train'), ('21', 'ussuri'), + ('22', 'victoria'), ]), 'neutron-common': OrderedDict([ ('7', 'liberty'), @@ -133,6 +139,7 @@ PACKAGE_CODENAMES = { ('14', 'stein'), ('15', 'train'), ('16', 'ussuri'), + ('17', 'victoria'), ]), 'cinder-common': OrderedDict([ ('7', 'liberty'), @@ -145,6 +152,7 @@ PACKAGE_CODENAMES = { ('14', 'stein'), ('15', 'train'), ('16', 'ussuri'), + ('17', 'victoria'), ]), 'keystone': OrderedDict([ ('8', 'liberty'), @@ -157,6 +165,7 @@ PACKAGE_CODENAMES = { ('15', 'stein'), ('16', 'train'), ('17', 'ussuri'), + ('18', 'victoria'), ]), 'horizon-common': OrderedDict([ ('8', 'liberty'), @@ -169,6 +178,7 @@ PACKAGE_CODENAMES = { ('15', 'stein'), ('16', 'train'), ('17', 'ussuri'), + ('19', 'victoria'), ]), 'ceilometer-common': OrderedDict([ ('5', 'liberty'), @@ -181,6 +191,7 @@ PACKAGE_CODENAMES = { ('12', 'stein'), ('13', 'train'), ('14', 'ussuri'), + ('15', 'victoria'), ]), 'heat-common': OrderedDict([ ('5', 'liberty'), @@ -193,6 +204,7 @@ PACKAGE_CODENAMES = { ('12', 'stein'), ('13', 'train'), ('14', 'ussuri'), + ('15', 'victoria'), ]), 'glance-common': OrderedDict([ ('11', 'liberty'), @@ -205,6 +217,7 @@ PACKAGE_CODENAMES = { ('18', 'stein'), ('19', 'train'), ('20', 'ussuri'), + ('21', 'victoria'), ]), 'openstack-dashboard': OrderedDict([ ('8', 'liberty'), @@ -216,7 +229,8 @@ PACKAGE_CODENAMES = { ('14', 'rocky'), ('15', 'stein'), ('16', 'train'), - ('17', 'ussuri'), + ('18', 'ussuri'), + ('19', 'victoria'), ]), 'designate-common': OrderedDict([ ('1', 'liberty'), @@ -229,6 +243,7 @@ PACKAGE_CODENAMES = { ('8', 'stein'), ('9', 'train'), ('10', 'ussuri'), + ('11', 'victoria'), ]), 'ovn-common': OrderedDict([ ('2', 'train'), From 2d3eaa6e8478679c7bc62c8eef6b18ab96d7ed30 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 14 Jul 2020 12:42:29 +0000 Subject: [PATCH 607/898] Fix typo --- zaza/openstack/utilities/os_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 648a73b..fcf15df 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -69,7 +69,7 @@ OPENSTACK_RELEASES_PAIRS = [ 'bionic_queens', 'bionic_rocky', 'cosmic_rocky', 'bionic_stein', 'disco_stein', 'bionic_train', 'eoan_train', 'bionic_ussuri', 'focal_ussuri', - 'focal-victoria', 'groovy-victoria'] + 'focal_victoria', 'groovy_victoria'] # The ugly duckling - must list releases oldest to newest SWIFT_CODENAMES = OrderedDict([ From e729c64956ce6103fd6dca790139474e39278723 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 14 Jul 2020 12:53:07 +0000 Subject: [PATCH 608/898] Fixed existing codename errors spotted by coreycb --- zaza/openstack/utilities/os_versions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index fcf15df..b46fc21 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -108,7 +108,7 @@ SWIFT_CODENAMES = OrderedDict([ ('train', ['2.22.0']), ('ussuri', - ['2.24.0']), + ['2.24.0', '2.25.0']), ('victoria', ['2.25.0']), ]) @@ -177,7 +177,7 @@ PACKAGE_CODENAMES = { ('14', 'rocky'), ('15', 'stein'), ('16', 'train'), - ('17', 'ussuri'), + ('18', 'ussuri'), ('19', 'victoria'), ]), 'ceilometer-common': OrderedDict([ From 9896c8d9c54be397deef20ea104043a99d6e2ca2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 14 Jul 2020 14:07:59 +0000 Subject: [PATCH 609/898] Skip test_003_test_overide_is_observed Skip test_003_test_overide_is_observed until Bug #1880959 is fix released. --- zaza/openstack/charm_tests/policyd/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 88d6699..67b9909 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -401,6 +401,10 @@ class BasePolicydSpecialization(PolicydTest, def test_003_test_overide_is_observed(self): """Test that the override is observed by the underlying service.""" + if (openstack_utils.get_os_release() < + openstack_utils.get_os_release('groovy_victoria')): + raise unittest.SkipTest( + "Test skipped until Bug #1880959 is fix released") if self._test_name is None: logging.info("Doing policyd override for {}" .format(self._service_name)) From a670e274fafe032525bb8fd342262b87a47ae797 Mon Sep 17 00:00:00 2001 From: Andrew McLeod Date: Wed, 15 Jul 2020 16:52:27 +0200 Subject: [PATCH 610/898] add image arch and properties specification --- zaza/openstack/charm_tests/glance/setup.py | 26 +++++++++++++++++----- zaza/openstack/utilities/openstack.py | 10 ++++++++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index 8c7bd3e..ab0a3b3 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -16,6 +16,7 @@ import logging import zaza.openstack.utilities.openstack as openstack_utils +import zaza.utilities.deployment_env as deployment_env CIRROS_IMAGE_NAME = "cirros" CIRROS_ALT_IMAGE_NAME = "cirros_alt" @@ -31,7 +32,8 @@ def basic_setup(): """ -def add_image(image_url, glance_client=None, image_name=None, tags=[]): +def add_image(image_url, glance_client=None, image_name=None, tags=[], + properties=None): """Retrieve image from ``image_url`` and add it to glance. :param image_url: Retrievable URL with image data @@ -42,6 +44,8 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[]): :type image_name: str :param tags: List of tags to add to image :type tags: list of str + :param properties: Properties to add to image + :type properties: dict """ if not glance_client: keystone_session = openstack_utils.get_overcloud_keystone_session() @@ -60,7 +64,8 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[]): glance_client, image_url, image_name, - tags=tags) + tags=tags, + properties=properties) def add_cirros_image(glance_client=None, image_name=None): @@ -90,7 +95,8 @@ def add_cirros_alt_image(glance_client=None, image_name=None): add_cirros_image(glance_client, image_name) -def add_lts_image(glance_client=None, image_name=None, release=None): +def add_lts_image(glance_client=None, image_name=None, release=None, + properties=None): """Add an Ubuntu LTS image to the current deployment. :param glance: Authenticated glanceclient @@ -99,12 +105,22 @@ def add_lts_image(glance_client=None, image_name=None, release=None): :type image_name: str :param release: Name of ubuntu release. :type release: str + :param properties: Custom image properties + :type properties: dict """ + deploy_ctxt = deployment_env.get_deployment_context() + image_arch = deploy_ctxt.get('TEST_IMAGE_ARCH', 'amd64') + arch_image_properties = { + 'arm64': {'hw_firmware_type': 'uefi'}, + 'ppc64el': {'architecture': 'ppc64'}} + properties = properties or arch_image_properties.get(image_arch) + logging.info("Image architecture set to {}".format(image_arch)) image_name = image_name or LTS_IMAGE_NAME release = release or LTS_RELEASE image_url = openstack_utils.find_ubuntu_image( release=release, - arch='amd64') + arch=image_arch) add_image(image_url, glance_client=glance_client, - image_name=image_name) + image_name=image_name, + properties=properties) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index bf7a041..76c80d2 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2036,7 +2036,8 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', return image -def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[]): +def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], + properties=None): """Download the image and upload it to glance. Download an image from image_url and upload it to glance labelling @@ -2053,6 +2054,8 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[]): :type image_cache_dir: Option[str, None] :param tags: Tags to add to image :type tags: list of str + :param properties: Properties and values to add to image + :type properties: dict :returns: glance image pointer :rtype: glanceclient.common.utils.RequestIdProxy """ @@ -2074,6 +2077,11 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[]): logging.debug( 'applying tag to image: glance.image_tags.update({}, {}) = {}' .format(image.id, tags, result)) + + logging.info("Setting image properties: {}".format(properties)) + if properties: + result = glance.images.update(image.id, **properties) + return image From ce6cfe68b31ed3266964b8f118fb95d48e5ac12d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 15 Jul 2020 20:07:17 +0000 Subject: [PATCH 611/898] Add defaut app name for vrrp tests Set a default application_name for vrrp tests so they can be called by charms other than the neutron-api gate tests. --- zaza/openstack/charm_tests/neutron/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 620767e..ec6998d 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -613,7 +613,8 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): @classmethod def setUpClass(cls): """Run class setup for running Neutron API Networking tests.""" - super(NeutronNetworkingBase, cls).setUpClass() + super(NeutronNetworkingBase, cls).setUpClass( + application_name='neutron-api') cls.neutron_client = ( openstack_utils.get_neutron_session_client(cls.keystone_session)) From 5579f2f110b8b08097c6a7b189669886a10cb378 Mon Sep 17 00:00:00 2001 From: Alvaro Uria Date: Mon, 20 Jul 2020 19:24:33 +0200 Subject: [PATCH 612/898] HaclusterTest: support scaleback test on bionic --- zaza/openstack/charm_tests/hacluster/tests.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 199d915..3ae13d1 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -21,6 +21,7 @@ import os import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.hacluster +import zaza.utilities.juju as juju_utils class HaclusterTest(test_utils.OpenStackBaseTest): @@ -69,3 +70,71 @@ class HaclusterTest(test_utils.OpenStackBaseTest): self._toggle_maintenance_and_wait('true') self._toggle_maintenance_and_wait('false') + + def test_930_scaleback_bionic(self): + """Remove a unit, recalculate quorum and add a new one.""" + principle_app = 'keystone' + principle_units = zaza.model.get_status().applications[ + principle_app]['units'] + self.assertEqual(len(principle_units), 3) + doomed_principle = sorted(principle_units.keys())[0] + series = juju_utils.get_machine_series( + principle_units[doomed_principle].machine) + if series != 'bionic': + logging.debug("noop - only run test in bionic") + logging.info('SKIP') + return + + doomed_unit = juju_utils.get_subordinate_units( + [doomed_principle], charm_name='hac')[0] + + logging.info('Pausing unit {}'.format(doomed_unit)) + zaza.model.run_action( + doomed_unit, + 'pause', + raise_on_failure=True) + logging.info('OK') + + logging.info('Resuming unit {}'.format(doomed_unit)) + zaza.model.run_action( + doomed_unit, + 'resume', + raise_on_failure=True) + logging.info('OK') + + logging.info('Removing {}'.format(doomed_principle)) + zaza.model.destroy_unit( + principle_app, + doomed_principle, + wait_disappear=True) + logging.info('OK') + + logging.info('Updating corosync ring') + zaza.model.run_action_on_leader( + self.application_name, + 'update-ring', + action_params={'i-really-mean-it': True}, + raise_on_failure=True) + + _states = { + self.application_name: { + "workload-status": "blocked", + "workload-status-message": + "Insufficient peer units for ha cluster (require 3)" + }, + 'keystone': { + "workload-status": "blocked", + "workload-status-message": "Database not initialised", + }, + } + zaza.model.wait_for_application_states(states=_states) + zaza.model.block_until_all_units_idle() + logging.info('OK') + + logging.info('Adding a hacluster unit') + zaza.model.add_unit(principle_app, wait_appear=True) + _states = {self.application_name: { + "workload-status": "active", + "workload-status-message": "Unit is ready and clustered"}} + zaza.model.wait_for_application_states(states=_states) + logging.debug('OK') From 158e8ce37f9d9240a396136e91c0387d840a2891 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 22 Jul 2020 10:40:29 +0200 Subject: [PATCH 613/898] octavia/policyd: Change target for policyd test List available provider drivers as policyd test, this should be more light weight than creating and retrieving a load-balancer. Fixes #349 --- zaza/openstack/charm_tests/policyd/tests.py | 87 +-------------------- 1 file changed, 3 insertions(+), 84 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 67b9909..e69aaff 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -659,7 +659,7 @@ class HeatTests(BasePolicydSpecialization): class OctaviaTests(BasePolicydSpecialization): """Test the policyd override using the octavia client.""" - _rule = {'rule.yaml': "{'os_load-balancer_api:loadbalancer:get_one': '!'}"} + _rule = {'rule.yaml': "{'os_load-balancer_api:provider:get_all': '!'}"} @classmethod def setUpClass(cls, application_name=None): @@ -667,89 +667,8 @@ class OctaviaTests(BasePolicydSpecialization): super(OctaviaTests, cls).setUpClass(application_name="octavia") cls.application_name = "octavia" - def setup_for_attempt_operation(self, ip): - """Create a loadbalancer. - - This is necessary so that the attempt is to show the load-balancer and - this is an operator that the policy can stop. Unfortunately, octavia, - whilst it has a policy for just listing load-balancers, unfortunately, - it doesn't work; whereas showing the load-balancer can be stopped. - - NB this only works if the setup phase of the octavia tests have been - completed. - - :param ip: the ip of for keystone. - :type ip: str - """ - logging.info("Setting up loadbalancer.") - auth = openstack_utils.get_overcloud_auth(address=ip) - sess = openstack_utils.get_keystone_session(auth) - - octavia_client = openstack_utils.get_octavia_session_client(sess) - neutron_client = openstack_utils.get_neutron_session_client(sess) - - if openstack_utils.dvr_enabled(): - network_name = 'private_lb_fip_network' - else: - network_name = 'private' - resp = neutron_client.list_networks(name=network_name) - - vip_subnet_id = resp['networks'][0]['subnets'][0] - - res = octavia_client.load_balancer_create( - json={ - 'loadbalancer': { - 'description': 'Created by Zaza', - 'admin_state_up': True, - 'vip_subnet_id': vip_subnet_id, - 'name': 'zaza-lb-0', - }}) - self.lb_id = res['loadbalancer']['id'] - # now wait for it to get to the active state - - @tenacity.retry(wait=tenacity.wait_fixed(1), - reraise=True, stop=tenacity.stop_after_delay(900)) - def wait_for_lb_resource(client, resource_id): - resp = client.load_balancer_show(resource_id) - logging.info(resp['provisioning_status']) - assert resp['provisioning_status'] == 'ACTIVE', ( - 'load balancer resource has not reached ' - 'expected provisioning status: {}' - .format(resp)) - return resp - - logging.info('Awaiting loadbalancer to reach provisioning_status ' - '"ACTIVE"') - resp = wait_for_lb_resource(octavia_client, self.lb_id) - logging.info(resp) - logging.info("Setup loadbalancer complete.") - - def cleanup_for_attempt_operation(self, ip): - """Remove the loadbalancer. - - :param ip: the ip of for keystone. - :type ip: str - """ - logging.info("Deleting loadbalancer {}.".format(self.lb_id)) - auth = openstack_utils.get_overcloud_auth(address=ip) - sess = openstack_utils.get_keystone_session(auth) - - octavia_client = openstack_utils.get_octavia_session_client(sess) - octavia_client.load_balancer_delete(self.lb_id) - logging.info("Deleting loadbalancer in progress ...") - - @tenacity.retry(wait=tenacity.wait_fixed(1), - reraise=True, stop=tenacity.stop_after_delay(900)) - def wait_til_deleted(client, lb_id): - lb_list = client.load_balancer_list() - ids = [lb['id'] for lb in lb_list['loadbalancers']] - assert lb_id not in ids, 'load balancer still deleting' - - wait_til_deleted(octavia_client, self.lb_id) - logging.info("Deleted loadbalancer.") - def get_client_and_attempt_operation(self, ip): - """Attempt to show the loadbalancer as a policyd override. + """Attempt to list available provider drivers. This operation should pass normally, and fail when the rule has been overriden (see the `rule` class variable. @@ -761,6 +680,6 @@ class OctaviaTests(BasePolicydSpecialization): octavia_client = openstack_utils.get_octavia_session_client( self.get_keystone_session_admin_user(ip)) try: - octavia_client.load_balancer_show(self.lb_id) + octavia_client.provider_list() except octaviaclient.OctaviaClientException: raise PolicydOperationFailedException() From a1eb37c36562766ca69ba6d320e563bc09cac35b Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 23 Jul 2020 09:49:48 +0200 Subject: [PATCH 614/898] Make neutron/arista tests more robust When creating an Arista-backed Neutron network, the API call from Neutron to Arista sometimes spuriously fails. This is not fatal and Neutron simply retries a few seconds later. The tests now account for that. --- .../charm_tests/neutron_arista/tests.py | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron_arista/tests.py b/zaza/openstack/charm_tests/neutron_arista/tests.py index 3f78bf7..ba29bd6 100644 --- a/zaza/openstack/charm_tests/neutron_arista/tests.py +++ b/zaza/openstack/charm_tests/neutron_arista/tests.py @@ -39,15 +39,36 @@ class NeutronCreateAristaNetworkTest(neutron_tests.NeutronCreateNetworkTest): cls.neutron_client.list_networks() def _assert_test_network_exists_and_return_id(self): - actual_network_names = arista_utils.query_fixture_networks( - arista_utils.fixture_ip_addr()) - self.assertEqual(actual_network_names, [self._TEST_NET_NAME]) + logging.info('Checking that the test network exists on the Arista ' + 'test fixture...') + + # Sometimes the API call from Neutron to Arista fails and Neutron + # retries a couple of seconds later, which is why the newly created + # test network may not be immediately visible on Arista's API. + for attempt in tenacity.Retrying( + wait=tenacity.wait_fixed(10), # seconds + stop=tenacity.stop_after_attempt(3), + reraise=True): + with attempt: + actual_network_names = arista_utils.query_fixture_networks( + arista_utils.fixture_ip_addr()) + self.assertEqual(actual_network_names, [self._TEST_NET_NAME]) + return super(NeutronCreateAristaNetworkTest, self)._assert_test_network_exists_and_return_id() def _assert_test_network_doesnt_exist(self): - actual_network_names = arista_utils.query_fixture_networks( - arista_utils.fixture_ip_addr()) - self.assertEqual(actual_network_names, []) + logging.info("Checking that the test network doesn't exist on the " + "Arista test fixture...") + + for attempt in tenacity.Retrying( + wait=tenacity.wait_fixed(10), # seconds + stop=tenacity.stop_after_attempt(3), + reraise=True): + with attempt: + actual_network_names = arista_utils.query_fixture_networks( + arista_utils.fixture_ip_addr()) + self.assertEqual(actual_network_names, []) + super(NeutronCreateAristaNetworkTest, self)._assert_test_network_doesnt_exist() From 59d68e49a5830741a602aee0a6e7fa73d8eda776 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Thu, 23 Jul 2020 14:07:21 +0100 Subject: [PATCH 615/898] Ensure that the tests directory exists for the local cacert (#373) A recent change to allow multiple zazas to run at the same time fixed the cacert file into a local 'tests/' directory. Unfortunately, that doesn't exist for every environment where zaza runs (e.g. mojo) and so this patch ensures that the path exists prior to trying to download into it. --- zaza/openstack/utilities/openstack.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 4424cea..14c708c 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1737,6 +1737,15 @@ def get_overcloud_auth(address=None, model_name=None): } if tls_rid: unit = model.get_first_unit_name('keystone', model_name=model_name) + + # ensure that the path to put the local cacert in actually exists. The + # assumption that 'tests/' exists for, say, mojo is false. + # Needed due to: + # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 + _dir = os.path.dirname(KEYSTONE_LOCAL_CACERT) + if not os.path.exists(_dir): + os.makedirs(_dir) + model.scp_from_unit( unit, KEYSTONE_REMOTE_CACERT, From 6bbb4fd4e34102b0bd924b00461abfd0abaa745f Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Fri, 24 Jul 2020 10:13:50 -0500 Subject: [PATCH 616/898] add iscsi-connector files and setup --- .../charm_tests/iscsi-connector/__init__.py | 15 +++++ .../charm_tests/iscsi-connector/setup.py | 56 +++++++++++++++++++ .../charm_tests/iscsi-connector/tests.py | 15 +++++ 3 files changed, 86 insertions(+) create mode 100644 zaza/openstack/charm_tests/iscsi-connector/__init__.py create mode 100644 zaza/openstack/charm_tests/iscsi-connector/setup.py create mode 100644 zaza/openstack/charm_tests/iscsi-connector/tests.py diff --git a/zaza/openstack/charm_tests/iscsi-connector/__init__.py b/zaza/openstack/charm_tests/iscsi-connector/__init__.py new file mode 100644 index 0000000..c0cf179 --- /dev/null +++ b/zaza/openstack/charm_tests/iscsi-connector/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing iscsi-connector charm.""" \ No newline at end of file diff --git a/zaza/openstack/charm_tests/iscsi-connector/setup.py b/zaza/openstack/charm_tests/iscsi-connector/setup.py new file mode 100644 index 0000000..f18a4ad --- /dev/null +++ b/zaza/openstack/charm_tests/iscsi-connector/setup.py @@ -0,0 +1,56 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for setting up iscsi-connector tests.""" + +# """ +# steps for testing without password: +# - deploy ubuntu for storage +# - deploy ubuntu for initiator +# configure that unit: +# - install tgt -y +# - configure iscsi target +# - need to know the initiator ip +# - restart tgt +# - configure ubuntu initiator with init dict, target, port +# """ + +import zaza.model +import zaza.openstack.utilities.openstack as openstack_utils + + +def basic_target_setup(): + """Run basic setup for iscsi guest.""" + unit = zaza.model.get_units('ubuntu-target')[0] + setup_cmds = [ + "apt install --yes tgt", + "systemctl start tgt",] + for cmd in setup_cmds: + zaza.model.run_on_unit( + unit.entity_id, + cmd) + +def configure_iscsi_target(): + lun = 'iqn.2020-07.canonical.com:lun1' + backing_store = 'dev/vdb' + initiator_address = zaza.model.get_app_ips('ubuntu')[0] + write_file = ( + """echo -e '\n\tbacking-store {}\n\tinitiator-address {}\n' | """ + """sudo tee /etc/tgt/conf.d/iscsi.conf""".format(lun, backing_store,initiator_address) + ) + zaza.model.run_on_unit('ubuntu-target/0', write_file) + # Restart tgt to load new config + restart_tgt = "systemctl restart tgt" + zaza.model.run_on_unit('ubuntu-target/0', restart_tgt) + diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py new file mode 100644 index 0000000..c30f38c --- /dev/null +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate iscsi-connector testing.""" \ No newline at end of file From fb8de5cc81221ecc2e09a628540616da627c65ce Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Fri, 24 Jul 2020 11:09:22 -0500 Subject: [PATCH 617/898] add open-port cmd for iscsi target --- zaza/openstack/charm_tests/iscsi-connector/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/iscsi-connector/setup.py b/zaza/openstack/charm_tests/iscsi-connector/setup.py index f18a4ad..f453ed4 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/setup.py +++ b/zaza/openstack/charm_tests/iscsi-connector/setup.py @@ -35,7 +35,8 @@ def basic_target_setup(): unit = zaza.model.get_units('ubuntu-target')[0] setup_cmds = [ "apt install --yes tgt", - "systemctl start tgt",] + "systemctl start tgt", + "open-port 3260"] for cmd in setup_cmds: zaza.model.run_on_unit( unit.entity_id, From 277272547f06b04d06ea94149f79d0b47f44a3ed Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Fri, 24 Jul 2020 14:14:19 -0500 Subject: [PATCH 618/898] add tests iscsi-connector --- .../charm_tests/iscsi-connector/setup.py | 4 +++ .../charm_tests/iscsi-connector/tests.py | 32 ++++++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/iscsi-connector/setup.py b/zaza/openstack/charm_tests/iscsi-connector/setup.py index f453ed4..d016a36 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/setup.py +++ b/zaza/openstack/charm_tests/iscsi-connector/setup.py @@ -26,12 +26,15 @@ # - configure ubuntu initiator with init dict, target, port # """ +import logging + import zaza.model import zaza.openstack.utilities.openstack as openstack_utils def basic_target_setup(): """Run basic setup for iscsi guest.""" + logging.info('Installing package tgt on ubuntu-target') unit = zaza.model.get_units('ubuntu-target')[0] setup_cmds = [ "apt install --yes tgt", @@ -50,6 +53,7 @@ def configure_iscsi_target(): """echo -e '\n\tbacking-store {}\n\tinitiator-address {}\n' | """ """sudo tee /etc/tgt/conf.d/iscsi.conf""".format(lun, backing_store,initiator_address) ) + logging.info('Writing target iscsi.conf') zaza.model.run_on_unit('ubuntu-target/0', write_file) # Restart tgt to load new config restart_tgt = "systemctl restart tgt" diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py index c30f38c..5c2babf 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -12,4 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Encapsulate iscsi-connector testing.""" \ No newline at end of file +"""Encapsulate iscsi-connector testing.""" + +import logging +import tempfile + +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.generic as generic_utils + + +class ISCSIConnectorTest(test_utils.BaseCharmTest): + """Class for iscsi-connector tests.""" + + IQN = 'iqn.2020-07.canonical.com:lun1' + + def configure_iscsi_connector(self): + unit_fqdn = self.get_unit_full_hostname('ubuntu/0') + target_ip = zaza.model.get_app_ips('ubuntu-target')[0] + conf = { + 'initiator-dictionary': '{"{unit_fqdn}":"{IQN}"}', + 'target': target_ip, + 'port': '3260', + } + zaza.model.set_application_config('iscsi-connector', conf) + + def get_unit_full_hostname(self, unit_name): + """Retrieve the full hostname of a unit.""" + for unit in zaza.model.get_units(unit_name): + result = zaza.model.run_on_unit(unit.entity_id, 'hostname -f') + hostname = result['Stdout'].rstrip() + return hostname From bad9b4c768d9669f6c14bda6a3fc6bb48992de0d Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Fri, 24 Jul 2020 14:14:19 -0500 Subject: [PATCH 619/898] add tests iscsi-connector --- .../charm_tests/iscsi-connector/setup.py | 4 +++ .../charm_tests/iscsi-connector/tests.py | 35 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/iscsi-connector/setup.py b/zaza/openstack/charm_tests/iscsi-connector/setup.py index f453ed4..d016a36 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/setup.py +++ b/zaza/openstack/charm_tests/iscsi-connector/setup.py @@ -26,12 +26,15 @@ # - configure ubuntu initiator with init dict, target, port # """ +import logging + import zaza.model import zaza.openstack.utilities.openstack as openstack_utils def basic_target_setup(): """Run basic setup for iscsi guest.""" + logging.info('Installing package tgt on ubuntu-target') unit = zaza.model.get_units('ubuntu-target')[0] setup_cmds = [ "apt install --yes tgt", @@ -50,6 +53,7 @@ def configure_iscsi_target(): """echo -e '\n\tbacking-store {}\n\tinitiator-address {}\n' | """ """sudo tee /etc/tgt/conf.d/iscsi.conf""".format(lun, backing_store,initiator_address) ) + logging.info('Writing target iscsi.conf') zaza.model.run_on_unit('ubuntu-target/0', write_file) # Restart tgt to load new config restart_tgt = "systemctl restart tgt" diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py index c30f38c..d7b67fc 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -12,4 +12,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Encapsulate iscsi-connector testing.""" \ No newline at end of file +"""Encapsulate iscsi-connector testing.""" + +import logging +import tempfile + +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.generic as generic_utils + + +class ISCSIConnectorTest(test_utils.BaseCharmTest): + """Class for iscsi-connector tests.""" + + IQN = 'iqn.2020-07.canonical.com:lun1' + + def configure_iscsi_connector(self): + unit_fqdn = self.get_unit_full_hostname('ubuntu/0') + target_ip = zaza.model.get_app_ips('ubuntu-target')[0] + conf = { + 'initiator-dictionary': '{"{unit_fqdn}":"{IQN}"}', + 'target': target_ip, + 'port': '3260', + } + zaza.model.set_application_config('iscsi-connector', conf) + + def get_unit_full_hostname(self, unit_name): + """Retrieve the full hostname of a unit.""" + for unit in zaza.model.get_units(unit_name): + result = zaza.model.run_on_unit(unit.entity_id, 'hostname -f') + hostname = result['Stdout'].rstrip() + return hostname + + def test_iscsi_connector(self): + self.configure_iscsi_connector() \ No newline at end of file From 5664251152e06597e78f20cd926fc0c830fd24db Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Fri, 24 Jul 2020 14:25:27 -0500 Subject: [PATCH 620/898] tests.py --- zaza/openstack/charm_tests/iscsi-connector/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py index f7b67ea..8f707eb 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -25,6 +25,11 @@ import zaza.openstack.utilities.generic as generic_utils class ISCSIConnectorTest(test_utils.BaseCharmTest): """Class for iscsi-connector tests.""" + @classmethod + def setUpClass(cls): + """Run class setup for running glance tests.""" + super(ISCSIConnectorTest, cls).setUpClass() + IQN = 'iqn.2020-07.canonical.com:lun1' def configure_iscsi_connector(self): From 4c57e73f73ab18d9d9fe85dd1bdf798a6b805ee8 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Fri, 24 Jul 2020 14:25:27 -0500 Subject: [PATCH 621/898] tests.py --- zaza/openstack/charm_tests/iscsi-connector/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py index f7b67ea..8f707eb 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -25,6 +25,11 @@ import zaza.openstack.utilities.generic as generic_utils class ISCSIConnectorTest(test_utils.BaseCharmTest): """Class for iscsi-connector tests.""" + @classmethod + def setUpClass(cls): + """Run class setup for running glance tests.""" + super(ISCSIConnectorTest, cls).setUpClass() + IQN = 'iqn.2020-07.canonical.com:lun1' def configure_iscsi_connector(self): From 411d9aeb50b2165feac7a43494521ed854ec3549 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Fri, 24 Jul 2020 15:14:12 -0500 Subject: [PATCH 622/898] fix hostname --- zaza/openstack/charm_tests/iscsi-connector/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py index 8f707eb..be5cb24 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -33,7 +33,7 @@ class ISCSIConnectorTest(test_utils.BaseCharmTest): IQN = 'iqn.2020-07.canonical.com:lun1' def configure_iscsi_connector(self): - unit_fqdn = self.get_unit_full_hostname('ubuntu/0') + unit_fqdn = self.get_unit_full_hostname('ubuntu') target_ip = zaza.model.get_app_ips('ubuntu-target')[0] conf = { 'initiator-dictionary': '{"{unit_fqdn}":"{IQN}"}', From 738664e8bab6cbacb51deedfbeb1bfbe7ffea429 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Fri, 24 Jul 2020 15:53:00 -0500 Subject: [PATCH 623/898] assert iscsi session --- .../openstack/charm_tests/iscsi-connector/tests.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py index be5cb24..33a85dd 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -14,6 +14,7 @@ """Encapsulate iscsi-connector testing.""" +import json import logging import tempfile @@ -29,14 +30,14 @@ class ISCSIConnectorTest(test_utils.BaseCharmTest): def setUpClass(cls): """Run class setup for running glance tests.""" super(ISCSIConnectorTest, cls).setUpClass() - - IQN = 'iqn.2020-07.canonical.com:lun1' def configure_iscsi_connector(self): + iqn = 'iqn.2020-07.canonical.com:lun1' unit_fqdn = self.get_unit_full_hostname('ubuntu') target_ip = zaza.model.get_app_ips('ubuntu-target')[0] + initiator_dictionary = json.dumps({unit_fqdn:iqn}) conf = { - 'initiator-dictionary': '{"{unit_fqdn}":"{IQN}"}', + 'initiator-dictionary': initiator_dictionary, 'target': target_ip, 'port': '3260', } @@ -51,3 +52,10 @@ class ISCSIConnectorTest(test_utils.BaseCharmTest): def test_iscsi_connector(self): self.configure_iscsi_connector() + logging.info('Wait for idle/ready status...') + zaza_model.wait_for_application_states() + + def test_validate_iscsi_session(self): + unit = zaza.model.get_units('ubuntu')[0] + run = zaza.model.run_on_unit(unit.entity_id, 'iscsiadm -m session') + assert run['Stdout'] != "iscsiadm: No active sessions." From ab91b94d779a260cf572e34114ea64448fc3cd65 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Mon, 27 Jul 2020 10:25:21 -0500 Subject: [PATCH 624/898] add logs to iscsi test --- zaza/openstack/charm_tests/iscsi-connector/tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py index 33a85dd..bbd4e87 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -53,9 +53,11 @@ class ISCSIConnectorTest(test_utils.BaseCharmTest): def test_iscsi_connector(self): self.configure_iscsi_connector() logging.info('Wait for idle/ready status...') - zaza_model.wait_for_application_states() + zaza.model.wait_for_application_states() def test_validate_iscsi_session(self): unit = zaza.model.get_units('ubuntu')[0] + logging.info('Checking if iscsi session is active.') run = zaza.model.run_on_unit(unit.entity_id, 'iscsiadm -m session') - assert run['Stdout'] != "iscsiadm: No active sessions." + logging.info('iscsiadm -m session: Stdout: {}, Stderr: {}, Code: {}'.format(run['Stdout'], run['Stderr'], run['Code'])) + assert run['Code'] == '0' From e997647870e6f194eeaf9a29da05af862de48934 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Mon, 27 Jul 2020 15:47:21 -0500 Subject: [PATCH 625/898] Add CHAP auth to tests --- zaza/openstack/charm_tests/iscsi-connector/setup.py | 9 +++++++-- zaza/openstack/charm_tests/iscsi-connector/tests.py | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/iscsi-connector/setup.py b/zaza/openstack/charm_tests/iscsi-connector/setup.py index d016a36..dbfc6f0 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/setup.py +++ b/zaza/openstack/charm_tests/iscsi-connector/setup.py @@ -49,9 +49,14 @@ def configure_iscsi_target(): lun = 'iqn.2020-07.canonical.com:lun1' backing_store = 'dev/vdb' initiator_address = zaza.model.get_app_ips('ubuntu')[0] + username = 'iscsi-user' + password = 'password123' + username_in = 'iscsi-target' + password_in = 'secretpass' write_file = ( - """echo -e '\n\tbacking-store {}\n\tinitiator-address {}\n' | """ - """sudo tee /etc/tgt/conf.d/iscsi.conf""".format(lun, backing_store,initiator_address) + """echo -e '\n\tbacking-store {}\n\tinitiator-address {}\n\tincominguser {} {}\n\t""" + """outgoinguser {} {}' | sudo tee /etc/tgt/conf.d/iscsi.conf""".format(lun, backing_store, + initiator_address, username, password, username_in, password_in) ) logging.info('Writing target iscsi.conf') zaza.model.run_on_unit('ubuntu-target/0', write_file) diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py index bbd4e87..0507b81 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -40,6 +40,11 @@ class ISCSIConnectorTest(test_utils.BaseCharmTest): 'initiator-dictionary': initiator_dictionary, 'target': target_ip, 'port': '3260', + 'iscsi-node-session-auth-authmethod': 'CHAP', + 'iscsi-node-session-auth-username': 'iscsi-user', + 'iscsi-node-session-auth-password': 'password123', + 'iscsi-node-session-auth-username_in': 'iscsi-target', + 'iscsi-node-session-auth-password_in': 'secretpass', } zaza.model.set_application_config('iscsi-connector', conf) From 127ac7c87c9f3592292dddd88db4dd2e1d2c93bf Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Mon, 27 Jul 2020 15:59:06 -0500 Subject: [PATCH 626/898] fix typo --- zaza/openstack/charm_tests/iscsi-connector/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py index 0507b81..89c1223 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -43,8 +43,8 @@ class ISCSIConnectorTest(test_utils.BaseCharmTest): 'iscsi-node-session-auth-authmethod': 'CHAP', 'iscsi-node-session-auth-username': 'iscsi-user', 'iscsi-node-session-auth-password': 'password123', - 'iscsi-node-session-auth-username_in': 'iscsi-target', - 'iscsi-node-session-auth-password_in': 'secretpass', + 'iscsi-node-session-auth-username-in': 'iscsi-target', + 'iscsi-node-session-auth-password-in': 'secretpass', } zaza.model.set_application_config('iscsi-connector', conf) From 7c9e2edbd1ea37b5b0f96d1e292327a1a9e35175 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Mon, 27 Jul 2020 16:23:36 -0500 Subject: [PATCH 627/898] add missing newline --- zaza/openstack/charm_tests/iscsi-connector/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/iscsi-connector/setup.py b/zaza/openstack/charm_tests/iscsi-connector/setup.py index dbfc6f0..ff58c4f 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/setup.py +++ b/zaza/openstack/charm_tests/iscsi-connector/setup.py @@ -55,7 +55,7 @@ def configure_iscsi_target(): password_in = 'secretpass' write_file = ( """echo -e '\n\tbacking-store {}\n\tinitiator-address {}\n\tincominguser {} {}\n\t""" - """outgoinguser {} {}' | sudo tee /etc/tgt/conf.d/iscsi.conf""".format(lun, backing_store, + """outgoinguser {} {}\n' | sudo tee /etc/tgt/conf.d/iscsi.conf""".format(lun, backing_store, initiator_address, username, password, username_in, password_in) ) logging.info('Writing target iscsi.conf') From e5305333454d4a47cd6aa2f900231474fc858ff7 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 30 Jul 2020 10:16:19 +0200 Subject: [PATCH 628/898] Make NeutronCreateNetworkTest more robust --- zaza/openstack/charm_tests/neutron/tests.py | 16 ++++++++++++++-- .../charm_tests/neutron_arista/tests.py | 9 +-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index ec6998d..311e763 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -295,19 +295,31 @@ class NeutronCreateNetworkTest(test_utils.OpenStackBaseTest): def test_400_create_network(self): """Create a network, verify that it exists, and then delete it.""" + self._wait_for_neutron_ready() self._assert_test_network_doesnt_exist() self._create_test_network() net_id = self._assert_test_network_exists_and_return_id() self._delete_test_network(net_id) self._assert_test_network_doesnt_exist() + @classmethod + def _wait_for_neutron_ready(cls): + logging.info('Waiting for Neutron to become ready...') + zaza.model.wait_for_application_states() + for attempt in tenacity.Retrying( + wait=tenacity.wait_fixed(5), # seconds + stop=tenacity.stop_after_attempt(12), + reraise=True): + with attempt: + cls.neutron_client.list_networks() + def _create_test_network(self): - logging.debug('Creating neutron network...') + logging.info('Creating neutron network...') network = {'name': self._TEST_NET_NAME} self.neutron_client.create_network({'network': network}) def _delete_test_network(self, net_id): - logging.debug('Deleting neutron network...') + logging.info('Deleting neutron network...') self.neutron_client.delete_network(net_id) def _assert_test_network_exists_and_return_id(self): diff --git a/zaza/openstack/charm_tests/neutron_arista/tests.py b/zaza/openstack/charm_tests/neutron_arista/tests.py index ba29bd6..354dfd1 100644 --- a/zaza/openstack/charm_tests/neutron_arista/tests.py +++ b/zaza/openstack/charm_tests/neutron_arista/tests.py @@ -29,14 +29,7 @@ class NeutronCreateAristaNetworkTest(neutron_tests.NeutronCreateNetworkTest): def setUpClass(cls): """Run class setup for running Neutron Arista tests.""" super(NeutronCreateAristaNetworkTest, cls).setUpClass() - - logging.info('Waiting for Neutron to become ready...') - for attempt in tenacity.Retrying( - wait=tenacity.wait_fixed(5), # seconds - stop=tenacity.stop_after_attempt(12), - reraise=True): - with attempt: - cls.neutron_client.list_networks() + cls._wait_for_neutron_ready() def _assert_test_network_exists_and_return_id(self): logging.info('Checking that the test network exists on the Arista ' From a005cf32263b3133ea72427ead080b0eda342698 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Tue, 4 Aug 2020 09:12:42 -0500 Subject: [PATCH 629/898] pep8 --- .../charm_tests/iscsi-connector/__init__.py | 2 +- .../charm_tests/iscsi-connector/setup.py | 28 ++++++++----------- .../charm_tests/iscsi-connector/tests.py | 14 ++++++---- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/zaza/openstack/charm_tests/iscsi-connector/__init__.py b/zaza/openstack/charm_tests/iscsi-connector/__init__.py index c0cf179..659b2b0 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/__init__.py +++ b/zaza/openstack/charm_tests/iscsi-connector/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Collection of code for setting up and testing iscsi-connector charm.""" \ No newline at end of file +"""Collection of code for setting up and testing iscsi-connector charm.""" diff --git a/zaza/openstack/charm_tests/iscsi-connector/setup.py b/zaza/openstack/charm_tests/iscsi-connector/setup.py index ff58c4f..b7a3f2d 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/setup.py +++ b/zaza/openstack/charm_tests/iscsi-connector/setup.py @@ -14,22 +14,9 @@ """Code for setting up iscsi-connector tests.""" -# """ -# steps for testing without password: -# - deploy ubuntu for storage -# - deploy ubuntu for initiator -# configure that unit: -# - install tgt -y -# - configure iscsi target -# - need to know the initiator ip -# - restart tgt -# - configure ubuntu initiator with init dict, target, port -# """ - import logging import zaza.model -import zaza.openstack.utilities.openstack as openstack_utils def basic_target_setup(): @@ -45,7 +32,9 @@ def basic_target_setup(): unit.entity_id, cmd) + def configure_iscsi_target(): + """Configure the iscsi target.""" lun = 'iqn.2020-07.canonical.com:lun1' backing_store = 'dev/vdb' initiator_address = zaza.model.get_app_ips('ubuntu')[0] @@ -54,13 +43,18 @@ def configure_iscsi_target(): username_in = 'iscsi-target' password_in = 'secretpass' write_file = ( - """echo -e '\n\tbacking-store {}\n\tinitiator-address {}\n\tincominguser {} {}\n\t""" - """outgoinguser {} {}\n' | sudo tee /etc/tgt/conf.d/iscsi.conf""".format(lun, backing_store, - initiator_address, username, password, username_in, password_in) + """echo -e '\n\tbacking-store {}\n\tinitiator-address """ + """{}\n\tincominguser {} {}\n\toutgoinguser {} {}\n' """ + """ | sudo tee /etc/tgt/conf.d/iscsi.conf""".format(lun, + backing_store, + initiator_address, + username, + password, + username_in, + password_in) ) logging.info('Writing target iscsi.conf') zaza.model.run_on_unit('ubuntu-target/0', write_file) # Restart tgt to load new config restart_tgt = "systemctl restart tgt" zaza.model.run_on_unit('ubuntu-target/0', restart_tgt) - diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py index 89c1223..a703a88 100644 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ b/zaza/openstack/charm_tests/iscsi-connector/tests.py @@ -16,11 +16,9 @@ import json import logging -import tempfile import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.utilities.generic as generic_utils class ISCSIConnectorTest(test_utils.BaseCharmTest): @@ -32,10 +30,11 @@ class ISCSIConnectorTest(test_utils.BaseCharmTest): super(ISCSIConnectorTest, cls).setUpClass() def configure_iscsi_connector(self): + """Configure iscsi connector.""" iqn = 'iqn.2020-07.canonical.com:lun1' unit_fqdn = self.get_unit_full_hostname('ubuntu') target_ip = zaza.model.get_app_ips('ubuntu-target')[0] - initiator_dictionary = json.dumps({unit_fqdn:iqn}) + initiator_dictionary = json.dumps({unit_fqdn: iqn}) conf = { 'initiator-dictionary': initiator_dictionary, 'target': target_ip, @@ -56,13 +55,18 @@ class ISCSIConnectorTest(test_utils.BaseCharmTest): return hostname def test_iscsi_connector(self): + """Test iscsi configuration and wait for idle status.""" self.configure_iscsi_connector() logging.info('Wait for idle/ready status...') zaza.model.wait_for_application_states() - + def test_validate_iscsi_session(self): + """Validate that the iscsi session is active.""" unit = zaza.model.get_units('ubuntu')[0] logging.info('Checking if iscsi session is active.') run = zaza.model.run_on_unit(unit.entity_id, 'iscsiadm -m session') - logging.info('iscsiadm -m session: Stdout: {}, Stderr: {}, Code: {}'.format(run['Stdout'], run['Stderr'], run['Code'])) + logging.info("""iscsiadm -m session: Stdout: {}, Stderr: {}, """ + """Code: {}""".format(run['Stdout'], + run['Stderr'], + run['Code'])) assert run['Code'] == '0' From 8a47b981b9651706f1211912a81fb70379249869 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 6 Aug 2020 23:06:01 +0000 Subject: [PATCH 630/898] Tell horizon to use the "defaul" region SAML Mellon tests have been broken on Focal. The handling of the region selection behaves differently from bionic. Or it may be that bionic accidentally was working by seeing a string as int 0. Horizon has a "default" region which returns the keystone URL. --- zaza/openstack/charm_tests/saml_mellon/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/saml_mellon/tests.py b/zaza/openstack/charm_tests/saml_mellon/tests.py index 28ceb4e..f0f5d18 100644 --- a/zaza/openstack/charm_tests/saml_mellon/tests.py +++ b/zaza/openstack/charm_tests/saml_mellon/tests.py @@ -93,7 +93,7 @@ class CharmKeystoneSAMLMellonTest(BaseKeystoneTest): proto = "http" url = "{}://{}/horizon/auth/login/".format(proto, horizon_ip) - region = "{}://{}:5000/v3".format(proto, keystone_ip) + region = "default" horizon_expect = ('') From c498b358527a7bdacbfeeb5494af57407c74a402 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 6 Aug 2020 23:16:11 +0000 Subject: [PATCH 631/898] Lint fix --- zaza/openstack/charm_tests/saml_mellon/tests.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zaza/openstack/charm_tests/saml_mellon/tests.py b/zaza/openstack/charm_tests/saml_mellon/tests.py index f0f5d18..7bb0abc 100644 --- a/zaza/openstack/charm_tests/saml_mellon/tests.py +++ b/zaza/openstack/charm_tests/saml_mellon/tests.py @@ -72,12 +72,6 @@ class CharmKeystoneSAMLMellonTest(BaseKeystoneTest): def test_saml_mellon_redirects(self): """Validate the horizon -> keystone -> IDP redirects.""" - if self.vip: - keystone_ip = self.vip - else: - unit = zaza.model.get_units(self.application_name)[0] - keystone_ip = unit.public_address - horizon = "openstack-dashboard" horizon_vip = (zaza.model.get_application_config(horizon) .get("vip").get("value")) From 7d4d40700aecbb5ed5aaa6f69f766eb0a00e7558 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 7 Aug 2020 01:12:37 +0000 Subject: [PATCH 632/898] Region is different for before and after Focal For the Horizon region setting use the Keystone URL prior to Focal and "default" thereafter. --- zaza/openstack/charm_tests/saml_mellon/tests.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/saml_mellon/tests.py b/zaza/openstack/charm_tests/saml_mellon/tests.py index 7bb0abc..9b5b211 100644 --- a/zaza/openstack/charm_tests/saml_mellon/tests.py +++ b/zaza/openstack/charm_tests/saml_mellon/tests.py @@ -21,6 +21,7 @@ import requests import zaza.model from zaza.openstack.charm_tests.keystone import BaseKeystoneTest import zaza.charm_lifecycle.utils as lifecycle_utils +import zaza.openstack.utilities.openstack as openstack_utils class FailedToReachIDP(Exception): @@ -42,6 +43,8 @@ class CharmKeystoneSAMLMellonTest(BaseKeystoneTest): cls.test_config = lifecycle_utils.get_charm_config() cls.application_name = cls.test_config['charm_name'] cls.action = "get-sp-metadata" + cls.current_release = openstack_utils.get_os_release() + cls.FOCAL_USSURI = openstack_utils.get_os_release("focal_ussuri") def test_run_get_sp_metadata_action(self): """Validate the get-sp-metadata action.""" @@ -72,6 +75,12 @@ class CharmKeystoneSAMLMellonTest(BaseKeystoneTest): def test_saml_mellon_redirects(self): """Validate the horizon -> keystone -> IDP redirects.""" + if self.vip: + keystone_ip = self.vip + else: + unit = zaza.model.get_units(self.application_name)[0] + keystone_ip = unit.public_address + horizon = "openstack-dashboard" horizon_vip = (zaza.model.get_application_config(horizon) .get("vip").get("value")) @@ -86,8 +95,13 @@ class CharmKeystoneSAMLMellonTest(BaseKeystoneTest): else: proto = "http" + # Use Keystone URL for < Focal + if self.current_release < self.FOCAL_USSURI: + region = "{}://{}:5000/v3".format(proto, keystone_ip) + else: + region = "default" + url = "{}://{}/horizon/auth/login/".format(proto, horizon_ip) - region = "default" horizon_expect = ('') From 80995ccf2341329b67507eba5063e4d3bb56a898 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 10 Aug 2020 09:09:02 +0200 Subject: [PATCH 633/898] Vault tests should leave Vault unsealed When cleaning up after a Vault test case, Vault should be left in the same state we found it, unsealed. Closes-Bug: #379 --- zaza/openstack/charm_tests/vault/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 5c93046..40227fd 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -51,6 +51,10 @@ class BaseVaultTest(test_utils.OpenStackBaseTest): vault_utils.auth_all(cls.clients, cls.vault_creds['root_token']) vault_utils.ensure_secret_backend(cls.clients[0]) + def tearDown(self): + """Tun test cleanup for Vault tests.""" + vault_utils.unseal_all(self.clients, self.vault_creds['keys'][0]) + @contextlib.contextmanager def pause_resume(self, services, pgrep_full=False): """Override pause_resume for Vault behavior.""" From a3433a1276a6ca013a4c8ca7fb8a3e31eecb8ee9 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 10 Aug 2020 12:32:49 +0000 Subject: [PATCH 634/898] Add Ceph test CheckPoolTypes Add a ceph test to check that the type of pools requested by clients matches the pools that were created. --- zaza/openstack/charm_tests/ceph/tests.py | 44 ++++++++++++++++++++++++ zaza/openstack/utilities/ceph.py | 38 ++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 1200934..d20486f 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -794,6 +794,50 @@ class CephPrometheusTest(unittest.TestCase): '3', _get_mon_count_from_prometheus(unit.public_address)) +class CephPoolConfig(Exception): + """Custom Exception for bad Ceph pool config.""" + + pass + + +class CheckPoolTypes(unittest.TestCase): + """Test the ceph pools created for clients are of the expected type.""" + + def test_check_pool_types(self): + """Check type of pools created for clients.""" + app_pools = [ + ('glance', 'glance'), + ('nova-compute', 'nova'), + ('cinder-ceph', 'cinder-ceph')] + runtime_pool_details = zaza_ceph.get_ceph_pool_details() + for app, pool_name in app_pools: + juju_pool_config = zaza_model.get_application_config(app).get( + 'pool-type') + if juju_pool_config: + expected_pool_type = juju_pool_config['value'] + else: + # If the pool-type option is absent assume the default of + # replicated. + expected_pool_type = zaza_ceph.REPLICATED_POOL_TYPE + for pool_config in runtime_pool_details: + if pool_config['pool_name'] == pool_name: + logging.info('Checking {} is {}'.format( + pool_name, + expected_pool_type)) + expected_pool_code = -1 + if expected_pool_type == zaza_ceph.REPLICATED_POOL_TYPE: + expected_pool_code = zaza_ceph.REPLICATED_POOL_CODE + elif expected_pool_type == zaza_ceph.ERASURE_POOL_TYPE: + expected_pool_code = zaza_ceph.ERASURE_POOL_CODE + self.assertEqual( + pool_config['type'], + expected_pool_code) + break + else: + raise CephPoolConfig( + "Failed to find config for {}".format(pool_name)) + + # NOTE: We might query before prometheus has fetch data @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), diff --git a/zaza/openstack/utilities/ceph.py b/zaza/openstack/utilities/ceph.py index 61d7d66..893ea5e 100644 --- a/zaza/openstack/utilities/ceph.py +++ b/zaza/openstack/utilities/ceph.py @@ -5,6 +5,11 @@ import logging import zaza.openstack.utilities.openstack as openstack_utils import zaza.model as zaza_model +REPLICATED_POOL_TYPE = 'replicated' +ERASURE_POOL_TYPE = 'erasure-coded' +REPLICATED_POOL_CODE = 1 +ERASURE_POOL_CODE = 3 + def get_expected_pools(radosgw=False): """Get expected ceph pools. @@ -97,6 +102,39 @@ def get_ceph_pools(unit_name, model_name=None): return pools +def get_ceph_pool_details(query_leader=True, unit_name=None, model_name=None): + """Get ceph pool details. + + Return a list of ceph pools details dicts. + + :param query_leader: Whether to query the leader for pool details. + :type query_leader: bool + :param unit_name: Name of unit to get the pools on if query_leader is False + :type unit_name: string + :param model_name: Name of model to operate in + :type model_name: str + :returns: Dict of ceph pools + :rtype: List[Dict,] + :raise: zaza_model.CommandRunFailed + """ + cmd = 'sudo ceph osd pool ls detail -f json' + if query_leader and unit_name: + raise ValueError("Cannot set query_leader and unit_name") + if query_leader: + result = zaza_model.run_on_leader( + 'ceph-mon', + cmd, + model_name=model_name) + else: + result = zaza_model.run_on_unit( + unit_name, + cmd, + model_name=model_name) + if int(result.get('Code')) != 0: + raise zaza_model.CommandRunFailed(cmd, result) + return json.loads(result.get('Stdout')) + + def get_ceph_df(unit_name, model_name=None): """Return dict of ceph df json output, including ceph pool state. From 3343582ddd80dc71c5f25d3bf140dc3b06957943 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Mon, 17 Aug 2020 17:32:22 -0500 Subject: [PATCH 635/898] add test for external cert config option for gnocchi --- zaza/openstack/charm_tests/gnocchi/tests.py | 38 ++++++++++ .../charm_tests/iscsi-connector/__init__.py | 15 ---- .../charm_tests/iscsi-connector/setup.py | 60 ---------------- .../charm_tests/iscsi-connector/tests.py | 72 ------------------- 4 files changed, 38 insertions(+), 147 deletions(-) delete mode 100644 zaza/openstack/charm_tests/iscsi-connector/__init__.py delete mode 100644 zaza/openstack/charm_tests/iscsi-connector/setup.py delete mode 100644 zaza/openstack/charm_tests/iscsi-connector/tests.py diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 680bf3c..93051d5 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -16,12 +16,15 @@ """Encapsulate Gnocchi testing.""" +import base64 import boto3 import logging import pprint from gnocchiclient.v1 import client as gnocchi_client +import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities as utilities import zaza.openstack.utilities.openstack as openstack_utils @@ -89,6 +92,41 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): # Create AWS compatible application credentials in Keystone cls.ec2_creds = ks_client.ec2.create(user_id, project_id) + def test_upload_external_cert(self): + """Verify that the external CA is uploaded correctly.""" + logging.info('Changing value for trusted-external-ca-cert.') + ca_cert_option = 'trusted-external-ca-cert' + ppk, cert = utilities.cert.generate_cert('gnocchi_test.ci.local') + b64_cert = base64.b64encode(cert).decode() + config = { + ca_cert_option: b64_cert, + } + model.set_application_config( + 'gnocchi', + config + ) + model.block_until_all_units_idle() + + def test_validate_cert_location(self): + """Validate that the cert is correctly uploaded to the unit.""" + cert_location = '/usr/local/share/ca-certificates' + cert_name = 'gnocchi-external.crt' + cmd = 'ls ' + cert_location + '/' + cert_name + logging.info("Validating that the file {} is created in \ + {}".format(cert_name, cert_location)) + result = model.run_on_unit('gnocchi/0', cmd) + self.assertEqual(result['Code'], '0') + + def test_validate_update_ca_certificates(self): + """Validate that /usr/sbin/update-ca-certificates ran successfully.""" + linked_cert_location = '/etc/ssl/certs' + cert_name = 'gnocchi-external.pem' + cmd = 'ls ' + linked_cert_location + '/' + cert_name + logging.info("Validating that the link {} is created in \ + {}".format(cert_name, linked_cert_location)) + result = model.run_on_unit('gnocchi/0', cmd) + self.assertEqual(result['Code'], '0') + def test_s3_list_gnocchi_buckets(self): """Verify that the gnocchi buckets were created in the S3 backend.""" kwargs = { diff --git a/zaza/openstack/charm_tests/iscsi-connector/__init__.py b/zaza/openstack/charm_tests/iscsi-connector/__init__.py deleted file mode 100644 index 659b2b0..0000000 --- a/zaza/openstack/charm_tests/iscsi-connector/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Collection of code for setting up and testing iscsi-connector charm.""" diff --git a/zaza/openstack/charm_tests/iscsi-connector/setup.py b/zaza/openstack/charm_tests/iscsi-connector/setup.py deleted file mode 100644 index b7a3f2d..0000000 --- a/zaza/openstack/charm_tests/iscsi-connector/setup.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Code for setting up iscsi-connector tests.""" - -import logging - -import zaza.model - - -def basic_target_setup(): - """Run basic setup for iscsi guest.""" - logging.info('Installing package tgt on ubuntu-target') - unit = zaza.model.get_units('ubuntu-target')[0] - setup_cmds = [ - "apt install --yes tgt", - "systemctl start tgt", - "open-port 3260"] - for cmd in setup_cmds: - zaza.model.run_on_unit( - unit.entity_id, - cmd) - - -def configure_iscsi_target(): - """Configure the iscsi target.""" - lun = 'iqn.2020-07.canonical.com:lun1' - backing_store = 'dev/vdb' - initiator_address = zaza.model.get_app_ips('ubuntu')[0] - username = 'iscsi-user' - password = 'password123' - username_in = 'iscsi-target' - password_in = 'secretpass' - write_file = ( - """echo -e '\n\tbacking-store {}\n\tinitiator-address """ - """{}\n\tincominguser {} {}\n\toutgoinguser {} {}\n' """ - """ | sudo tee /etc/tgt/conf.d/iscsi.conf""".format(lun, - backing_store, - initiator_address, - username, - password, - username_in, - password_in) - ) - logging.info('Writing target iscsi.conf') - zaza.model.run_on_unit('ubuntu-target/0', write_file) - # Restart tgt to load new config - restart_tgt = "systemctl restart tgt" - zaza.model.run_on_unit('ubuntu-target/0', restart_tgt) diff --git a/zaza/openstack/charm_tests/iscsi-connector/tests.py b/zaza/openstack/charm_tests/iscsi-connector/tests.py deleted file mode 100644 index a703a88..0000000 --- a/zaza/openstack/charm_tests/iscsi-connector/tests.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Encapsulate iscsi-connector testing.""" - -import json -import logging - -import zaza.model -import zaza.openstack.charm_tests.test_utils as test_utils - - -class ISCSIConnectorTest(test_utils.BaseCharmTest): - """Class for iscsi-connector tests.""" - - @classmethod - def setUpClass(cls): - """Run class setup for running glance tests.""" - super(ISCSIConnectorTest, cls).setUpClass() - - def configure_iscsi_connector(self): - """Configure iscsi connector.""" - iqn = 'iqn.2020-07.canonical.com:lun1' - unit_fqdn = self.get_unit_full_hostname('ubuntu') - target_ip = zaza.model.get_app_ips('ubuntu-target')[0] - initiator_dictionary = json.dumps({unit_fqdn: iqn}) - conf = { - 'initiator-dictionary': initiator_dictionary, - 'target': target_ip, - 'port': '3260', - 'iscsi-node-session-auth-authmethod': 'CHAP', - 'iscsi-node-session-auth-username': 'iscsi-user', - 'iscsi-node-session-auth-password': 'password123', - 'iscsi-node-session-auth-username-in': 'iscsi-target', - 'iscsi-node-session-auth-password-in': 'secretpass', - } - zaza.model.set_application_config('iscsi-connector', conf) - - def get_unit_full_hostname(self, unit_name): - """Retrieve the full hostname of a unit.""" - for unit in zaza.model.get_units(unit_name): - result = zaza.model.run_on_unit(unit.entity_id, 'hostname -f') - hostname = result['Stdout'].rstrip() - return hostname - - def test_iscsi_connector(self): - """Test iscsi configuration and wait for idle status.""" - self.configure_iscsi_connector() - logging.info('Wait for idle/ready status...') - zaza.model.wait_for_application_states() - - def test_validate_iscsi_session(self): - """Validate that the iscsi session is active.""" - unit = zaza.model.get_units('ubuntu')[0] - logging.info('Checking if iscsi session is active.') - run = zaza.model.run_on_unit(unit.entity_id, 'iscsiadm -m session') - logging.info("""iscsiadm -m session: Stdout: {}, Stderr: {}, """ - """Code: {}""".format(run['Stdout'], - run['Stderr'], - run['Code'])) - assert run['Code'] == '0' From a67b4906eef7e199b2a0949f6723a6d64097f670 Mon Sep 17 00:00:00 2001 From: coreycb Date: Tue, 18 Aug 2020 09:54:09 -0400 Subject: [PATCH 636/898] Use juju model as workspace and store in home (#383) * Use juju model as workspace and store in home The current juju model will now be used as the tempest workspace name. Additionally, all workspaces will be stored in ~/.tempest/. This patch also introduces a new option 'keep-workspace' that can be specified along with other tempest options to keep the workspace after tempest test execution. It defaults to False. Also minor adjustment to smoke option to test boolean value. --- zaza/openstack/charm_tests/tempest/setup.py | 19 +++--- zaza/openstack/charm_tests/tempest/tests.py | 20 ++++-- zaza/openstack/charm_tests/tempest/utils.py | 67 +++++++++++++++++++++ 3 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 zaza/openstack/charm_tests/tempest/utils.py diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index af1ee4c..7ce4478 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -16,11 +16,12 @@ import jinja2 import urllib.parse -import subprocess +import os import zaza.utilities.deployment_env as deployment_env import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.tempest.utils as tempest_utils import zaza.openstack.charm_tests.glance.setup as glance_setup SETUP_ENV_VARS = { @@ -279,21 +280,15 @@ def setup_tempest(tempest_template, accounts_template): :returns: None :rtype: None """ - try: - subprocess.check_call(['tempest', 'workspace', 'remove', '--rmdir', - '--name', 'tempest-workspace']) - except subprocess.CalledProcessError: - pass - try: - subprocess.check_call(['tempest', 'init', 'tempest-workspace']) - except subprocess.CalledProcessError: - pass + workspace_name, workspace_path = tempest_utils.get_workspace() + tempest_utils.destroy_workspace(workspace_name, workspace_path) + tempest_utils.init_workspace(workspace_path) render_tempest_config( - 'tempest-workspace/etc/tempest.conf', + os.path.join(workspace_path, 'etc/tempest.conf'), get_tempest_context(), tempest_template) render_tempest_config( - 'tempest-workspace/etc/accounts.yaml', + os.path.join(workspace_path, 'etc/accounts.yaml'), get_tempest_context(), accounts_template) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index d726930..979caff 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -20,6 +20,7 @@ import subprocess import zaza import zaza.charm_lifecycle.utils import zaza.charm_lifecycle.test +import zaza.openstack.charm_tests.tempest.utils as tempest_utils import tempfile @@ -33,21 +34,24 @@ class TempestTest(): Test keys are parsed from ['tests_options']['tempest']['model'], where valid test keys are: smoke (bool), whitelist (list of tests), blacklist - (list of tests), and regex (list of regex's). + (list of tests), regex (list of regex's), and keep-workspace (bool). :returns: Status of tempest run :rtype: bool """ + result = True charm_config = zaza.charm_lifecycle.utils.get_charm_config() + workspace_name, workspace_path = tempest_utils.get_workspace() tempest_options = ['tempest', 'run', '--workspace', - 'tempest-workspace', '--config', - 'tempest-workspace/etc/tempest.conf'] + workspace_name, '--config', + os.path.join(workspace_path, 'etc/tempest.conf')] for model_alias in zaza.model.get_juju_model_aliases().keys(): tempest_test_key = model_alias if model_alias == zaza.charm_lifecycle.utils.DEFAULT_MODEL_ALIAS: tempest_test_key = 'default' config = charm_config['tests_options']['tempest'][tempest_test_key] - if config.get('smoke'): + smoke = config.get('smoke') + if smoke and smoke is True: tempest_options.extend(['--smoke']) if config.get('regex'): tempest_options.extend( @@ -74,5 +78,9 @@ class TempestTest(): try: subprocess.check_call(tempest_options) except subprocess.CalledProcessError: - return False - return True + result = False + break + keep_workspace = config.get('keep-workspace') + if not keep_workspace or keep_workspace is not True: + tempest_utils.destroy_workspace(workspace_name, workspace_path) + return result diff --git a/zaza/openstack/charm_tests/tempest/utils.py b/zaza/openstack/charm_tests/tempest/utils.py new file mode 100644 index 0000000..8f3aea7 --- /dev/null +++ b/zaza/openstack/charm_tests/tempest/utils.py @@ -0,0 +1,67 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility code for working with tempest workspaces.""" + +import os +from pathlib import Path +import shutil +import subprocess + +import zaza.model as model + + +def get_workspace(): + """Get tempest workspace name and path. + + :returns: A tuple containing tempest workspace name and workspace path + :rtype: Tuple[str, str] + """ + home = str(Path.home()) + workspace_name = model.get_juju_model() + workspace_path = os.path.join(home, '.tempest', workspace_name) + return (workspace_name, workspace_path) + + +def destroy_workspace(workspace_name, workspace_path): + """Delete tempest workspace. + + :param workspace_name: name of workspace + :type workspace_name: str + :param workspace_path: directory path where workspace is stored + :type workspace_path: str + :returns: None + :rtype: None + """ + try: + subprocess.check_call(['tempest', 'workspace', 'remove', '--rmdir', + '--name', workspace_name]) + except subprocess.CalledProcessError: + pass + if os.path.isdir(workspace_path): + shutil.rmtree(workspace_path) + + +def init_workspace(workspace_path): + """Initialize tempest workspace. + + :param workspace_path: directory path where workspace is stored + :type workspace_path: str + :returns: None + :rtype: None + """ + try: + subprocess.check_call(['tempest', 'init', workspace_path]) + except subprocess.CalledProcessError: + pass From 8681b02353c45c389d9e88e2753f345b334defa8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 19 Aug 2020 12:36:23 +0000 Subject: [PATCH 637/898] Support https in horizon tests Update horizin tests to support https if available. --- .../charm_tests/openstack_dashboard/tests.py | 83 +++++++++++++------ 1 file changed, 56 insertions(+), 27 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 7ac0dd0..d492b42 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -46,11 +46,11 @@ class FailedAuth(AuthExceptions): @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=5, max=10), reraise=True) -def _login(dashboard_ip, domain, username, password): +def _login(dashboard_url, domain, username, password, cafile=None): """Login to the website to get a session. - :param dashboard_ip: The IP address of the dashboard to log in to. - :type dashboard_ip: str + :param dashboard_url: The URL of the dashboard to log in to. + :type dashboard_url: str :param domain: the domain to login into :type domain: str :param username: the username to login as @@ -62,11 +62,11 @@ def _login(dashboard_ip, domain, username, password): :rtype: (requests.sessions.Session, requests.models.Response) :raises: FailedAuth if the authorisation doesn't work """ - auth_url = 'http://{}/horizon/auth/login/'.format(dashboard_ip) + auth_url = '{}/auth/login/'.format(dashboard_url) # start session, get csrftoken client = requests.session() - client.get(auth_url) + client.get(auth_url, verify=cafile) if 'csrftoken' in client.cookies: csrftoken = client.cookies['csrftoken'] @@ -117,7 +117,11 @@ def _login(dashboard_ip, domain, username, password): del auth['domain'] logging.info('POST data: "{}"'.format(auth)) - response = client.post(auth_url, data=auth, headers={'Referer': auth_url}) + response = client.post( + auth_url, + data=auth, + headers={'Referer': auth_url}, + verify=cafile) # NOTE(ajkavanagh) there used to be a trusty/icehouse test in the # amulet test, but as the zaza tests only test from trusty/mitaka @@ -147,7 +151,7 @@ def _login(dashboard_ip, domain, username, password): retry=tenacity.retry_if_exception_type( http.client.RemoteDisconnected), reraise=True) -def _do_request(request): +def _do_request(request, cafile=None): """Open a webpage via urlopen. :param request: A urllib request object. @@ -156,7 +160,7 @@ def _do_request(request): :rtype: object :raises: URLError on protocol errors """ - return urllib.request.urlopen(request) + return urllib.request.urlopen(request, cafile=cafile) class OpenStackDashboardTests(test_utils.OpenStackBaseTest): @@ -167,6 +171,13 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): """Run class setup for running openstack dashboard charm tests.""" super(OpenStackDashboardTests, cls).setUpClass() cls.application = 'openstack-dashboard' + cls.use_https = False + vault_relation = zaza_model.get_relation_id( + cls.application, + 'vault', + remote_interface_name='certificates') + if vault_relation: + cls.use_https = True def test_050_local_settings_permissions_regression_check_lp1755027(self): """Assert regression check lp1755027. @@ -226,6 +237,7 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): request = urllib.request.Request( 'http://{}:{}'.format(unit.public_address, port)) + request = urllib.request.Request(self.get_base_url()) output = str(generic_utils.get_file_contents(unit, conf)) for line in output.split('\n'): @@ -291,22 +303,42 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): mismatches.append(msg) return mismatches + def get_base_url(self): + """Return the base url for http(s) requests. + + :returns: URL + :rtype: str + """ + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name(self.application_name)) + logging.debug("Dashboard ip is:{}".format(unit.public_address)) + scheme = 'http' + if self.use_https: + scheme = 'https' + url = '{}://{}'.format(scheme, unit.public_address) + logging.debug("Base URL is: {}".format(url)) + return url + + def get_horizon_url(self): + """Return the url for acccessing horizon. + + :returns: Horizon URL + :rtype: str + """ + url = '{}/horizon'.format(self.get_base_url()) + logging.info("Horizon URL is: {}".format(url)) + return url + def test_400_connection(self): """Test that dashboard responds to http request. Ported from amulet tests. """ logging.info('Checking dashboard http response...') - - unit = zaza_model.get_unit_from_name( - zaza_model.get_lead_unit_name(self.application_name)) - logging.debug("... dashboard_ip is:{}".format(unit.public_address)) - - request = urllib.request.Request( - 'http://{}/horizon'.format(unit.public_address)) + request = urllib.request.Request(self.get_horizon_url()) try: logging.info("... trying to fetch the page") - html = _do_request(request) + html = _do_request(request, cafile=self.cacert) logging.info("... fetched page") except Exception as e: logging.info("... exception raised was {}".format(str(e))) @@ -319,8 +351,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): """Validate that authentication succeeds for client log in.""" logging.info('Checking authentication through dashboard...') - unit = zaza_model.get_unit_from_name( - zaza_model.get_lead_unit_name(self.application_name)) overcloud_auth = openstack_utils.get_overcloud_auth() password = overcloud_auth['OS_PASSWORD'], logging.info("admin password is {}".format(password)) @@ -329,7 +359,12 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): domain = 'admin_domain', username = 'admin', password = overcloud_auth['OS_PASSWORD'], - _login(unit.public_address, domain, username, password) + _login( + self.get_horizon_url(), + domain, + username, + password, + cafile=self.cacert) logging.info('OK') def test_404_connection(self): @@ -338,22 +373,16 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): Ported from amulet tests. """ logging.info('Checking apache mod_status gets disabled.') - - unit = zaza_model.get_unit_from_name( - zaza_model.get_lead_unit_name(self.application_name)) - logging.debug("... dashboard_ip is: {}".format(unit.public_address)) - logging.debug('Maybe enabling hardening for apache...') _app_config = zaza_model.get_application_config(self.application_name) logging.info(_app_config['harden']) - request = urllib.request.Request( - 'http://{}/server-status'.format(unit.public_address)) + request = urllib.request.Request(self.get_horizon_url()) with self.config_change( {'harden': _app_config['harden'].get('value', '')}, {'harden': 'apache'}): try: - _do_request(request) + _do_request(request, cafile=self.cacert) except urllib.request.HTTPError as e: # test failed if it didn't return 404 msg = "Apache mod_status check failed." From 8a79a93e8305c31926d317f2aaea362811695a50 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Wed, 19 Aug 2020 08:59:04 -0500 Subject: [PATCH 638/898] combine 3 functions into one --- zaza/openstack/charm_tests/gnocchi/tests.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 93051d5..3487996 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -107,8 +107,7 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): ) model.block_until_all_units_idle() - def test_validate_cert_location(self): - """Validate that the cert is correctly uploaded to the unit.""" + logging.info('Validate that the cert is correctly uploaded to the unit') cert_location = '/usr/local/share/ca-certificates' cert_name = 'gnocchi-external.crt' cmd = 'ls ' + cert_location + '/' + cert_name @@ -117,13 +116,12 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): result = model.run_on_unit('gnocchi/0', cmd) self.assertEqual(result['Code'], '0') - def test_validate_update_ca_certificates(self): - """Validate that /usr/sbin/update-ca-certificates ran successfully.""" + logging.info('Validate that /usr/sbin/update-ca-certificates ran successfully') linked_cert_location = '/etc/ssl/certs' - cert_name = 'gnocchi-external.pem' - cmd = 'ls ' + linked_cert_location + '/' + cert_name + linked_cert_name = 'gnocchi-external.pem' + cmd = 'ls ' + linked_cert_location + '/' + linked_cert_name logging.info("Validating that the link {} is created in \ - {}".format(cert_name, linked_cert_location)) + {}".format(linked_cert_name, linked_cert_location)) result = model.run_on_unit('gnocchi/0', cmd) self.assertEqual(result['Code'], '0') From cd548b01cfd470983f1a6f49748be3297c7e9b7d Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Wed, 19 Aug 2020 09:32:09 -0500 Subject: [PATCH 639/898] pep8, duplicate logs --- zaza/openstack/charm_tests/gnocchi/tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 3487996..e3b9ae1 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -107,7 +107,6 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): ) model.block_until_all_units_idle() - logging.info('Validate that the cert is correctly uploaded to the unit') cert_location = '/usr/local/share/ca-certificates' cert_name = 'gnocchi-external.crt' cmd = 'ls ' + cert_location + '/' + cert_name @@ -116,7 +115,6 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): result = model.run_on_unit('gnocchi/0', cmd) self.assertEqual(result['Code'], '0') - logging.info('Validate that /usr/sbin/update-ca-certificates ran successfully') linked_cert_location = '/etc/ssl/certs' linked_cert_name = 'gnocchi-external.pem' cmd = 'ls ' + linked_cert_location + '/' + linked_cert_name From c5c964506e18c92cca8bea5b47e9981ae8d17e67 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 19 Aug 2020 16:43:04 +0200 Subject: [PATCH 640/898] Increase neutron-arista timeout --- zaza/openstack/charm_tests/neutron_arista/tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron_arista/tests.py b/zaza/openstack/charm_tests/neutron_arista/tests.py index 354dfd1..f9eb2fb 100644 --- a/zaza/openstack/charm_tests/neutron_arista/tests.py +++ b/zaza/openstack/charm_tests/neutron_arista/tests.py @@ -38,9 +38,10 @@ class NeutronCreateAristaNetworkTest(neutron_tests.NeutronCreateNetworkTest): # Sometimes the API call from Neutron to Arista fails and Neutron # retries a couple of seconds later, which is why the newly created # test network may not be immediately visible on Arista's API. + # NOTE(lourot): I experienced a run where it took 53 seconds. for attempt in tenacity.Retrying( wait=tenacity.wait_fixed(10), # seconds - stop=tenacity.stop_after_attempt(3), + stop=tenacity.stop_after_attempt(12), reraise=True): with attempt: actual_network_names = arista_utils.query_fixture_networks( @@ -56,7 +57,7 @@ class NeutronCreateAristaNetworkTest(neutron_tests.NeutronCreateNetworkTest): for attempt in tenacity.Retrying( wait=tenacity.wait_fixed(10), # seconds - stop=tenacity.stop_after_attempt(3), + stop=tenacity.stop_after_attempt(12), reraise=True): with attempt: actual_network_names = arista_utils.query_fixture_networks( From 110dc534dd4f5fcd85f3f45b3c1605de9d4834cc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 19 Aug 2020 15:47:44 +0000 Subject: [PATCH 641/898] Add vip support --- .../charm_tests/openstack_dashboard/tests.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index d492b42..d3fc625 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -237,7 +237,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): request = urllib.request.Request( 'http://{}:{}'.format(unit.public_address, port)) - request = urllib.request.Request(self.get_base_url()) output = str(generic_utils.get_file_contents(unit, conf)) for line in output.split('\n'): @@ -309,13 +308,20 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): :returns: URL :rtype: str """ - unit = zaza_model.get_unit_from_name( - zaza_model.get_lead_unit_name(self.application_name)) - logging.debug("Dashboard ip is:{}".format(unit.public_address)) + vip = (zaza_model.get_application_config(self.application_name) + .get("vip").get("value")) + if vip: + ip = vip + else: + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name(self.application_name)) + ip = unit.public_address + + logging.debug("Dashboard ip is:{}".format(ip)) scheme = 'http' if self.use_https: scheme = 'https' - url = '{}://{}'.format(scheme, unit.public_address) + url = '{}://{}'.format(scheme, ip) logging.debug("Base URL is: {}".format(url)) return url From 5ad5f85f99958bfbefacc184a90d10b0d4f19a5c Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 19 Aug 2020 20:47:00 +0100 Subject: [PATCH 642/898] Fix the charm-upgrade tests as used by CoT The biggest change is that the upgrade_groups() is a list of tuples ... [()] ... rathern than a collections.OrderedDict(). This tends to make more sense as they are processed in order and don't actually need to be indexed. Also excludes easyrsa (as it's not an upgrade target) and adds a "Database Services" which upgrades mysql or percona FIRST before moving on to rabbitmq and other stateful services. This is because some charms really need to talk to mysql if one of the other stateful services does a relation changed hook. This makes it more likely that the system will ugprade correctly. --- .../test_zaza_utilities_upgrade_utils.py | 15 +++-- .../charm_tests/charm_upgrade/tests.py | 7 ++- .../series_upgrade/parallel_tests.py | 7 ++- .../charm_tests/series_upgrade/tests.py | 23 ++------ zaza/openstack/utilities/upgrade_utils.py | 57 +++++++++++++------ 5 files changed, 64 insertions(+), 45 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py index 814772e..e038daf 100644 --- a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py +++ b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections import copy import mock import pprint @@ -89,12 +88,14 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): expected) def test_get_upgrade_groups(self): - expected = collections.OrderedDict([ + # expected = collections.OrderedDict([ + expected = [ + ('Database Services', []), ('Stateful Services', []), ('Core Identity', []), ('Control Plane', ['cinder']), ('Data Plane', ['nova-compute']), - ('sweep_up', [])]) + ('sweep_up', [])] actual = openstack_upgrade.get_upgrade_groups() pprint.pprint(expected) pprint.pprint(actual) @@ -103,12 +104,14 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): expected) def test_get_series_upgrade_groups(self): - expected = collections.OrderedDict([ - ('Stateful Services', ['mydb']), + # expected = collections.OrderedDict([ + expected = [ + ('Database Services', ['mydb']), + ('Stateful Services', []), ('Core Identity', []), ('Control Plane', ['cinder']), ('Data Plane', ['nova-compute']), - ('sweep_up', ['ntp'])]) + ('sweep_up', ['ntp'])] actual = openstack_upgrade.get_series_upgrade_groups() pprint.pprint(expected) pprint.pprint(actual) diff --git a/zaza/openstack/charm_tests/charm_upgrade/tests.py b/zaza/openstack/charm_tests/charm_upgrade/tests.py index 0ffd30b..f5f301a 100644 --- a/zaza/openstack/charm_tests/charm_upgrade/tests.py +++ b/zaza/openstack/charm_tests/charm_upgrade/tests.py @@ -53,8 +53,11 @@ class FullCloudCharmUpgradeTest(unittest.TestCase): """Run charm upgrade.""" self.lts.test_launch_small_instance() applications = zaza.model.get_status().applications - groups = upgrade_utils.get_charm_upgrade_groups() - for group_name, group in groups.items(): + groups = upgrade_utils.get_charm_upgrade_groups( + extra_filters=[upgrade_utils._filter_etcd, + upgrade_utils._filter_easyrsa, + upgrade_utils._filter_memcached]) + for group_name, group in groups: logging.info("About to upgrade {} ({})".format(group_name, group)) for application, app_details in applications.items(): if application not in group: diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py index 6408cf0..a49eae6 100644 --- a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -73,11 +73,14 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): workaround_script = None files = [] applications = model.get_status().applications - for group_name, apps in upgrade_groups.items(): + for group_name, apps in upgrade_groups: logging.info("About to upgrade {} from {} to {}".format( group_name, from_series, to_series)) upgrade_functions = [] - if group_name in ["Stateful Services", "Data Plane", "sweep_up"]: + if group_name in ["Database Services", + "Stateful Services", + "Data Plane", + "sweep_up"]: logging.info("Going to upgrade {} unit by unit".format(apps)) upgrade_function = \ parallel_series_upgrade.serial_series_upgrade diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 9e72338..4b06dbb 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -30,22 +30,6 @@ from zaza.openstack.utilities import ( from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest -def _filter_easyrsa(app, app_config, model_name=None): - charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) - if "easyrsa" in charm_name: - logging.warn("Skipping series upgrade of easyrsa Bug #1850121") - return True - return False - - -def _filter_etcd(app, app_config, model_name=None): - charm_name = upgrade_utils.extract_charm_name_from_url(app_config['charm']) - if "etcd" in charm_name: - logging.warn("Skipping series upgrade of easyrsa Bug #1850124") - return True - return False - - class SeriesUpgradeTest(unittest.TestCase): """Class to encapsulate Series Upgrade Tests.""" @@ -75,7 +59,7 @@ class SeriesUpgradeTest(unittest.TestCase): continue if "etcd" in app_details["charm"]: logging.warn( - "Skipping series upgrade of easyrsa Bug #1850124") + "Skipping series upgrade of etcd Bug #1850124") continue charm_name = upgrade_utils.extract_charm_name_from_url( app_details['charm']) @@ -208,10 +192,11 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): # Set Feature Flag os.environ["JUJU_DEV_FEATURE_FLAGS"] = "upgrade-series" upgrade_groups = upgrade_utils.get_series_upgrade_groups( - extra_filters=[_filter_etcd, _filter_easyrsa]) + extra_filters=[upgrade_utils._filter_etcd, + upgrade_utils._filter_easyrsa]) applications = model.get_status().applications completed_machines = [] - for group_name, group in upgrade_groups.items(): + for group_name, group in upgrade_groups: logging.warn("About to upgrade {} ({})".format(group_name, group)) upgrade_group = [] for application, app_details in applications.items(): diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index b134a69..995e0bb 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -13,15 +13,17 @@ # limitations under the License. """Collection of functions to support upgrade testing.""" -import re + +import itertools import logging -import collections +import re + import zaza.model -SERVICE_GROUPS = collections.OrderedDict([ - ('Stateful Services', ['percona-cluster', 'rabbitmq-server', 'ceph-mon', - 'mysql-innodb-cluster']), +SERVICE_GROUPS = ( + ('Database Services', ['percona-cluster', 'mysql-innodb-cluster']), + ('Stateful Services', ['rabbitmq-server', 'ceph-mon']), ('Core Identity', ['keystone']), ('Control Plane', [ 'aodh', 'barbican', 'ceilometer', 'ceph-fs', @@ -31,8 +33,7 @@ SERVICE_GROUPS = collections.OrderedDict([ 'nova-cloud-controller', 'openstack-dashboard']), ('Data Plane', [ 'nova-compute', 'ceph-osd', - 'swift-proxy', 'swift-storage']) -]) + 'swift-proxy', 'swift-storage'])) UPGRADE_EXCLUDE_LIST = ['rabbitmq-server', 'percona-cluster'] @@ -106,6 +107,30 @@ def _apply_extra_filters(filters, extra_filters): return filters +def _filter_easyrsa(app, app_config, model_name=None): + charm_name = extract_charm_name_from_url(app_config['charm']) + if "easyrsa" in charm_name: + logging.warn("Skipping upgrade of easyrsa Bug #1850121") + return True + return False + + +def _filter_etcd(app, app_config, model_name=None): + charm_name = extract_charm_name_from_url(app_config['charm']) + if "etcd" in charm_name: + logging.warn("Skipping upgrade of easyrsa Bug #1850124") + return True + return False + + +def _filter_memcached(app, app_config, model_name=None): + charm_name = extract_charm_name_from_url(app_config['charm']) + if "memcached" in charm_name: + logging.warn("Skipping upgrade of memcached charm") + return True + return False + + def get_upgrade_groups(model_name=None, extra_filters=None): """Place apps in the model into their upgrade groups. @@ -170,21 +195,21 @@ def get_charm_upgrade_groups(model_name=None, extra_filters=None): def _build_service_groups(applications): - groups = collections.OrderedDict() - for phase_name, charms in SERVICE_GROUPS.items(): + groups = [] + for phase_name, charms in SERVICE_GROUPS: group = [] for app, app_config in applications.items(): charm_name = extract_charm_name_from_url(app_config['charm']) if charm_name in charms: group.append(app) - groups[phase_name] = group + groups.append((phase_name, group)) - sweep_up = [] - for app in applications: - if not (app in [a for group in groups.values() for a in group]): - sweep_up.append(app) - groups['sweep_up'] = sweep_up - for name, group in groups.items(): + # collect all the values into a list, and then a lookup hash + values = list(itertools.chain(*(ls for _, ls in groups))) + vhash = {v: 1 for v in values} + sweep_up = [app for app in applications if app not in vhash] + groups.append(('sweep_up', sweep_up)) + for name, group in groups: group.sort() return groups From 5455bee2e968124cce5ab726d32fa491011fb28f Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 20 Aug 2020 08:51:57 +0200 Subject: [PATCH 643/898] Fix protocol rendering for keystone URL Closes #387 --- zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 index f5505ad..83a05ca 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 @@ -34,7 +34,7 @@ attach_encrypted_volume = false {% if 'keystone' in enabled_services %} [identity] -uri = {proto}://{{ keystone }}:5000/v2.0 +uri = {{ proto }}://{{ keystone }}:5000/v2.0 auth_version = v2 admin_role = Admin region = RegionOne From 056e29cbdc71850ba8a50341c146279cf9fd5997 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 20 Aug 2020 11:15:16 +0100 Subject: [PATCH 644/898] Remove commented out code sections These were left over from the refactor. --- unit_tests/utilities/test_zaza_utilities_upgrade_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py index e038daf..f42ba5a 100644 --- a/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py +++ b/unit_tests/utilities/test_zaza_utilities_upgrade_utils.py @@ -88,7 +88,6 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): expected) def test_get_upgrade_groups(self): - # expected = collections.OrderedDict([ expected = [ ('Database Services', []), ('Stateful Services', []), @@ -104,7 +103,6 @@ class TestUpgradeUtils(ut_utils.BaseTestCase): expected) def test_get_series_upgrade_groups(self): - # expected = collections.OrderedDict([ expected = [ ('Database Services', ['mydb']), ('Stateful Services', []), From 6a18a37c13bec181375ffa541d1a396652f9431b Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 20 Aug 2020 17:08:33 +0200 Subject: [PATCH 645/898] Ensure that the workspace deletion doesn't fail with file not found. --- zaza/openstack/charm_tests/tempest/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/utils.py b/zaza/openstack/charm_tests/tempest/utils.py index 8f3aea7..52ae126 100644 --- a/zaza/openstack/charm_tests/tempest/utils.py +++ b/zaza/openstack/charm_tests/tempest/utils.py @@ -47,7 +47,7 @@ def destroy_workspace(workspace_name, workspace_path): try: subprocess.check_call(['tempest', 'workspace', 'remove', '--rmdir', '--name', workspace_name]) - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): pass if os.path.isdir(workspace_path): shutil.rmtree(workspace_path) From d7bc2d32986ebe1718f7e2e3f4d8981a9c71c80c Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 20 Aug 2020 17:22:31 +0200 Subject: [PATCH 646/898] Revert "Merge pull request #384 from camille-rodriguez/master" This reverts commit 73bab8395bb793578e3992cf4d2a31e89d642791, reversing changes made to c165c42c0030269ea62feda7a6471985e0eb172c. This change introduced a new test under an existing test class, which is a breaking change and prevents us to keep testing older releases. This new test will be soon re-introduced under a new test class. --- zaza/openstack/charm_tests/gnocchi/tests.py | 34 --------------------- 1 file changed, 34 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index e3b9ae1..680bf3c 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -16,15 +16,12 @@ """Encapsulate Gnocchi testing.""" -import base64 import boto3 import logging import pprint from gnocchiclient.v1 import client as gnocchi_client -import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.utilities as utilities import zaza.openstack.utilities.openstack as openstack_utils @@ -92,37 +89,6 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): # Create AWS compatible application credentials in Keystone cls.ec2_creds = ks_client.ec2.create(user_id, project_id) - def test_upload_external_cert(self): - """Verify that the external CA is uploaded correctly.""" - logging.info('Changing value for trusted-external-ca-cert.') - ca_cert_option = 'trusted-external-ca-cert' - ppk, cert = utilities.cert.generate_cert('gnocchi_test.ci.local') - b64_cert = base64.b64encode(cert).decode() - config = { - ca_cert_option: b64_cert, - } - model.set_application_config( - 'gnocchi', - config - ) - model.block_until_all_units_idle() - - cert_location = '/usr/local/share/ca-certificates' - cert_name = 'gnocchi-external.crt' - cmd = 'ls ' + cert_location + '/' + cert_name - logging.info("Validating that the file {} is created in \ - {}".format(cert_name, cert_location)) - result = model.run_on_unit('gnocchi/0', cmd) - self.assertEqual(result['Code'], '0') - - linked_cert_location = '/etc/ssl/certs' - linked_cert_name = 'gnocchi-external.pem' - cmd = 'ls ' + linked_cert_location + '/' + linked_cert_name - logging.info("Validating that the link {} is created in \ - {}".format(linked_cert_name, linked_cert_location)) - result = model.run_on_unit('gnocchi/0', cmd) - self.assertEqual(result['Code'], '0') - def test_s3_list_gnocchi_buckets(self): """Verify that the gnocchi buckets were created in the S3 backend.""" kwargs = { From cf04bbfc7dec010bac9e08126d3b176594ea47ed Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Thu, 20 Aug 2020 11:02:07 -0500 Subject: [PATCH 647/898] move new test in separate class --- zaza/openstack/charm_tests/gnocchi/tests.py | 56 +++++++++++---------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index e3b9ae1..808f374 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -65,32 +65,8 @@ class GnocchiTest(test_utils.OpenStackBaseTest): logging.info("Testing pause and resume") -class GnocchiS3Test(test_utils.OpenStackBaseTest): - """Test Gnocchi for S3 storage backend.""" - - @classmethod - def setUpClass(cls): - """Run class setup for running tests.""" - super(GnocchiS3Test, cls).setUpClass() - - session = openstack_utils.get_overcloud_keystone_session() - ks_client = openstack_utils.get_keystone_session_client(session) - - # Get token data so we can glean our user_id and project_id - token_data = ks_client.tokens.get_token_data(session.get_token()) - project_id = token_data['token']['project']['id'] - user_id = token_data['token']['user']['id'] - - # Store URL to service providing S3 compatible API - for entry in token_data['token']['catalog']: - if entry['type'] == 's3': - for endpoint in entry['endpoints']: - if endpoint['interface'] == 'public': - cls.s3_region = endpoint['region'] - cls.s3_endpoint = endpoint['url'] - - # Create AWS compatible application credentials in Keystone - cls.ec2_creds = ks_client.ec2.create(user_id, project_id) +class GnocchiExternalCATest(test_utils.OpenStackBaseTest): + """Test Gnocchi for external root CA config option.""" def test_upload_external_cert(self): """Verify that the external CA is uploaded correctly.""" @@ -123,6 +99,34 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): result = model.run_on_unit('gnocchi/0', cmd) self.assertEqual(result['Code'], '0') + +class GnocchiS3Test(test_utils.OpenStackBaseTest): + """Test Gnocchi for S3 storage backend.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(GnocchiS3Test, cls).setUpClass() + + session = openstack_utils.get_overcloud_keystone_session() + ks_client = openstack_utils.get_keystone_session_client(session) + + # Get token data so we can glean our user_id and project_id + token_data = ks_client.tokens.get_token_data(session.get_token()) + project_id = token_data['token']['project']['id'] + user_id = token_data['token']['user']['id'] + + # Store URL to service providing S3 compatible API + for entry in token_data['token']['catalog']: + if entry['type'] == 's3': + for endpoint in entry['endpoints']: + if endpoint['interface'] == 'public': + cls.s3_region = endpoint['region'] + cls.s3_endpoint = endpoint['url'] + + # Create AWS compatible application credentials in Keystone + cls.ec2_creds = ks_client.ec2.create(user_id, project_id) + def test_s3_list_gnocchi_buckets(self): """Verify that the gnocchi buckets were created in the S3 backend.""" kwargs = { From 6d0a59802bd676f4b8565bc005dc94c6cc97ed57 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Thu, 20 Aug 2020 11:35:38 -0500 Subject: [PATCH 648/898] fix merge conflict --- zaza/openstack/charm_tests/gnocchi/tests.py | 73 ++++++++++----------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 808f374..5147486 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -16,15 +16,12 @@ """Encapsulate Gnocchi testing.""" -import base64 import boto3 import logging import pprint from gnocchiclient.v1 import client as gnocchi_client -import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.utilities as utilities import zaza.openstack.utilities.openstack as openstack_utils @@ -65,41 +62,6 @@ class GnocchiTest(test_utils.OpenStackBaseTest): logging.info("Testing pause and resume") -class GnocchiExternalCATest(test_utils.OpenStackBaseTest): - """Test Gnocchi for external root CA config option.""" - - def test_upload_external_cert(self): - """Verify that the external CA is uploaded correctly.""" - logging.info('Changing value for trusted-external-ca-cert.') - ca_cert_option = 'trusted-external-ca-cert' - ppk, cert = utilities.cert.generate_cert('gnocchi_test.ci.local') - b64_cert = base64.b64encode(cert).decode() - config = { - ca_cert_option: b64_cert, - } - model.set_application_config( - 'gnocchi', - config - ) - model.block_until_all_units_idle() - - cert_location = '/usr/local/share/ca-certificates' - cert_name = 'gnocchi-external.crt' - cmd = 'ls ' + cert_location + '/' + cert_name - logging.info("Validating that the file {} is created in \ - {}".format(cert_name, cert_location)) - result = model.run_on_unit('gnocchi/0', cmd) - self.assertEqual(result['Code'], '0') - - linked_cert_location = '/etc/ssl/certs' - linked_cert_name = 'gnocchi-external.pem' - cmd = 'ls ' + linked_cert_location + '/' + linked_cert_name - logging.info("Validating that the link {} is created in \ - {}".format(linked_cert_name, linked_cert_location)) - result = model.run_on_unit('gnocchi/0', cmd) - self.assertEqual(result['Code'], '0') - - class GnocchiS3Test(test_utils.OpenStackBaseTest): """Test Gnocchi for S3 storage backend.""" @@ -148,3 +110,38 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): break else: AssertionError('Bucket "{}" not found'.format(gnocchi_bkt)) + + +class GnocchiExternalCATest(test_utils.OpenStackBaseTest): + """Test Gnocchi for external root CA config option.""" + + def test_upload_external_cert(self): + """Verify that the external CA is uploaded correctly.""" + logging.info('Changing value for trusted-external-ca-cert.') + ca_cert_option = 'trusted-external-ca-cert' + ppk, cert = utilities.cert.generate_cert('gnocchi_test.ci.local') + b64_cert = base64.b64encode(cert).decode() + config = { + ca_cert_option: b64_cert, + } + model.set_application_config( + 'gnocchi', + config + ) + model.block_until_all_units_idle() + + cert_location = '/usr/local/share/ca-certificates' + cert_name = 'gnocchi-external.crt' + cmd = 'ls ' + cert_location + '/' + cert_name + logging.info("Validating that the file {} is created in \ + {}".format(cert_name, cert_location)) + result = model.run_on_unit('gnocchi/0', cmd) + self.assertEqual(result['Code'], '0') + + linked_cert_location = '/etc/ssl/certs' + linked_cert_name = 'gnocchi-external.pem' + cmd = 'ls ' + linked_cert_location + '/' + linked_cert_name + logging.info("Validating that the link {} is created in \ + {}".format(linked_cert_name, linked_cert_location)) + result = model.run_on_unit('gnocchi/0', cmd) + self.assertEqual(result['Code'], '0') \ No newline at end of file From 1dc8ee4287f3d7237f4478f699deac216be707fc Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Thu, 20 Aug 2020 11:37:03 -0500 Subject: [PATCH 649/898] add imports --- zaza/openstack/charm_tests/gnocchi/tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 5147486..7bd4523 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -16,12 +16,15 @@ """Encapsulate Gnocchi testing.""" +import base64 import boto3 import logging import pprint from gnocchiclient.v1 import client as gnocchi_client +import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities as utilities import zaza.openstack.utilities.openstack as openstack_utils @@ -144,4 +147,4 @@ class GnocchiExternalCATest(test_utils.OpenStackBaseTest): logging.info("Validating that the link {} is created in \ {}".format(linked_cert_name, linked_cert_location)) result = model.run_on_unit('gnocchi/0', cmd) - self.assertEqual(result['Code'], '0') \ No newline at end of file + self.assertEqual(result['Code'], '0') From 28de49811e556c3963206929d30d4c5c686e2950 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Thu, 20 Aug 2020 13:47:12 -0500 Subject: [PATCH 650/898] test block function instead of ls --- zaza/openstack/charm_tests/gnocchi/tests.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 7bd4523..73c32a9 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -133,13 +133,15 @@ class GnocchiExternalCATest(test_utils.OpenStackBaseTest): ) model.block_until_all_units_idle() - cert_location = '/usr/local/share/ca-certificates' - cert_name = 'gnocchi-external.crt' - cmd = 'ls ' + cert_location + '/' + cert_name - logging.info("Validating that the file {} is created in \ - {}".format(cert_name, cert_location)) - result = model.run_on_unit('gnocchi/0', cmd) - self.assertEqual(result['Code'], '0') + # cert_location = '/usr/local/share/ca-certificates' + # cert_name = 'gnocchi-external.crt' + # cmd = 'ls ' + cert_location + '/' + cert_name + # logging.info("Validating that the file {} is created in \ + # {}".format(cert_name, cert_location)) + # result = model.run_on_unit('gnocchi/0', cmd) + remote_file = '/usr/local/share/ca-certificates/gnocchi-external.crt' + result = model.block_until_file_ready('gnocchi', remote_file, b64_cert) + self.assertTrue(result) linked_cert_location = '/etc/ssl/certs' linked_cert_name = 'gnocchi-external.pem' From cece0d3096fff512303c23de10f611e1e27be768 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Thu, 20 Aug 2020 13:47:12 -0500 Subject: [PATCH 651/898] test block function instead of ls --- zaza/openstack/charm_tests/gnocchi/tests.py | 36 ++++++++++++--------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index 7bd4523..b099fa6 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -76,7 +76,7 @@ class GnocchiS3Test(test_utils.OpenStackBaseTest): session = openstack_utils.get_overcloud_keystone_session() ks_client = openstack_utils.get_keystone_session_client(session) - # Get token data so we can glean our user_id and project_id + # Get token data so we can clean our user_id and project_id token_data = ks_client.tokens.get_token_data(session.get_token()) project_id = token_data['token']['project']['id'] user_id = token_data['token']['user']['id'] @@ -133,18 +133,24 @@ class GnocchiExternalCATest(test_utils.OpenStackBaseTest): ) model.block_until_all_units_idle() - cert_location = '/usr/local/share/ca-certificates' - cert_name = 'gnocchi-external.crt' - cmd = 'ls ' + cert_location + '/' + cert_name - logging.info("Validating that the file {} is created in \ - {}".format(cert_name, cert_location)) - result = model.run_on_unit('gnocchi/0', cmd) - self.assertEqual(result['Code'], '0') + # cert_location = '/usr/local/share/ca-certificates' + # cert_name = 'gnocchi-external.crt' + # cmd = 'ls ' + cert_location + '/' + cert_name + # logging.info("Validating that the file {} is created in \ + # {}".format(cert_name, cert_location)) + # result = model.run_on_unit('gnocchi/0', cmd) + remote_file = '/usr/local/share/ca-certificates/gnocchi-external.crt' + logging.info("Validating that {} is created.".format(remote_file)) + result = model.block_until_file_ready('gnocchi', remote_file, b64_cert) + self.assertTrue(result) - linked_cert_location = '/etc/ssl/certs' - linked_cert_name = 'gnocchi-external.pem' - cmd = 'ls ' + linked_cert_location + '/' + linked_cert_name - logging.info("Validating that the link {} is created in \ - {}".format(linked_cert_name, linked_cert_location)) - result = model.run_on_unit('gnocchi/0', cmd) - self.assertEqual(result['Code'], '0') + # linked_cert_location = '/etc/ssl/certs' + # linked_cert_name = 'gnocchi-external.pem' + # cmd = 'ls ' + linked_cert_location + '/' + linked_cert_name + # logging.info("Validating that the link {} is created in \ + # {}".format(linked_cert_name, linked_cert_location)) + # result = model.run_on_unit('gnocchi/0', cmd) + remote_linked_file = '/etc/ssl/certs/gnocchi-external.pem' + result = model.block_until_file_ready('gnocchi', remote_linked_file, b64_cert) + # self.assertEqual(result['Code'], '0') + self.assertTrue(result) From 6b822ee8ecec179ad5d888666d79f2decbc48172 Mon Sep 17 00:00:00 2001 From: "camille.rodriguez" Date: Thu, 20 Aug 2020 16:03:04 -0500 Subject: [PATCH 652/898] replace ls with file_has_contents --- zaza/openstack/charm_tests/gnocchi/tests.py | 28 ++++++--------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/zaza/openstack/charm_tests/gnocchi/tests.py b/zaza/openstack/charm_tests/gnocchi/tests.py index b099fa6..ed4bac0 100644 --- a/zaza/openstack/charm_tests/gnocchi/tests.py +++ b/zaza/openstack/charm_tests/gnocchi/tests.py @@ -133,24 +133,12 @@ class GnocchiExternalCATest(test_utils.OpenStackBaseTest): ) model.block_until_all_units_idle() - # cert_location = '/usr/local/share/ca-certificates' - # cert_name = 'gnocchi-external.crt' - # cmd = 'ls ' + cert_location + '/' + cert_name - # logging.info("Validating that the file {} is created in \ - # {}".format(cert_name, cert_location)) - # result = model.run_on_unit('gnocchi/0', cmd) - remote_file = '/usr/local/share/ca-certificates/gnocchi-external.crt' - logging.info("Validating that {} is created.".format(remote_file)) - result = model.block_until_file_ready('gnocchi', remote_file, b64_cert) - self.assertTrue(result) + files = [ + '/usr/local/share/ca-certificates/gnocchi-external.crt', + '/etc/ssl/certs/gnocchi-external.pem', + ] - # linked_cert_location = '/etc/ssl/certs' - # linked_cert_name = 'gnocchi-external.pem' - # cmd = 'ls ' + linked_cert_location + '/' + linked_cert_name - # logging.info("Validating that the link {} is created in \ - # {}".format(linked_cert_name, linked_cert_location)) - # result = model.run_on_unit('gnocchi/0', cmd) - remote_linked_file = '/etc/ssl/certs/gnocchi-external.pem' - result = model.block_until_file_ready('gnocchi', remote_linked_file, b64_cert) - # self.assertEqual(result['Code'], '0') - self.assertTrue(result) + for file in files: + logging.info("Validating that {} is created.".format(file)) + model.block_until_file_has_contents('gnocchi', file, 'CERTIFICATE') + logging.info("Found {} successfully.".format(file)) From 0ca011afecc86bccac819ee80eaf8625e9ff886d Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 20 Aug 2020 14:28:18 -0700 Subject: [PATCH 653/898] Ceph rados benchmarking --- .../charm_tests/ceph/benchmarking/__init__.py | 15 +++ .../charm_tests/ceph/benchmarking/tests.py | 124 ++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 zaza/openstack/charm_tests/ceph/benchmarking/__init__.py create mode 100644 zaza/openstack/charm_tests/ceph/benchmarking/tests.py diff --git a/zaza/openstack/charm_tests/ceph/benchmarking/__init__.py b/zaza/openstack/charm_tests/ceph/benchmarking/__init__.py new file mode 100644 index 0000000..74fd9bd --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/benchmarking/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for benchmarking ceph.""" diff --git a/zaza/openstack/charm_tests/ceph/benchmarking/tests.py b/zaza/openstack/charm_tests/ceph/benchmarking/tests.py new file mode 100644 index 0000000..9bbf60e --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/benchmarking/tests.py @@ -0,0 +1,124 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Ceph Benchmark Tests.""" + +import logging +import re +import unittest + +import zaza.model + + +class BenchmarkTests(unittest.TestCase): + """Ceph Bencharmk Tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph benchmark tests.""" + super().setUpClass() + cls.results_match = "^[A-Z].*" + cls.pool = "zaza_benchmarks" + cls.test_results = {} + cls.time_in_secs = 30 + + def parse_bench_results(self, results_string): + """Parse bench results from string. + + :param results string: Output from rados bench command. + With newlines due to juju run's output. + :type results_string: string + :returns: Dictionary of results summary + :rtype: dict + """ + _results = {} + _lines = results_string.split("\n") + for _line in _lines: + _line = _line.strip() + if re.match(self.results_match, _line): + _keyvalues = _line.split(":") + try: + _results[_keyvalues[0].strip()] = _keyvalues[1].strip() + except IndexError: + # Skipping detailed output for summary details + pass + return _results + + def run_rados_bench(self, action, params=None): + """Run rados bench. + + :param action: String rados bench command i.e. write, rand, seq + :type action: string + :param params: List of string extra parameters to rados bench command + :type params: List[strings] + :returns: Unit run dict result + :rtype: dict + """ + _cmd = "rados bench -p {} {} {}".format( + self.pool, self.time_in_secs, action) + if params: + _cmd += " " + _cmd += " ".join(params) + logging.info( + "Running '{}' for {} seconds ...".format(_cmd, self.time_in_secs)) + _result = zaza.model.run_on_leader( + "ceph-mon", _cmd, timeout=self.time_in_secs + 60) + return _result + + def test_001_create_pool(self): + """Create ceph pool.""" + _cmd = "ceph osd pool create {} 100 100".format(self.pool) + _result = zaza.model.run_on_leader( + "ceph-mon", _cmd) + if _result.get("Code") and not _result.get("Code").startswith('0'): + if "already exists" in _result.get("Stderr", ""): + logging.warning( + "Ceph osd pool {} already exits.".format(self.pool)) + else: + logging.error("Ceph osd pool create failed") + raise Exception(_result.get("Stderr", "")) + + def test_100_rados_bench_write(self): + """Rados bench write test.""" + _result = self.run_rados_bench("write", params=["--no-cleanup"]) + self.test_results["write"] = ( + self.parse_bench_results(_result.get("Stdout", ""))) + + def test_200_rados_bench_read_seq(self): + """Rados bench read sequential test.""" + _result = self.run_rados_bench("seq") + self.test_results["read_seq"] = ( + self.parse_bench_results(_result.get("Stdout", ""))) + + def test_300_rados_bench_read_rand(self): + """Rados bench read random test.""" + _result = self.run_rados_bench("rand") + self.test_results["read_rand"] = ( + self.parse_bench_results(_result.get("Stdout", ""))) + + def test_998_rados_cleanup(self): + """Cleanup rados bench data.""" + _cmd = "rados -p {} cleanup".format(self.pool) + _result = zaza.model.run_on_leader("ceph-mon", _cmd) + if _result.get("Code") and not _result.get("Code").startswith('0'): + logging.warning("rados cleanup failed") + + def test_999_print_rados_bench_results(self): + """Print rados bench results.""" + print("######## Begin Ceph Results ########") + for test, results in self.test_results.items(): + print("##### {} ######".format(test)) + for key, value in results.items(): + print("{}: {}".format(key, value)) + print("######## End Ceph Results ########") From 287cc778d4d4a6aad0cd117d0646fde058f0822f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Fri, 21 Aug 2020 11:24:11 +0100 Subject: [PATCH 654/898] Extend the timeouts for create / backup / snapshot test (#390) On OSCI it's variable whether the test will pass as it's very close to the timeout. Extend it to give the test a reliable chance to pass. --- zaza/openstack/charm_tests/cinder_backup/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index b3f7ce7..7d3442e 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -97,7 +97,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): self.cinder_client.volumes, cinder_vol.id, wait_iteration_max_time=180, - stop_after_attempt=15, + stop_after_attempt=30, expected_status='available', msg='Volume status wait') @@ -109,7 +109,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): self.cinder_client.backups, vol_backup.id, wait_iteration_max_time=180, - stop_after_attempt=15, + stop_after_attempt=30, expected_status='available', msg='Volume status wait') # Delete the volume From cac63150d93026774f134b64f28eb2c246d2d2e1 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 26 Aug 2020 12:59:40 +0200 Subject: [PATCH 655/898] Moved test to its own new class --- zaza/openstack/charm_tests/neutron/tests.py | 50 +++++++++++---------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 37a99db..b313488 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -147,29 +147,6 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): self.assertIn('qos', ovs_agent['configurations']['extensions']) - def test_800_ovs_bridges_are_managed_by_us(self): - """Checking OVS bridges' external-id. - - OVS bridges created by us should be marked as managed by us in their - external-id. See - http://docs.openvswitch.org/en/latest/topics/integration/ - """ - for unit in zaza.model.get_units(self._APP_NAME, - model_name=self.model_name): - for bridge_name in ('br-int', 'br-ex'): - logging.info( - 'Checking that the bridge {}:{}'.format( - unit.name, bridge_name - ) + ' is marked as managed by us' - ) - expected_external_id = 'charm-neutron-gateway=managed' - actual_external_id = zaza.model.run_on_unit( - unit.entity_id, - 'ovs-vsctl br-get-external-id {}'.format(bridge_name), - model_name=self.model_name - )['Stdout'].strip() - self.assertEqual(actual_external_id, expected_external_id) - def test_900_restart_on_config_change(self): """Checking restart happens on config change. @@ -612,6 +589,33 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): logging.info('Testing pause resume') +class NeutronOvsVsctlTest(NeutronPluginApiSharedTests): + """Test 'ovs-vsctl'-related functionality on Neutron charms.""" + + def test_800_ovs_bridges_are_managed_by_us(self): + """Checking OVS bridges' external-id. + + OVS bridges created by us should be marked as managed by us in their + external-id. See + http://docs.openvswitch.org/en/latest/topics/integration/ + """ + for unit in zaza.model.get_units(self.application_name, + model_name=self.model_name): + for bridge_name in ('br-int', 'br-ex'): + logging.info( + 'Checking that the bridge {}:{}'.format( + unit.name, bridge_name + ) + ' is marked as managed by us' + ) + expected_external_id = 'charm-neutron-gateway=managed' + actual_external_id = zaza.model.run_on_unit( + unit.entity_id, + 'ovs-vsctl br-get-external-id {}'.format(bridge_name), + model_name=self.model_name + )['Stdout'].strip() + self.assertEqual(actual_external_id, expected_external_id) + + class NeutronNetworkingBase(test_utils.OpenStackBaseTest): """Base for checking openstack instances have valid networking.""" From 52dc1354ffd170fcaa2fb802d3bb4cb32810fb77 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 26 Aug 2020 14:19:00 +0200 Subject: [PATCH 656/898] Fix setUpClass' signature --- zaza/openstack/charm_tests/neutron/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index b313488..62aa33b 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -34,9 +34,10 @@ import zaza.openstack.utilities.openstack as openstack_utils class NeutronPluginApiSharedTests(test_utils.OpenStackBaseTest): """Shared tests for Neutron Plugin API Charms.""" + @classmethod def setUpClass(cls): """Run class setup for running Neutron Openvswitch tests.""" - super(NeutronPluginApiSharedTests, cls).setUpClass() + super(NeutronPluginApiSharedTests, cls).setUpClass(cls) cls.current_os_release = openstack_utils.get_os_release() cls.bionic_stein = openstack_utils.get_os_release('bionic_stein') From fe82bc76f50265bd7b883235f2ec602fb615e36a Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 26 Aug 2020 16:34:28 +0200 Subject: [PATCH 657/898] Fix setUpClass' signature --- zaza/openstack/charm_tests/neutron/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 62aa33b..5d2f019 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -37,7 +37,7 @@ class NeutronPluginApiSharedTests(test_utils.OpenStackBaseTest): @classmethod def setUpClass(cls): """Run class setup for running Neutron Openvswitch tests.""" - super(NeutronPluginApiSharedTests, cls).setUpClass(cls) + super(NeutronPluginApiSharedTests, cls).setUpClass() cls.current_os_release = openstack_utils.get_os_release() cls.bionic_stein = openstack_utils.get_os_release('bionic_stein') @@ -109,7 +109,7 @@ class NeutronGatewayTest(NeutronPluginApiSharedTests): @classmethod def setUpClass(cls): """Run class setup for running Neutron Gateway tests.""" - super(NeutronGatewayTest, cls).setUpClass(cls) + super(NeutronGatewayTest, cls).setUpClass() cls.services = cls._get_services() # set up clients @@ -427,7 +427,7 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): @classmethod def setUpClass(cls): """Run class setup for running Neutron Openvswitch tests.""" - super(NeutronOpenvSwitchTest, cls).setUpClass(cls) + super(NeutronOpenvSwitchTest, cls).setUpClass() # set up client cls.neutron_client = ( From 573acf9d1c0e2490589a4246b90083fc4113fdc9 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 27 Aug 2020 11:54:42 +0200 Subject: [PATCH 658/898] Move new scaleback test to new test class --- zaza/openstack/charm_tests/hacluster/tests.py | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 3ae13d1..7715a4a 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -24,15 +24,19 @@ import zaza.openstack.configure.hacluster import zaza.utilities.juju as juju_utils -class HaclusterTest(test_utils.OpenStackBaseTest): - """hacluster tests.""" +class HaclusterBaseTest(test_utils.OpenStackBaseTest): + """Base class for hacluster tests.""" @classmethod def setUpClass(cls): """Run class setup for running hacluster tests.""" - super(HaclusterTest, cls).setUpClass() + super(HaclusterBaseTest, cls).setUpClass() cls.vip = os.environ.get("TEST_VIP00") + +class HaclusterTest(HaclusterBaseTest): + """hacluster tests.""" + def test_900_action_cleanup(self): """The services can be cleaned up.""" zaza.model.run_action_on_leader( @@ -71,22 +75,20 @@ class HaclusterTest(test_utils.OpenStackBaseTest): self._toggle_maintenance_and_wait('true') self._toggle_maintenance_and_wait('false') - def test_930_scaleback_bionic(self): + +class HaclusterScalebackTest(HaclusterBaseTest): + """hacluster scaleback tests.""" + + _PRINCIPLE_APP = 'keystone' + + def test_930_scaleback(self): """Remove a unit, recalculate quorum and add a new one.""" - principle_app = 'keystone' principle_units = zaza.model.get_status().applications[ - principle_app]['units'] + self._PRINCIPLE_APP]['units'] self.assertEqual(len(principle_units), 3) doomed_principle = sorted(principle_units.keys())[0] - series = juju_utils.get_machine_series( - principle_units[doomed_principle].machine) - if series != 'bionic': - logging.debug("noop - only run test in bionic") - logging.info('SKIP') - return - doomed_unit = juju_utils.get_subordinate_units( - [doomed_principle], charm_name='hac')[0] + [doomed_principle], charm_name=self.application_name)[0] logging.info('Pausing unit {}'.format(doomed_unit)) zaza.model.run_action( @@ -104,7 +106,7 @@ class HaclusterTest(test_utils.OpenStackBaseTest): logging.info('Removing {}'.format(doomed_principle)) zaza.model.destroy_unit( - principle_app, + self._PRINCIPLE_APP, doomed_principle, wait_disappear=True) logging.info('OK') @@ -116,25 +118,25 @@ class HaclusterTest(test_utils.OpenStackBaseTest): action_params={'i-really-mean-it': True}, raise_on_failure=True) - _states = { + expected_states = { self.application_name: { "workload-status": "blocked", "workload-status-message": "Insufficient peer units for ha cluster (require 3)" }, - 'keystone': { + self._PRINCIPLE_APP: { "workload-status": "blocked", "workload-status-message": "Database not initialised", }, } - zaza.model.wait_for_application_states(states=_states) + zaza.model.wait_for_application_states(states=expected_states) zaza.model.block_until_all_units_idle() logging.info('OK') logging.info('Adding a hacluster unit') - zaza.model.add_unit(principle_app, wait_appear=True) - _states = {self.application_name: { + zaza.model.add_unit(self._PRINCIPLE_APP, wait_appear=True) + expected_states = {self.application_name: { "workload-status": "active", "workload-status-message": "Unit is ready and clustered"}} - zaza.model.wait_for_application_states(states=_states) + zaza.model.wait_for_application_states(states=expected_states) logging.debug('OK') From e48da475a2fe4a4b7d6b210def38aacfee05b83e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 27 Aug 2020 12:57:46 +0200 Subject: [PATCH 659/898] No need to resume hacluster unit before scaling down --- zaza/openstack/charm_tests/hacluster/tests.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 7715a4a..28f5248 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -97,13 +97,6 @@ class HaclusterScalebackTest(HaclusterBaseTest): raise_on_failure=True) logging.info('OK') - logging.info('Resuming unit {}'.format(doomed_unit)) - zaza.model.run_action( - doomed_unit, - 'resume', - raise_on_failure=True) - logging.info('OK') - logging.info('Removing {}'.format(doomed_principle)) zaza.model.destroy_unit( self._PRINCIPLE_APP, From 3fb4380c66c510b14d44584329c034ffa6f915f5 Mon Sep 17 00:00:00 2001 From: James Page Date: Fri, 28 Aug 2020 09:19:16 +0100 Subject: [PATCH 660/898] Improve ceph proxy permissions check Update the cinder-ceph permissions check to look for permissions specific to the cinder-ceph usage of ceph; this ensures that if we add glance and nova to the bundle (which creates additional permissions) the existing test will continue to pass. --- zaza/openstack/charm_tests/ceph/tests.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index d20486f..45b4920 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -766,16 +766,23 @@ class CephProxyTest(unittest.TestCase): msg = 'cinder-ceph pool was not found upon querying ceph-mon/0' raise zaza_exceptions.CephPoolNotFound(msg) - expected = "pool=cinder-ceph, allow class-read " \ - "object_prefix rbd_children" + # Checking for cinder-ceph specific permissions makes + # the test more rugged when we add additional relations + # to ceph for other applications (such as glance and nova). + expected_permissions = [ + "allow rwx pool=cinder-ceph", + "allow class-read object_prefix rbd_children", + ] cmd = "sudo ceph auth get client.cinder-ceph" result = zaza_model.run_on_unit('ceph-mon/0', cmd) output = result.get('Stdout').strip() - if expected not in output: - msg = ('cinder-ceph pool restriction was not configured correctly.' - ' Found: {}'.format(output)) - raise zaza_exceptions.CephPoolNotConfigured(msg) + for expected in expected_permissions: + if expected not in output: + msg = ('cinder-ceph pool restriction ({}) was not' + ' configured correctly.' + ' Found: {}'.format(expected, output)) + raise zaza_exceptions.CephPoolNotConfigured(msg) class CephPrometheusTest(unittest.TestCase): From b72855ca96d15a11c43e8b56a63a7f968e91d847 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 28 Aug 2020 13:15:31 +0200 Subject: [PATCH 661/898] Make HaclusterScalebackTest runnable against any charm --- zaza/openstack/charm_tests/hacluster/tests.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 28f5248..76eced8 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -79,16 +79,18 @@ class HaclusterTest(HaclusterBaseTest): class HaclusterScalebackTest(HaclusterBaseTest): """hacluster scaleback tests.""" - _PRINCIPLE_APP = 'keystone' + _PRINCIPLE_APP_NAME = 'keystone' + _HACLUSTER_APP_NAME = 'hacluster' + _HACLUSTER_CHARM_NAME = 'hacluster' def test_930_scaleback(self): """Remove a unit, recalculate quorum and add a new one.""" principle_units = zaza.model.get_status().applications[ - self._PRINCIPLE_APP]['units'] + self._PRINCIPLE_APP_NAME]['units'] self.assertEqual(len(principle_units), 3) doomed_principle = sorted(principle_units.keys())[0] doomed_unit = juju_utils.get_subordinate_units( - [doomed_principle], charm_name=self.application_name)[0] + [doomed_principle], charm_name=self._HACLUSTER_CHARM_NAME)[0] logging.info('Pausing unit {}'.format(doomed_unit)) zaza.model.run_action( @@ -99,25 +101,25 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('Removing {}'.format(doomed_principle)) zaza.model.destroy_unit( - self._PRINCIPLE_APP, + self._PRINCIPLE_APP_NAME, doomed_principle, wait_disappear=True) logging.info('OK') logging.info('Updating corosync ring') zaza.model.run_action_on_leader( - self.application_name, + self._HACLUSTER_APP_NAME, 'update-ring', action_params={'i-really-mean-it': True}, raise_on_failure=True) expected_states = { - self.application_name: { + self._HACLUSTER_APP_NAME: { "workload-status": "blocked", "workload-status-message": "Insufficient peer units for ha cluster (require 3)" }, - self._PRINCIPLE_APP: { + self._PRINCIPLE_APP_NAME: { "workload-status": "blocked", "workload-status-message": "Database not initialised", }, @@ -127,8 +129,8 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('OK') logging.info('Adding a hacluster unit') - zaza.model.add_unit(self._PRINCIPLE_APP, wait_appear=True) - expected_states = {self.application_name: { + zaza.model.add_unit(self._PRINCIPLE_APP_NAME, wait_appear=True) + expected_states = {self._HACLUSTER_APP_NAME: { "workload-status": "active", "workload-status-message": "Unit is ready and clustered"}} zaza.model.wait_for_application_states(states=expected_states) From dd2802057f7512b4ee537dfb09762c3992460835 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 28 Aug 2020 13:55:42 +0200 Subject: [PATCH 662/898] Ensure that octavia sets up application_name When running in a deployment that is focused on bundles rather than a specific charm, the Octavia setup can fail because there is no configured charm_name in the tests.yaml. This change ensures that the Octavia setup can continue. --- zaza/openstack/charm_tests/octavia/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/octavia/setup.py b/zaza/openstack/charm_tests/octavia/setup.py index 729fb16..677c368 100644 --- a/zaza/openstack/charm_tests/octavia/setup.py +++ b/zaza/openstack/charm_tests/octavia/setup.py @@ -92,7 +92,7 @@ def configure_octavia(): del test_config['target_deploy_status']['octavia'] _singleton = zaza.openstack.charm_tests.test_utils.OpenStackBaseTest() - _singleton.setUpClass() + _singleton.setUpClass(application_name='octavia') with _singleton.config_change(cert_config, cert_config): # wait for configuration to be applied then return pass From 57c8e0d91ab31c7f7bb347bec1cf7c7decc59144 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Sat, 29 Aug 2020 19:15:11 +0200 Subject: [PATCH 663/898] Retry on failures to download Arista image In the pre-deploy step of our neutron-arista tests we download an Arista image from our swift bucket. It sometimes fails when our CI system is loaded. This patch adds a tenacity retry. --- zaza/openstack/charm_tests/neutron_arista/setup.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron_arista/setup.py b/zaza/openstack/charm_tests/neutron_arista/setup.py index 4e35d33..f439abd 100644 --- a/zaza/openstack/charm_tests/neutron_arista/setup.py +++ b/zaza/openstack/charm_tests/neutron_arista/setup.py @@ -45,9 +45,15 @@ def download_arista_image(): if os.environ['TEST_ARISTA_IMAGE_REMOTE']: logging.info('Downloading Arista image from {}' .format(os.environ['TEST_ARISTA_IMAGE_REMOTE'])) - openstack_utils.download_image( - os.environ['TEST_ARISTA_IMAGE_REMOTE'], - os.environ['TEST_ARISTA_IMAGE_LOCAL']) + + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + reraise=True): + with attempt: + openstack_utils.download_image( + os.environ['TEST_ARISTA_IMAGE_REMOTE'], + os.environ['TEST_ARISTA_IMAGE_LOCAL']) + except KeyError: # TEST_ARISTA_IMAGE_REMOTE isn't set, which means the image is already # available at TEST_ARISTA_IMAGE_LOCAL From b38c270036a3e3efeb4d18c288786439fd2301b1 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 2 Sep 2020 11:51:04 +0200 Subject: [PATCH 664/898] Adapt list of swift services on focal-victoria --- zaza/openstack/charm_tests/swift/tests.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index 75f82b7..fa73c16 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -114,6 +114,17 @@ class SwiftStorageTests(test_utils.OpenStackBaseTest): 'swift-object-replicator', 'swift-object-updater', 'swift-container-sync'] + + focal_victoria = openstack_utils.get_os_release('focal_victoria') + if self.current_os_release < focal_victoria: + services += ['swift-account-replicator', + 'swift-container-replicator', + 'swift-object-replicator'] + else: + services += ['swift-account-server', + 'swift-container-server', + 'swift-object-server'] + with self.pause_resume(services): logging.info("Testing pause resume") From 23c3dde105b481ef34749132f993d4452c56e53a Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 2 Sep 2020 13:40:08 +0200 Subject: [PATCH 665/898] Fix SwiftStorageTests.test_901_pause_resume --- zaza/openstack/charm_tests/swift/tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/swift/tests.py b/zaza/openstack/charm_tests/swift/tests.py index fa73c16..62fe02c 100644 --- a/zaza/openstack/charm_tests/swift/tests.py +++ b/zaza/openstack/charm_tests/swift/tests.py @@ -104,19 +104,17 @@ class SwiftStorageTests(test_utils.OpenStackBaseTest): services = ['swift-account-server', 'swift-account-auditor', 'swift-account-reaper', - 'swift-account-replicator', 'swift-container-server', 'swift-container-auditor', - 'swift-container-replicator', 'swift-container-updater', 'swift-object-server', 'swift-object-auditor', - 'swift-object-replicator', 'swift-object-updater', 'swift-container-sync'] + current_os_release = openstack_utils.get_os_release() focal_victoria = openstack_utils.get_os_release('focal_victoria') - if self.current_os_release < focal_victoria: + if current_os_release < focal_victoria: services += ['swift-account-replicator', 'swift-container-replicator', 'swift-object-replicator'] From 828883bb70b3d786cece6a467f93b2b03d52cc30 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 3 Sep 2020 10:06:18 +0100 Subject: [PATCH 666/898] Make test_check_pool_types handle missing apps (#405) Currently test_check_pool_types will fail when it tries to check the corresponding pool type for an application that is not present in the deployment, this patch changes the behaviour to skip missing applications. --- zaza/openstack/charm_tests/ceph/tests.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 45b4920..001fc0f 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -818,8 +818,15 @@ class CheckPoolTypes(unittest.TestCase): ('cinder-ceph', 'cinder-ceph')] runtime_pool_details = zaza_ceph.get_ceph_pool_details() for app, pool_name in app_pools: - juju_pool_config = zaza_model.get_application_config(app).get( - 'pool-type') + try: + app_config = zaza_model.get_application_config(app) + except KeyError: + logging.info( + 'Skipping pool check of %s, application %s not present', + pool_name, + app) + continue + juju_pool_config = app_config.get('pool-type') if juju_pool_config: expected_pool_type = juju_pool_config['value'] else: From 66d95d65d7637b6b4f56c575ed552f13ebfa02b3 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Fri, 4 Sep 2020 12:33:42 +0100 Subject: [PATCH 667/898] Fix series-upgrade tests (#406) A couple of changes here: * Ensure that the post-upgrade-hook runs BEFORE the config-changed to set the openstack-origin/source back to distro. The former behaviour breaks keystone quite badly. * Ensure that the charm name is used, as discovered from the model, for rabbitmq-server and percona-cluster to cope with different names for the application in the model vs the charm name from the charm-store. * Check whether the machine needs to be rebooted after the dist-upgrade (before the do-release-upgrade), and reboot the machine first. Otherwise, do-release-upgrade will fail. --- requirements.txt | 2 +- .../charm_tests/series_upgrade/tests.py | 4 +- zaza/openstack/utilities/series_upgrade.py | 38 +++++++++++++++++-- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index fa0e460..8b15edc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiounittest async_generator boto3 -juju +juju!=2.8.3 # blacklist 2.8.3 as it appears to have a connection bug juju_wait PyYAML<=4.2,>=3.0 flake8>=2.2.4 diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index 4b06dbb..c595448 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -81,7 +81,7 @@ class SeriesUpgradeTest(unittest.TestCase): logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( - 'rabbitmq-server', + charm_name, 'complete-cluster-series-upgrade', action_params={}) model.block_until_all_units_idle() @@ -90,7 +90,7 @@ class SeriesUpgradeTest(unittest.TestCase): logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( - 'mysql', + charm_name, 'complete-cluster-series-upgrade', action_params={}) model.block_until_all_units_idle() diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index f12f77b..97ba153 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -14,11 +14,13 @@ """Collection of functions for testing series upgrade.""" +import asyncio import collections import copy import concurrent import logging import os +import time from zaza import model from zaza.charm_lifecycle import utils as cl_utils @@ -642,14 +644,14 @@ def series_upgrade(unit_name, machine_num, model.block_until_unit_wl_status(unit_name, "blocked") logging.info("Waiting for model idleness") model.block_until_all_units_idle() + logging.info("Complete series upgrade on {}".format(machine_num)) + model.complete_series_upgrade(machine_num) + model.block_until_all_units_idle() logging.info("Set origin on {}".format(application)) # Allow for charms which have neither source nor openstack-origin if origin: os_utils.set_origin(application, origin) model.block_until_all_units_idle() - logging.info("Complete series upgrade on {}".format(machine_num)) - model.complete_series_upgrade(machine_num) - model.block_until_all_units_idle() logging.info("Running run_post_upgrade_functions {}".format( post_upgrade_functions)) run_post_upgrade_functions(post_upgrade_functions) @@ -882,6 +884,21 @@ def dist_upgrade(unit_name): """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") model.run_on_unit(unit_name, dist_upgrade_cmd) + rdict = model.run_on_unit(unit_name, "cat /var/run/reboot-required") + if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): + logging.info("dist-upgrade required reboot {}".format(unit_name)) + os_utils.reboot(unit_name) + logging.info("Waiting for workload status 'unknown' on {}" + .format(unit_name)) + model.block_until_unit_wl_status(unit_name, "unknown") + logging.info("Waiting for workload status to return to normal on {}" + .format(unit_name)) + model.block_until_unit_wl_status( + unit_name, "unknown", negate_match=True) + logging.info("Waiting for model idleness") + # pause for a big + time.sleep(5.0) + model.block_until_all_units_idle() async def async_dist_upgrade(unit_name): @@ -902,6 +919,21 @@ async def async_dist_upgrade(unit_name): """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") await model.async_run_on_unit(unit_name, dist_upgrade_cmd) + rdict = await model.async_run_on_unit(unit_name, + "cat /var/run/reboot-required") + if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): + logging.info("dist-upgrade required reboot {}".format(unit_name)) + await os_utils.async_reboot(unit_name) + logging.info("Waiting for workload status 'unknown' on {}" + .format(unit_name)) + await model.async_block_until_unit_wl_status(unit_name, "unknown") + logging.info("Waiting for workload status to return to normal on {}" + .format(unit_name)) + await model.async_block_until_unit_wl_status( + unit_name, "unknown", negate_match=True) + logging.info("Waiting for model idleness") + await asyncio.sleep(5.0) + await model.async_block_until_all_units_idle() def do_release_upgrade(unit_name): From fe329c182453de93584b88469514c3dee260abdc Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 1 Jul 2020 22:18:02 +0200 Subject: [PATCH 668/898] Add helper to retrieve tests_options for specific test This is done by building a prefix for options key from dot-notated absolute path to calling method or function. --- unit_tests/charm_tests/test_utils.py | 20 +++++++++++ zaza/openstack/charm_tests/test_utils.py | 45 ++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/unit_tests/charm_tests/test_utils.py b/unit_tests/charm_tests/test_utils.py index a415360..277d86c 100644 --- a/unit_tests/charm_tests/test_utils.py +++ b/unit_tests/charm_tests/test_utils.py @@ -18,6 +18,26 @@ import zaza.openstack.charm_tests.test_utils as test_utils from unittest.mock import patch +class TestBaseCharmTest(unittest.TestCase): + + def test_get_my_tests_options(self): + + class FakeTest(test_utils.BaseCharmTest): + + def method(self, test_config): + self.test_config = test_config + return self.get_my_tests_options('aKey', 'aDefault') + + f = FakeTest() + self.assertEquals(f.method({}), 'aDefault') + self.assertEquals(f.method({ + 'tests_options': { + 'unit_tests.charm_tests.test_utils.' + 'FakeTest.method.aKey': 'aValue', + }, + }), 'aValue') + + class TestOpenStackBaseTest(unittest.TestCase): @patch.object(test_utils.openstack_utils, 'get_cacert') diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 83595b0..e3ce508 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -16,6 +16,7 @@ import contextlib import logging import ipaddress import subprocess +import sys import tenacity import unittest @@ -426,6 +427,50 @@ class BaseCharmTest(unittest.TestCase): model_name=self.model_name, pgrep_full=pgrep_full) + def get_my_tests_options(self, key, default=None): + """Retrieve tests_options for specific test. + + Prefix for key is built from dot-notated absolute path to calling + method or function. + + Example: + # In tests.yaml: + tests_options: + zaza.charm_tests.noop.tests.NoopTest.test_foo.key: true + # called from zaza.charm_tests.noop.tests.NoopTest.test_foo() + >>> get_my_tests_options('key') + True + + :param key: Suffix for tests_options key. + :type key: str + :param default: Default value to return if key is not found. + :type default: any + :returns: Value associated with key in tests_options. + :rtype: any + """ + # note that we need to do this in-line otherwise we would get the path + # to ourself. I guess we could create a common method that would go two + # frames back, but that would be kind of useless for anyone else than + # this method. + caller_path = [] + + # get path to module + caller_path.append(sys.modules[ + sys._getframe().f_back.f_globals['__name__']].__name__) + + # attempt to get class name + try: + caller_path.append( + sys._getframe().f_back.f_locals['self'].__class__.__name__) + except KeyError: + pass + + # get method or function name + caller_path.append(sys._getframe().f_back.f_code.co_name) + + return self.test_config.get('tests_options', {}).get( + '.'.join(caller_path + [key]), default) + class OpenStackBaseTest(BaseCharmTest): """Generic helpers for testing OpenStack API charms.""" From f33d166c5e1e44d8b6dc75120855f2b4612da3bb Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 24 Jun 2020 10:57:31 +0200 Subject: [PATCH 669/898] neutron: Allow to run networking test multiple times Adjust NeutronNetworkingTest to optionally run tearDown and re-use existing instances on subsequent runs. tearDown is controlled through a key under the `tests_options` dictionary in tests.yaml. Useful for morphing a deployment and then validating connectivity for existing instances afterwards. --- zaza/openstack/charm_tests/neutron/tests.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 5d2f019..2747569 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -794,11 +794,22 @@ class NeutronNetworkingTest(NeutronNetworkingBase): """Ensure that openstack instances have valid networking.""" def test_instances_have_networking(self): - """Validate North/South and East/West networking.""" - self.launch_guests() + """Validate North/South and East/West networking. + + Tear down can optionally be disabled by setting the module path + + class name + run_tearDown key under the `tests_options` key in + tests.yaml. + + Abbreviated example: + ...charm_tests.neutron.tests.NeutronNetworkingTest.run_tearDown: false + """ instance_1, instance_2 = self.retrieve_guests() + if not all([instance_1, instance_2]): + self.launch_guests() + instance_1, instance_2 = self.retrieve_guests() self.check_connectivity(instance_1, instance_2) - self.run_resource_cleanup = True + self.run_resource_cleanup = self.get_my_tests_options( + 'run_resource_cleanup', True) class NeutronNetworkingVRRPTests(NeutronNetworkingBase): From 4d8c5091330ef2468c64386eac30cf6b890ca1cf Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 8 Jul 2020 22:06:39 +0200 Subject: [PATCH 670/898] Make the use of juju_wait configurable For compatibility with existing scenario tests the `configure_gateway_ext_port` helper currently make use of `juju_wait` when configuring the deployed cloud. This does not work well if the model you are testing has applications with non-standard workload status messaging. Allow to override the behaviour through config step options. --- zaza/openstack/utilities/openstack.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 14c708c..c5e2bfa 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -63,6 +63,9 @@ import tenacity import textwrap import urllib +import zaza +import zaza.charm_lifecycle.utils as lifecycle_utils + from zaza import model from zaza.openstack.utilities import ( exceptions, @@ -794,7 +797,15 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, # NOTE(fnordahl): We are stuck with juju_wait until we figure out how # to deal with all the non ['active', 'idle', 'Unit is ready.'] # workload/agent states and msgs that our mojo specs are exposed to. - juju_wait.wait(wait_for_workload=True) + if lifecycle_utils.get_config_options().get( + 'configure_gateway_ext_port_use_juju_wait', True): + juju_wait.wait(wait_for_workload=True) + else: + zaza.model.wait_for_agent_status() + test_config = zaza.charm_lifecycle.utils.get_charm_config( + fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get('target_deploy_status', {})) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), From 4ae1b39ed21a4f9eedc501ad609708a80646ae15 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 17 Jul 2020 13:41:53 +0200 Subject: [PATCH 671/898] neutron: Check for non-frag connectivity at MTU size --- zaza/openstack/charm_tests/neutron/tests.py | 131 +++++++++++++++++--- 1 file changed, 113 insertions(+), 18 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 2747569..bebe3b6 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -24,6 +24,8 @@ import copy import logging import tenacity +from neutronclient.common import exceptions as neutronexceptions + import zaza import zaza.openstack.charm_tests.nova.utils as nova_utils import zaza.openstack.charm_tests.test_utils as test_utils @@ -635,7 +637,8 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): def validate_instance_can_reach_other(self, instance_1, instance_2, - verify): + verify, + mtu=None): """ Validate that an instance can reach a fixed and floating of another. @@ -644,6 +647,12 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): :param instance_2: The instance to check networking from :type instance_2: nova_client.Server + + :param verify: callback to verify result + :type verify: callable + + :param mtu: Check that we can send non-fragmented packets of given size + :type mtu: Optional[int] """ floating_1 = floating_ips_from_instance(instance_1)[0] floating_2 = floating_ips_from_instance(instance_2)[0] @@ -653,19 +662,30 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): password = guest.boot_tests['bionic'].get('password') privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) - openstack_utils.ssh_command( - username, floating_1, 'instance-1', - 'ping -c 1 {}'.format(address_2), - password=password, privkey=privkey, verify=verify) + cmds = [ + 'ping -c 1', + ] + if mtu: + # the on-wire packet will be 28 bytes larger than the value + # provided to ping(8) -s parameter + packetsize = mtu - 28 + cmds.append( + 'ping -M do -s {} -c 1'.format(packetsize)) - openstack_utils.ssh_command( - username, floating_1, 'instance-1', - 'ping -c 1 {}'.format(floating_2), - password=password, privkey=privkey, verify=verify) + for cmd in cmds: + openstack_utils.ssh_command( + username, floating_1, 'instance-1', + '{} {}'.format(cmd, address_2), + password=password, privkey=privkey, verify=verify) + + openstack_utils.ssh_command( + username, floating_1, 'instance-1', + '{} {}'.format(cmd, floating_2), + password=password, privkey=privkey, verify=verify) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, stop=tenacity.stop_after_attempt(8)) - def validate_instance_can_reach_router(self, instance, verify): + def validate_instance_can_reach_router(self, instance, verify, mtu=None): """ Validate that an instance can reach it's primary gateway. @@ -676,6 +696,12 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): :param instance: The instance to check networking from :type instance: nova_client.Server + + :param verify: callback to verify result + :type verify: callable + + :param mtu: Check that we can send non-fragmented packets of given size + :type mtu: Optional[int] """ address = floating_ips_from_instance(instance)[0] @@ -683,9 +709,20 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): password = guest.boot_tests['bionic'].get('password') privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) - openstack_utils.ssh_command( - username, address, 'instance', 'ping -c 1 192.168.0.1', - password=password, privkey=privkey, verify=verify) + cmds = [ + 'ping -c 1', + ] + if mtu: + # the on-wire packet will be 28 bytes larger than the value + # provided to ping(8) -s parameter + packetsize = mtu - 28 + cmds.append( + 'ping -M do -s {} -c 1'.format(packetsize)) + + for cmd in cmds: + openstack_utils.ssh_command( + username, address, 'instance', '{} 192.168.0.1'.format(cmd), + password=password, privkey=privkey, verify=verify) @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60), reraise=True, stop=tenacity.stop_after_attempt(8), @@ -726,21 +763,67 @@ class NeutronNetworkingBase(test_utils.OpenStackBaseTest): assert agent['admin_state_up'] assert agent['alive'] + def effective_network_mtu(self, network_name): + """Retrieve effective MTU for a network. + + If the `instance-mtu` configuration option is set to a value lower than + the network MTU this method will return the value of that. Otherwise + Neutron's value for MTU on a network will be returned. + + :param network_name: Name of network to query + :type network_name: str + :returns: MTU for network + :rtype: int + """ + cfg_instance_mtu = None + for app in ('neutron-gateway', 'neutron-openvswitch'): + try: + cfg = zaza.model.get_application_config(app) + cfg_instance_mtu = int(cfg['instance-mtu']['value']) + break + except KeyError: + pass + + networks = self.neutron_client.show_network('', name=network_name) + network_mtu = int(next(iter(networks['networks']))['mtu']) + + if cfg_instance_mtu and cfg_instance_mtu < network_mtu: + logging.info('Using MTU from application "{}" config: {}' + .format(app, cfg_instance_mtu)) + return cfg_instance_mtu + else: + logging.info('Using MTU from network "{}": {}' + .format(network_name, network_mtu)) + return network_mtu + def check_connectivity(self, instance_1, instance_2): """Run North/South and East/West connectivity tests.""" def verify(stdin, stdout, stderr): """Validate that the SSH command exited 0.""" self.assertEqual(stdout.channel.recv_exit_status(), 0) + try: + mtu_1 = self.effective_network_mtu( + network_name_from_instance(instance_1)) + mtu_2 = self.effective_network_mtu( + network_name_from_instance(instance_2)) + mtu_min = min(mtu_1, mtu_2) + except neutronexceptions.NotFound: + # Older versions of OpenStack cannot look up network by name, just + # skip the check if that is the case. + mtu_1 = mtu_2 = mtu_min = None + # Verify network from 1 to 2 - self.validate_instance_can_reach_other(instance_1, instance_2, verify) + self.validate_instance_can_reach_other( + instance_1, instance_2, verify, mtu_min) # Verify network from 2 to 1 - self.validate_instance_can_reach_other(instance_2, instance_1, verify) + self.validate_instance_can_reach_other( + instance_2, instance_1, verify, mtu_min) # Validate tenant to external network routing - self.validate_instance_can_reach_router(instance_1, verify) - self.validate_instance_can_reach_router(instance_2, verify) + self.validate_instance_can_reach_router(instance_1, verify, mtu_1) + self.validate_instance_can_reach_router(instance_2, verify, mtu_2) def floating_ips_from_instance(instance): @@ -769,6 +852,17 @@ def fixed_ips_from_instance(instance): return ips_from_instance(instance, 'fixed') +def network_name_from_instance(instance): + """Retrieve name of primary network the instance is attached to. + + :param instance: The instance to fetch name of network from. + :type instance: nova_client.Server + :returns: Name of primary network the instance is attached to. + :rtype: str + """ + return next(iter(instance.addresses)) + + def ips_from_instance(instance, ip_type): """ Retrieve IPs of a certain type from an instance. @@ -786,7 +880,8 @@ def ips_from_instance(instance, ip_type): "Only 'floating' and 'fixed' are valid IP types to search for" ) return list([ - ip['addr'] for ip in instance.addresses['private'] + ip['addr'] for ip in instance.addresses[ + network_name_from_instance(instance)] if ip['OS-EXT-IPS:type'] == ip_type]) From 3c457e4fbb52f22efd427849200ee0fb7dc516f6 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 24 Jun 2020 10:59:40 +0200 Subject: [PATCH 672/898] Add OVS to OVN migration tests --- zaza/openstack/charm_tests/ovn/setup.py | 139 ++++++++++++++ zaza/openstack/charm_tests/ovn/tests.py | 233 ++++++++++++++++++++++++ 2 files changed, 372 insertions(+) create mode 100644 zaza/openstack/charm_tests/ovn/setup.py diff --git a/zaza/openstack/charm_tests/ovn/setup.py b/zaza/openstack/charm_tests/ovn/setup.py new file mode 100644 index 0000000..116cdb8 --- /dev/null +++ b/zaza/openstack/charm_tests/ovn/setup.py @@ -0,0 +1,139 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for configuring OVN tests.""" + +import logging + +import zaza + +import zaza.openstack.charm_tests.test_utils as test_utils + + +class _OVNSetupHelper(test_utils.BaseCharmTest): + """Helper class to get at the common `config_change` helper.""" + + @staticmethod + def _get_instance_mtu_from_global_physnet_mtu(): + """Calculate instance mtu from Neutron API global-physnet-mtu. + + :returns: Value for instance mtu after migration. + :rtype: int + """ + n_api_config = zaza.model.get_application_config('neutron-api') + + # NOTE: we would have to adjust this calculation if we use IPv6 tunnel + # endpoints + GENEVE_ENCAP_OVERHEAD = 38 + IP4_HEADER_SIZE = 20 + return int(n_api_config['global-physnet-mtu']['value']) - ( + GENEVE_ENCAP_OVERHEAD + IP4_HEADER_SIZE) + + def configure_ngw_novs(self): + """Configure n-ovs and n-gw units.""" + cfg = { + # To be able to successfully clean up after the Neutron agents we + # need to use the 'openvswitch' `firewall-driver`. + 'firewall-driver': 'openvswitch', + # To be able to have instances successfully survive the migration + # without communication issues we need to lower the MTU announced + # to instances prior to migration. + # + # NOTE: In a real world scenario the end user would configure the + # MTU at least 24 hrs prior to doing the migration to allow + # instances to reconfigure as they renew the DHCP lease. + # + # NOTE: For classic n-gw topologies the `instance-mtu` config + # is a NOOP on neutron-openvswitch units, but that is ok. + 'instance-mtu': self._get_instance_mtu_from_global_physnet_mtu() + } + apps = ('neutron-gateway', 'neutron-openvswitch') + for app in apps: + try: + zaza.model.get_application(app) + for k, v in cfg.items(): + logging.info('Setting `{}` to "{}" on "{}"...' + .format(k, v, app)) + with self.config_change(cfg, cfg, app): + # The intent here is to change the config and not restore + # it. We accomplish that by passing in the same value for + # default and alternate. + # + # The reason for using the `config_change` helper for this + # is that it already deals with all the permutations of + # config already being set etc and does not get into + # trouble if the test bundle already have the values we try + # to set. + continue + except KeyError: + pass + + def configure_ovn_mappings(self): + """Copy mappings from n-gw or n-ovs application.""" + dst_apps = ('ovn-dedicated-chassis', 'ovn-chassis') + src_apps = ('neutron-gateway', 'neutron-openvswitch') + ovn_cfg = {} + for app in src_apps: + try: + app_cfg = zaza.model.get_application_config(app) + ovn_cfg['bridge-interface-mappings'] = app_cfg[ + 'data-port']['value'] + ovn_cfg['ovn-bridge-mappings'] = app_cfg[ + 'bridge-mappings']['value'] + # Use values from neutron-gateway when present, otherwise use + # values from neutron-openvswitch + break + except KeyError: + pass + else: + raise RuntimeError( + 'None of the expected apps ({}) are present in the model.' + .format(src_apps) + ) + + for app in dst_apps: + try: + zaza.model.get_application(app) + for k, v in ovn_cfg.items(): + logging.info('Setting `{}` to "{}" on "{}"...' + .format(k, v, app)) + with self.config_change(ovn_cfg, ovn_cfg, app): + # Set values only on ovn-dedicated-chassis when present, + # otherwise we set them on ovn-chassis. + break + except KeyError: + pass + else: + raise RuntimeError( + 'None of the expected apps ({}) are present in the model.' + .format(dst_apps) + ) + + +def pre_migration_configuration(): + """Perform pre-migration configuration steps. + + NOTE: Doing the configuration post-deploy and after doing initial network + configuration is an important part of the test as we need to prove that our + end users would be successful in doing this in the wild. + """ + # we use a helper class to leverage common setup code and the + # `config_change` helper + helper = _OVNSetupHelper() + helper.setUpClass() + # Configure `firewall-driver` and `instance-mtu` on n-gw and n-ovs units. + helper.configure_ngw_novs() + # Copy mappings from n-gw or n-ovs application to ovn-dedicated-chassis or + # ovn-chassis. + helper.configure_ovn_mappings() diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 0708c67..3b07920 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -15,8 +15,13 @@ """Encapsulate OVN testing.""" import logging +import tenacity +import juju + +import zaza import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -71,3 +76,231 @@ class ChassisCharmOperationTest(BaseCharmOperationTest): cls.services = [ 'ovn-controller', ] + + +class OVSOVNMigrationTest(test_utils.BaseCharmTest): + """OVS to OVN migration tests.""" + + def setUp(self): + """Perform migration steps prior to validation.""" + super(OVSOVNMigrationTest, self).setUp() + # These steps here due to them having to be executed once and in a + # specific order prior to running any tests. The steps should still + # be idempotent if at all possible as a courtesy to anyone iterating + # on the test code. + try: + if self.one_time_init_done: + logging.debug('Skipping migration steps as they have already ' + 'run.') + return + except AttributeError: + logging.info('Performing migration steps.') + + # as we progress through the steps our target deploy status changes + # store it in the class instance so the individual methods can + # update when appropriate. + self.target_deploy_status = self.test_config.get( + 'target_deploy_status', {}) + + # Stop Neutron agents on hypervisors + self._pause_units('neutron-openvswitch') + + # Add the neutron-api-plugin-ovn subordinate which will make the + # `neutron-api-plugin-ovn` unit appear in the deployment. + # + # NOTE: The OVN drivers will not be activated until we change the + # value for the `manage-neutron-plugin-legacy-mode` config. + self._add_neutron_api_plugin_ovn_subordinate_relation() + + # Adjust MTU on overlay networks + # + # Prior to this the end user will already have lowered the MTU on their + # running instances through the use of the `instance-mtu` configuration + # option and manual reconfiguration of instances that do not use DHCP. + # + # We update the value for the MTU on the overlay networks at this point + # in time because: + # + # - Agents are paused and will not actually reconfigure the networks. + # + # - Making changes to non-Geneve networks are prohibited as soon as the + # OVN drivers are activated. + # + # - Get the correct MTU value into the OVN database on first sync. + # + # - This will be particularly important for any instances using + # stateless IPv6 autoconfiguration (SLAAC) as there is currently + # no config knob to feed MTU information into the legacy ML2+OVS + # `radvd` configuration or the native OVN RA. + # + # - Said instances will reconfigure their IPv6 MTU as soon as they + # receive an RA with correct MTU when OVN takes over control. + self._run_migrate_mtu_action() + + # Flip `manage-neutron-plugin-legacy-mode` to enable it + # + # NOTE(fnordahl): until we sync/repair the OVN DB this will make the + # `neutron-server` log errors. However we need the neutron unit to be + # unpaused while doing this to have the configuration rendered. The + # configuration is consumed by the `neutron-ovn-db-sync` tool. + self._configure_neutron_api() + + # Stop the Neutron server prior to OVN DB sync/repair + self._pause_units('neutron-api') + + # Sync the OVN DB + self._run_migrate_ovn_db_action() + # Perform the optional morphing of Neutron DB action + self._run_offline_neutron_morph_db_action() + self._resume_units('neutron-api') + + # Run `cleanup` action on neutron-openvswitch units/hypervisors + self._run_cleanup_action() + + # Start the OVN controller on hypervisors + # + # NOTE(fnordahl): it is very important to have run cleanup prior to + # starting these, if you don't do that it is almost guaranteed that + # you will program the network to a state of infinite loop. + self._resume_units('ovn-chassis') + + # And we should be off to the races + + self.one_time_init_done = True + + def _add_neutron_api_plugin_ovn_subordinate_relation(self): + try: + logging.info('Adding relation neutron-api-plugin-ovn ' + '-> neutron-api') + zaza.model.add_relation( + 'neutron-api-plugin-ovn', 'neutron-plugin', + 'neutron-api:neutron-plugin-api-subordinate') + zaza.model.wait_for_agent_status() + zaza.model.wait_for_application_states( + states=self.test_config.get('target_deploy_status', {})) + except juju.errors.JujuAPIError: + # we were not able to add the relation, let's make sure it's + # because it's already there + assert (zaza.model.get_relation_id( + 'neutron-api-plugin-ovn', 'neutron-api', + remote_interface_name='neutron-plugin-api-subordinate') + is not None), 'Unable to add relation required for test' + logging.info('--> On the other hand, did not need to add the ' + 'relation as it was already there.') + + def _configure_neutron_api(self): + """Set configuration option `manage-neutron-plugin-legacy-mode`.""" + logging.info('Configuring `manage-neutron-plugin-legacy-mode` for ' + 'neutron-api...') + n_api_config = { + 'manage-neutron-plugin-legacy-mode': False, + } + with self.config_change( + n_api_config, n_api_config, 'neutron-api'): + logging.info('done') + + def _run_offline_neutron_morph_db_action(self): + logging.info('Running the optional `offline-neutron-morph-db` action ' + 'on neutron-api-plugin-ovn/leader') + generic_utils.assertActionRanOK( + zaza.model.run_action_on_leader( + 'neutron-api-plugin-ovn', + 'offline-neutron-morph-db', + action_params={ + 'i-really-mean-it': True}, + raise_on_failure=True, + ) + ) + + def _run_migrate_ovn_db_action(self): + logging.info('Running `migrate-ovn-db` action on ' + 'neutron-api-plugin-ovn/leader') + generic_utils.assertActionRanOK( + zaza.model.run_action_on_leader( + 'neutron-api-plugin-ovn', + 'migrate-ovn-db', + action_params={ + 'i-really-mean-it': True}, + raise_on_failure=True, + ) + ) + + # Charm readiness is no guarantee for API being ready to serve requests. + # https://bugs.launchpad.net/charm-neutron-api/+bug/1854518 + @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60), + reraise=True, stop=tenacity.stop_after_attempt(3)) + def _run_migrate_mtu_action(self): + logging.info('Running `migrate-mtu` action on ' + 'neutron-api-plugin-ovn/leader') + generic_utils.assertActionRanOK( + zaza.model.run_action_on_leader( + 'neutron-api-plugin-ovn', + 'migrate-mtu', + action_params={ + 'i-really-mean-it': True}, + raise_on_failure=True, + ) + ) + + def _pause_units(self, application): + logging.info('Pausing {} units'.format(application)) + zaza.model.run_action_on_units( + [unit.entity_id + for unit in zaza.model.get_units(application)], + 'pause', + raise_on_failure=True, + ) + self.target_deploy_status.update( + { + application: { + 'workload-status': 'maintenance', + 'workload-status-message': 'Paused', + }, + }, + ) + + def _run_cleanup_action(self): + logging.info('Running `cleanup` action on neutron-openvswitch units.') + zaza.model.run_action_on_units( + [unit.entity_id + for unit in zaza.model.get_units('neutron-openvswitch')], + 'cleanup', + action_params={ + 'i-really-mean-it': True}, + raise_on_failure=True, + ) + + def _resume_units(self, application): + logging.info('Resuming {} units'.format(application)) + zaza.model.run_action_on_units( + [unit.entity_id + for unit in zaza.model.get_units(application)], + 'resume', + raise_on_failure=True, + ) + self.target_deploy_status.pop(application) + + def test_ovs_ovn_migration(self): + """Test migration of existing Neutron ML2+OVS deployment to OVN. + + The test should be run after deployment and validation of a legacy + deployment combined with subsequent run of a network connectivity test + on instances created prior to the migration. + """ + # The setUp method of this test class will perform the migration steps. + # The tests.yaml is programmed to do further validation after the + # migration. + + # Reset the n-gw and n-ovs instance-mtu configuration option so it does + # not influence how further tests are executed. + reset_config_keys = ['instance-mtu'] + for app in ('neutron-gateway', 'neutron-openvswitch'): + try: + zaza.model.reset_application_config(app, reset_config_keys) + logging.info('Reset configuration to default on "{}" for "{}"' + .format(app, reset_config_keys)) + except KeyError: + pass + zaza.model.wait_for_agent_status() + zaza.model.wait_for_application_states( + states=self.target_deploy_status) From bb9899650aca21092afadb41ec6442258d43a100 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 7 Sep 2020 12:34:55 +0000 Subject: [PATCH 673/898] Skip ceph pool type check if relation is missing --- zaza/openstack/charm_tests/ceph/tests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 001fc0f..99285a3 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -826,6 +826,15 @@ class CheckPoolTypes(unittest.TestCase): pool_name, app) continue + rel_id = zaza_model.get_relation_id( + app, + 'ceph-mon', + remote_interface_name='client') + if not rel_id: + logging.info( + 'Skipping pool check of %s, ceph relation not present', + app) + continue juju_pool_config = app_config.get('pool-type') if juju_pool_config: expected_pool_type = juju_pool_config['value'] From 86547a4efc44bde88d2e335b166c05d0d4c9c5c4 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 7 Sep 2020 15:09:14 +0200 Subject: [PATCH 674/898] Add more insights on trilio test failures (#400) * Add more insights on trilio test failures * Fix import * Fix wrong signature --- zaza/openstack/charm_tests/trilio/tests.py | 59 ++++++++++++++++++---- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 99ef3e0..991b7f5 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -21,15 +21,16 @@ import tenacity import zaza.model as zaza_model -import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.utilities.juju as juju_utils -import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.guest as guest_utils +import zaza.openstack.utilities.openstack as openstack_utils +from zaza.utilities import juju as juju_utils def _resource_reaches_status( - unit, auth_args, command, resource_id, target_status + unit, auth_args, status_command, full_status_command, resource_id, + target_status ): """Wait for a workload resource to reach a status. @@ -37,8 +38,12 @@ def _resource_reaches_status( :type unit: zaza_model.Unit :param auth_args: authentication arguments for command :type auth_args: str - :param command: command to execute - :type command: str + :param status_command: command to execute to get the resource status that + is expected to reach target_status + :type status_command: str + :param full_status_command: command to execute to get insights on why the + resource failed to reach target_status + :type full_status_command: str :param resource_id: resource ID to monitor :type resource_id: str :param target_status: status to monitor for @@ -47,7 +52,7 @@ def _resource_reaches_status( resource_status = ( juju_utils.remote_run( unit, - remote_cmd=command.format( + remote_cmd=status_command.format( auth_args=auth_args, resource_id=resource_id ), timeout=180, @@ -63,7 +68,20 @@ def _resource_reaches_status( ) if resource_status == target_status: return - raise Exception("Resource not ready: {}".format(resource_status)) + + full_resource_status = ( + juju_utils.remote_run( + unit, + remote_cmd=full_status_command.format( + auth_args=auth_args, resource_id=resource_id + ), + timeout=180, + fatal=True, + ) + .strip() + ) + + raise Exception("Resource not ready:\n{}".format(full_resource_status)) class WorkloadmgrCLIHelper(object): @@ -78,7 +96,12 @@ class WorkloadmgrCLIHelper(object): WORKLOAD_STATUS_CMD = ( "openstack {auth_args} workload show " "-f value -c status " - " {resource_id} " + "{resource_id}" + ) + + WORKLOAD_FULL_STATUS_CMD = ( + "openstack {auth_args} workload show " + "{resource_id}" ) SNAPSHOT_CMD = ( @@ -94,7 +117,12 @@ class WorkloadmgrCLIHelper(object): SNAPSHOT_STATUS_CMD = ( "openstack {auth_args} workload snapshot show " "-f value -c status " - "{resource_id} " + "{resource_id}" + ) + + SNAPSHOT_FULL_STATUS_CMD = ( + "openstack {auth_args} workload snapshot show " + "{resource_id}" ) ONECLICK_RESTORE_CMD = ( @@ -110,7 +138,13 @@ class WorkloadmgrCLIHelper(object): RESTORE_STATUS_CMD = ( "openstack {auth_args} workloadmgr restore show " - "-f value -c status {resource_id}" + "-f value -c status " + "{resource_id}" + ) + + RESTORE_FULL_STATUS_CMD = ( + "openstack {auth_args} workloadmgr restore show " + "{resource_id}" ) def __init__(self, keystone_client): @@ -193,6 +227,7 @@ class WorkloadmgrCLIHelper(object): self.trilio_wlm_unit, self.auth_args, self.WORKLOAD_STATUS_CMD, + self.WORKLOAD_FULL_STATUS_CMD, workload_id, "available", ) @@ -235,6 +270,7 @@ class WorkloadmgrCLIHelper(object): self.trilio_wlm_unit, self.auth_args, self.SNAPSHOT_STATUS_CMD, + self.SNAPSHOT_FULL_STATUS_CMD, snapshot_id, "available", ) @@ -275,6 +311,7 @@ class WorkloadmgrCLIHelper(object): self.trilio_wlm_unit, self.auth_args, self.RESTORE_STATUS_CMD, + self.RESTORE_FULL_STATUS_CMD, restore_id, "available", ) From c75f2fcb6d4843a249ed7f5f1c38a1b605c9c3c2 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 8 Sep 2020 08:40:30 +0200 Subject: [PATCH 675/898] Ensure that zaza-openstack-tests can talk to an IPv6 Keystone When configuring the address to use to talk to Keystone, the format_addr helper should be used to ensure that an IPv4 or IPv6 address can be correctly handled. --- unit_tests/charm_tests/test_utils.py | 12 ------------ zaza/openstack/charm_tests/test_utils.py | 18 ------------------ zaza/openstack/charm_tests/vault/utils.py | 4 ++-- zaza/openstack/utilities/openstack.py | 3 +++ 4 files changed, 5 insertions(+), 32 deletions(-) diff --git a/unit_tests/charm_tests/test_utils.py b/unit_tests/charm_tests/test_utils.py index a415360..280e2e2 100644 --- a/unit_tests/charm_tests/test_utils.py +++ b/unit_tests/charm_tests/test_utils.py @@ -30,15 +30,3 @@ class TestOpenStackBaseTest(unittest.TestCase): MyTestClass.setUpClass('foo', 'bar') _setUpClass.assert_called_with('foo', 'bar') - - -class TestUtils(unittest.TestCase): - - def test_format_addr(self): - self.assertEquals('1.2.3.4', test_utils.format_addr('1.2.3.4')) - self.assertEquals( - '[2001:db8::42]', test_utils.format_addr('2001:db8::42')) - with self.assertRaises(ValueError): - test_utils.format_addr('999.999.999.999') - with self.assertRaises(ValueError): - test_utils.format_addr('2001:db8::g') diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 83595b0..2994629 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -14,7 +14,6 @@ """Module containing base class for implementing charm tests.""" import contextlib import logging -import ipaddress import subprocess import tenacity import unittest @@ -541,20 +540,3 @@ class OpenStackBaseTest(BaseCharmTest): instance_2 = self.retrieve_guest( '{}-ins-1'.format(self.RESOURCE_PREFIX)) return instance_1, instance_2 - - -def format_addr(addr): - """Validate and format IP address. - - :param addr: IPv6 or IPv4 address - :type addr: str - :returns: Address string, optionally encapsulated in brackets([]) - :rtype: str - :raises: ValueError - """ - ipaddr = ipaddress.ip_address(addr) - if isinstance(ipaddr, ipaddress.IPv6Address): - fmt = '[{}]' - else: - fmt = '{}' - return fmt.format(ipaddr) diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index a708d78..9814243 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -27,7 +27,7 @@ import yaml import collections import zaza.model -import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.utilities.networking as network_utils AUTH_FILE = "vault_tests.yaml" CharmVaultClient = collections.namedtuple( @@ -102,7 +102,7 @@ def get_unit_api_url(ip): transport = 'http' if vault_config['ssl-cert']['value']: transport = 'https' - return '{}://{}:8200'.format(transport, test_utils.format_addr(ip)) + return '{}://{}:8200'.format(transport, network_utils.format_addr(ip)) def get_hvac_client(vault_url, cacert=None): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 14c708c..34d7e7b 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -68,6 +68,8 @@ from zaza.openstack.utilities import ( exceptions, generic as generic_utils, ) +import zaza.utilities.networking as network_utils + CIRROS_RELEASE_URL = 'http://download.cirros-cloud.net/version/released' CIRROS_IMAGE_URL = 'http://download.cirros-cloud.net' @@ -1704,6 +1706,7 @@ def get_overcloud_auth(address=None, model_name=None): if not address: address = get_keystone_ip(model_name=model_name) + address = network_utils.format_addr(address) password = juju_utils.leader_get( 'keystone', From e6300dba20578a24a067ec449d6677413a964069 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 8 Sep 2020 12:11:05 +0200 Subject: [PATCH 676/898] Add HaclusterScalebackTest (#399) Co-authored-by: Alvaro Uria --- zaza/openstack/charm_tests/hacluster/tests.py | 67 ++++++++++++++++++- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 199d915..af47b76 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -21,17 +21,22 @@ import os import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.hacluster +import zaza.utilities.juju as juju_utils -class HaclusterTest(test_utils.OpenStackBaseTest): - """hacluster tests.""" +class HaclusterBaseTest(test_utils.OpenStackBaseTest): + """Base class for hacluster tests.""" @classmethod def setUpClass(cls): """Run class setup for running hacluster tests.""" - super(HaclusterTest, cls).setUpClass() + super(HaclusterBaseTest, cls).setUpClass() cls.vip = os.environ.get("TEST_VIP00") + +class HaclusterTest(HaclusterBaseTest): + """hacluster tests.""" + def test_900_action_cleanup(self): """The services can be cleaned up.""" zaza.model.run_action_on_leader( @@ -69,3 +74,59 @@ class HaclusterTest(test_utils.OpenStackBaseTest): self._toggle_maintenance_and_wait('true') self._toggle_maintenance_and_wait('false') + + +class HaclusterScalebackTest(HaclusterBaseTest): + """hacluster scaleback tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running hacluster scaleback tests.""" + super(HaclusterScalebackTest, cls).setUpClass() + test_config = cls.test_config['tests_options']['hacluster'] + cls._principle_app_name = test_config['principle-app-name'] + cls._hacluster_charm_name = test_config['hacluster-charm-name'] + + def test_930_scaleback(self): + """Remove a unit and add a new one.""" + principle_units = sorted(zaza.model.get_status().applications[ + self._principle_app_name]['units'].keys()) + self.assertEqual(len(principle_units), 3) + doomed_principle_unit = principle_units[0] + other_principle_unit = principle_units[1] + doomed_hacluster_unit = juju_utils.get_subordinate_units( + [doomed_principle_unit], charm_name=self._hacluster_charm_name)[0] + other_hacluster_unit = juju_utils.get_subordinate_units( + [other_principle_unit], charm_name=self._hacluster_charm_name)[0] + + logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) + zaza.model.run_action( + doomed_hacluster_unit, + 'pause', + raise_on_failure=True) + logging.info('OK') + + logging.info('Removing {}'.format(doomed_principle_unit)) + zaza.model.destroy_unit( + self._principle_app_name, + doomed_principle_unit, + wait_disappear=True) + logging.info('OK') + + logging.info('Waiting for model to settle') + zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') + zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + zaza.model.block_until_all_units_idle() + logging.info('OK') + + logging.info('Adding an hacluster unit') + zaza.model.add_unit(self._principle_app_name, wait_appear=True) + logging.info('OK') + + logging.info('Waiting for model to settle') + zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active') + # NOTE(lourot): the principle application remains blocked after scaling + # back up until lp:1400481 is solved. + zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + zaza.model.block_until_all_units_idle() + logging.debug('OK') From 38317c3bc202bf338d9560903e140e8bed3bcc23 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 8 Sep 2020 12:24:26 +0200 Subject: [PATCH 677/898] Make RmqTests.test_910_pause_and_resume more robust --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index bbb6c3b..3470bad 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -276,15 +276,23 @@ class RmqTests(test_utils.OpenStackBaseTest): """The services can be paused and resumed.""" logging.info('Checking pause and resume actions...') + logging.info('Waiting for the cluster to be ready') + rmq_utils.wait_for_cluster() unit = zaza.model.get_units(self.application_name)[0] assert unit.workload_status == "active" + logging.info('Pausing unit {}'.format(unit)) zaza.model.run_action(unit.entity_id, "pause") + logging.info('Waiting until unit {} reaches "maintenance" state' + ''.format(unit)) zaza.model.block_until_unit_wl_status(unit.entity_id, "maintenance") unit = zaza.model.get_unit_from_name(unit.entity_id) assert unit.workload_status == "maintenance" + logging.info('Resuming unit {}'.format(unit)) zaza.model.run_action(unit.entity_id, "resume") + logging.info('Waiting until unit {} reaches "active" state' + ''.format(unit)) zaza.model.block_until_unit_wl_status(unit.entity_id, "active") unit = zaza.model.get_unit_from_name(unit.entity_id) assert unit.workload_status == "active" From 8f2e3463cc3dedf215342120732d57680157e488 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 8 Sep 2020 12:38:39 +0200 Subject: [PATCH 678/898] Allow `add_interface_to_netplan` to find ovn-dedicated-chassis units --- zaza/openstack/utilities/openstack.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index c5e2bfa..8d6c172 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -631,15 +631,20 @@ def add_interface_to_netplan(server_name, mac_address): :type mac_address: string """ if dvr_enabled(): - application_name = 'neutron-openvswitch' + application_names = ('neutron-openvswitch',) elif ovn_present(): # OVN chassis is a subordinate to nova-compute - application_name = 'nova-compute' + application_names = ('nova-compute', 'ovn-dedicated-chassis') else: - application_name = 'neutron-gateway' + application_names = ('neutron-gateway',) - unit_name = juju_utils.get_unit_name_from_host_name( - server_name, application_name) + for app_name in application_names: + unit_name = juju_utils.get_unit_name_from_host_name( + server_name, app_name) + if unit_name: + break + else: + raise RuntimeError('Unable to find unit to run commands on.') run_cmd_nic = "ip -f link -br -o addr|grep {}".format(mac_address) interface = model.run_on_unit(unit_name, run_cmd_nic) interface = interface['Stdout'].split(' ')[0] @@ -746,8 +751,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, 'Neutron Gateway already has additional port') break else: - logging.info('Attaching additional port to instance, ' - 'connected to net id: {}'.format(net_id)) + logging.info('Attaching additional port to instance ("{}"), ' + 'connected to net id: {}' + .format(uuid, net_id)) body_value = { "port": { "admin_state_up": True, From 14f7e3848cb5989912769e3a7c47345cd1f56624 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 8 Sep 2020 12:51:53 +0200 Subject: [PATCH 679/898] Remove hacluster test option hacluster_app_name --- zaza/openstack/charm_tests/hacluster/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 2728a68..e02832b 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -85,7 +85,6 @@ class HaclusterScalebackTest(HaclusterBaseTest): super(HaclusterScalebackTest, cls).setUpClass() test_config = cls.test_config['tests_options']['hacluster'] cls._principle_app_name = test_config['principle-app-name'] - cls._hacluster_app_name = test_config['hacluster-app-name'] cls._hacluster_charm_name = test_config['hacluster-charm-name'] def test_930_scaleback(self): @@ -121,8 +120,9 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('OK') logging.info('Updating corosync ring') + hacluster_app_name = other_hacluster_unit.application zaza.model.run_action_on_leader( - self._hacluster_app_name, + hacluster_app_name, 'update-ring', action_params={'i-really-mean-it': True}, raise_on_failure=True) @@ -132,7 +132,7 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('OK') logging.info('Waiting for model to settle') - expected_states = {self._hacluster_app_name: { + expected_states = {hacluster_app_name: { "workload-status": "active", "workload-status-message": "Unit is ready and clustered"}} zaza.model.wait_for_application_states(states=expected_states) From 5a10779e4533cb38d42a15f01923789769aeb683 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 8 Sep 2020 16:11:39 +0200 Subject: [PATCH 680/898] Re-order checks in networking auto-detection To support OVS to OVN migration checks we want the basic overcloud configure job to set up N-OVS and/or N-GW when present and the OVN pre migration configure job will copy the configuration for us. --- .../test_zaza_utilities_openstack.py | 13 ++++++++++-- zaza/openstack/utilities/openstack.py | 20 +++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index e15ce96..02bd578 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1227,6 +1227,13 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.get_application.side_effect = [KeyError, KeyError] self.assertFalse(openstack_utils.ovn_present()) + def test_ngw_present(self): + self.patch_object(openstack_utils.model, 'get_application') + self.get_application.side_effect = None + self.assertTrue(openstack_utils.ngw_present()) + self.get_application.side_effect = KeyError + self.assertFalse(openstack_utils.ngw_present()) + def test_configure_gateway_ext_port(self): # FIXME: this is not a complete unit test for the function as one did # not exist at all I'm adding this to test one bit and we'll add more @@ -1234,10 +1241,12 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.patch_object(openstack_utils, 'deprecated_external_networking') self.patch_object(openstack_utils, 'dvr_enabled') self.patch_object(openstack_utils, 'ovn_present') + self.patch_object(openstack_utils, 'ngw_present') self.patch_object(openstack_utils, 'get_gateway_uuids') self.patch_object(openstack_utils, 'get_admin_net') - self.dvr_enabled = False - self.ovn_present = False + self.dvr_enabled.return_value = False + self.ovn_present.return_value = False + self.ngw_present.return_value = True self.get_admin_net.return_value = {'id': 'fakeid'} novaclient = mock.MagicMock() diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 8d6c172..e524f79 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -554,6 +554,20 @@ def dvr_enabled(): return get_application_config_option('neutron-api', 'enable-dvr') +def ngw_present(): + """Check whether Neutron Gateway is present in deployment. + + :returns: True when Neutron Gateway is present, False otherwise + :rtype: bool + """ + try: + model.get_application('neutron-gateway') + return True + except KeyError: + pass + return False + + def ovn_present(): """Check whether OVN is present in deployment. @@ -721,6 +735,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, except KeyError: # neutron-gateway not in deployment pass + elif ngw_present(): + uuids = itertools.islice(get_gateway_uuids(), limit_gws) + application_names = ['neutron-gateway'] elif ovn_present(): uuids = itertools.islice(get_ovn_uuids(), limit_gws) application_names = ['ovn-chassis'] @@ -735,8 +752,7 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) add_dataport_to_netplan = True else: - uuids = itertools.islice(get_gateway_uuids(), limit_gws) - application_names = ['neutron-gateway'] + raise RuntimeError('Unable to determine charm network topology.') if not net_id: net_id = get_admin_net(neutronclient)['id'] From 4eef19197dbceae5a2d0799211290f6f0facb169 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 9 Sep 2020 09:49:15 +0200 Subject: [PATCH 681/898] Add missing blank --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 3470bad..d4a3111 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -364,7 +364,7 @@ class RmqTests(test_utils.OpenStackBaseTest): return rmq_utils.check_unit_cluster_nodes(u, unit_node_names) @unittest.skip( - "Skipping as a significant rework is required, see" + "Skipping as a significant rework is required, see " "https://github.com/openstack-charmers/zaza-openstack-tests/issues/290" ) def test_921_remove_and_add_unit(self): From 11f3060f6b1564d6497cbfc2b2f80251bcc27ca3 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 9 Sep 2020 10:08:37 +0200 Subject: [PATCH 682/898] Fix unit-name -> app-name conversion --- zaza/openstack/charm_tests/hacluster/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index e02832b..29751e1 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -120,7 +120,8 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('OK') logging.info('Updating corosync ring') - hacluster_app_name = other_hacluster_unit.application + hacluster_app_name = zaza.model.get_unit_from_name( + other_hacluster_unit).application zaza.model.run_action_on_leader( hacluster_app_name, 'update-ring', From 9726179f49a522175f46919102bbcc15c1308659 Mon Sep 17 00:00:00 2001 From: James Page Date: Wed, 9 Sep 2020 10:38:24 +0100 Subject: [PATCH 683/898] trilio: add test for ghost-share action Restructure test classes a little to support addition of testing of the ghost-share action to charms that support it. Execute the action - if any operation fails the action will fail so an OK return code on the action indicates success. --- zaza/openstack/charm_tests/trilio/tests.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 991b7f5..581c395 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -25,6 +25,7 @@ import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.guest as guest_utils import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.utilities.generic as generic_utils from zaza.utilities import juju as juju_utils @@ -424,7 +425,22 @@ class TrilioBaseTest(test_utils.OpenStackBaseTest): workloadmgrcli.oneclick_restore(snapshot_id) -class TrilioWLMTest(TrilioBaseTest): +class TrilioGhostNFSShareTest(TrilioBaseTest): + """Tests for Trilio charms providing the ghost-share action.""" + + def test_ghost_nfs_share(self): + """Ensure ghost-share action bind mounts NFS share.""" + generic_utils.assertActionRanOK(zaza_model.run_action( + self.lead_unit, + 'ghost-share', + action_params={ + 'nfs-shares': '10.20.0.1:/srv/testing' + }, + model_name=self.model_name) + ) + + +class TrilioWLMTest(TrilioGhostNFSShareTest): """Tests for Trilio Workload Manager charm.""" conf_file = "/etc/workloadmgr/workloadmgr.conf" @@ -447,7 +463,7 @@ class TrilioDMAPITest(TrilioBaseTest): services = ["dmapi-api"] -class TrilioDataMoverTest(TrilioBaseTest): +class TrilioDataMoverTest(TrilioGhostNFSShareTest): """Tests for Trilio Data Mover charm.""" conf_file = "/etc/tvault-contego/tvault-contego.conf" From 332294d7527fab657baf4c699a2baffc31756656 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 9 Sep 2020 10:39:25 +0200 Subject: [PATCH 684/898] Handle OVS to OVN migration with neutron-gateway topologies Merge two similar config set patterns into a common method. Add missing docstrings. Fix typos. --- zaza/openstack/charm_tests/ovn/setup.py | 93 ++++++++++++++----------- zaza/openstack/charm_tests/ovn/tests.py | 54 ++++++++++++-- 2 files changed, 103 insertions(+), 44 deletions(-) diff --git a/zaza/openstack/charm_tests/ovn/setup.py b/zaza/openstack/charm_tests/ovn/setup.py index 116cdb8..122ce70 100644 --- a/zaza/openstack/charm_tests/ovn/setup.py +++ b/zaza/openstack/charm_tests/ovn/setup.py @@ -40,12 +40,54 @@ class _OVNSetupHelper(test_utils.BaseCharmTest): return int(n_api_config['global-physnet-mtu']['value']) - ( GENEVE_ENCAP_OVERHEAD + IP4_HEADER_SIZE) + def _configure_apps(self, apps, cfg, + first_match_raise_if_none_found=False): + """Conditionally configure a set of applications. + + :param apps: Applications. + :type apps: Iterator[str] + :param cfg: Configuration to apply. + :type cfg: Dict[str,any] + :param first_match_raise_if_none_found: When set the method will + configure the first application + it finds in the model and raise + an exception if none are found. + :type first_match_raise_if_none_found: bool + :raises: RuntimeError + """ + for app in apps: + try: + zaza.model.get_application(app) + for k, v in cfg.items(): + logging.info('Setting `{}` to "{}" on "{}"...' + .format(k, v, app)) + with self.config_change(cfg, cfg, app): + # The intent here is to change the config and not + # restore it. We accomplish that by passing in the same + # value for default and alternate. + # + # The reason for using the `config_change` helper for + # this is that it already deals with all the + # permutations of config already being set etc and does + # not get into trouble if the test bundle already has + # the values we try to set. + if first_match_raise_if_none_found: + break + else: + continue + else: + if first_match_raise_if_none_found: + raise RuntimeError( + 'None of the expected apps ({}) are present in ' + 'the model.' + .format(apps) + ) + except KeyError: + pass + def configure_ngw_novs(self): """Configure n-ovs and n-gw units.""" cfg = { - # To be able to successfully clean up after the Neutron agents we - # need to use the 'openvswitch' `firewall-driver`. - 'firewall-driver': 'openvswitch', # To be able to have instances successfully survive the migration # without communication issues we need to lower the MTU announced # to instances prior to migration. @@ -59,25 +101,13 @@ class _OVNSetupHelper(test_utils.BaseCharmTest): 'instance-mtu': self._get_instance_mtu_from_global_physnet_mtu() } apps = ('neutron-gateway', 'neutron-openvswitch') - for app in apps: - try: - zaza.model.get_application(app) - for k, v in cfg.items(): - logging.info('Setting `{}` to "{}" on "{}"...' - .format(k, v, app)) - with self.config_change(cfg, cfg, app): - # The intent here is to change the config and not restore - # it. We accomplish that by passing in the same value for - # default and alternate. - # - # The reason for using the `config_change` helper for this - # is that it already deals with all the permutations of - # config already being set etc and does not get into - # trouble if the test bundle already have the values we try - # to set. - continue - except KeyError: - pass + self._configure_apps(apps, cfg) + cfg_ovs = { + # To be able to successfully clean up after the Neutron agents we + # need to use the 'openvswitch' `firewall-driver`. + 'firewall-driver': 'openvswitch', + } + self._configure_apps(('neutron-openvswitch',), cfg_ovs) def configure_ovn_mappings(self): """Copy mappings from n-gw or n-ovs application.""" @@ -102,23 +132,8 @@ class _OVNSetupHelper(test_utils.BaseCharmTest): .format(src_apps) ) - for app in dst_apps: - try: - zaza.model.get_application(app) - for k, v in ovn_cfg.items(): - logging.info('Setting `{}` to "{}" on "{}"...' - .format(k, v, app)) - with self.config_change(ovn_cfg, ovn_cfg, app): - # Set values only on ovn-dedicated-chassis when present, - # otherwise we set them on ovn-chassis. - break - except KeyError: - pass - else: - raise RuntimeError( - 'None of the expected apps ({}) are present in the model.' - .format(dst_apps) - ) + self._configure_apps( + dst_apps, ovn_cfg, first_match_raise_if_none_found=True) def pre_migration_configuration(): diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 3b07920..d12911e 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -84,7 +84,7 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): def setUp(self): """Perform migration steps prior to validation.""" super(OVSOVNMigrationTest, self).setUp() - # These steps here due to them having to be executed once and in a + # These steps are here due to them having to be executed once and in a # specific order prior to running any tests. The steps should still # be idempotent if at all possible as a courtesy to anyone iterating # on the test code. @@ -104,6 +104,11 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): # Stop Neutron agents on hypervisors self._pause_units('neutron-openvswitch') + try: + self._pause_units('neutron-gateway') + except KeyError: + logging.info( + 'No neutron-gateway in deployment, skip pausing it.') # Add the neutron-api-plugin-ovn subordinate which will make the # `neutron-api-plugin-ovn` unit appear in the deployment. @@ -155,7 +160,13 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): self._resume_units('neutron-api') # Run `cleanup` action on neutron-openvswitch units/hypervisors - self._run_cleanup_action() + self._run_cleanup_action('neutron-openvswitch') + # Run `cleanup` action on neutron-gateway units when present + try: + self._run_cleanup_action('neutron-gateway') + except KeyError: + logging.info( + 'No neutron-gateway in deployment, skip cleanup of it.') # Start the OVN controller on hypervisors # @@ -164,11 +175,18 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): # you will program the network to a state of infinite loop. self._resume_units('ovn-chassis') + try: + self._resume_units('ovn-dedicated-chassis') + except KeyError: + logging.info( + 'No ovn-dedicated-chassis in deployment, skip resume.') + # And we should be off to the races self.one_time_init_done = True def _add_neutron_api_plugin_ovn_subordinate_relation(self): + """Add relation between neutron-api and neutron-api-plugin-ovn.""" try: logging.info('Adding relation neutron-api-plugin-ovn ' '-> neutron-api') @@ -200,6 +218,7 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): logging.info('done') def _run_offline_neutron_morph_db_action(self): + """Run offline-neutron-morph-db action.""" logging.info('Running the optional `offline-neutron-morph-db` action ' 'on neutron-api-plugin-ovn/leader') generic_utils.assertActionRanOK( @@ -213,6 +232,7 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): ) def _run_migrate_ovn_db_action(self): + """Run migrate-ovn-db action.""" logging.info('Running `migrate-ovn-db` action on ' 'neutron-api-plugin-ovn/leader') generic_utils.assertActionRanOK( @@ -230,6 +250,14 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): @tenacity.retry(wait=tenacity.wait_exponential(min=5, max=60), reraise=True, stop=tenacity.stop_after_attempt(3)) def _run_migrate_mtu_action(self): + """Run migrate-mtu action with retry. + + The action is idempotent. + + Due to LP: #1854518 and the point in time of the test life cycle we run + this action the probability for the Neutron API not being available + for the script to do its job is high, thus we retry. + """ logging.info('Running `migrate-mtu` action on ' 'neutron-api-plugin-ovn/leader') generic_utils.assertActionRanOK( @@ -243,6 +271,11 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): ) def _pause_units(self, application): + """Pause units of application. + + :param application: Name of application + :type application: str + """ logging.info('Pausing {} units'.format(application)) zaza.model.run_action_on_units( [unit.entity_id @@ -259,11 +292,17 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): }, ) - def _run_cleanup_action(self): - logging.info('Running `cleanup` action on neutron-openvswitch units.') + def _run_cleanup_action(self, application): + """Run cleanup action on application units. + + :param application: Name of application + :type application: str + """ + logging.info('Running `cleanup` action on {} units.' + .format(application)) zaza.model.run_action_on_units( [unit.entity_id - for unit in zaza.model.get_units('neutron-openvswitch')], + for unit in zaza.model.get_units(application)], 'cleanup', action_params={ 'i-really-mean-it': True}, @@ -271,6 +310,11 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): ) def _resume_units(self, application): + """Resume units of application. + + :param application: Name of application + :type application: str + """ logging.info('Resuming {} units'.format(application)) zaza.model.run_action_on_units( [unit.entity_id From e0795481d5a76896f7ddb00859cc0ac6fdb5fb98 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 11 Sep 2020 11:42:25 +0200 Subject: [PATCH 685/898] Add Octavia to Tempest service list --- zaza/openstack/charm_tests/tempest/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 7ce4478..e7a1c41 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -33,8 +33,8 @@ SETUP_ENV_VARS = { TEMPEST_FLAVOR_NAME = 'm1.tempest' TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest' TEMPEST_SVC_LIST = ['ceilometer', 'cinder', 'glance', 'heat', 'horizon', - 'ironic', 'neutron', 'nova', 'sahara', 'swift', 'trove', - 'zaqar'] + 'ironic', 'neutron', 'nova', 'octavia', 'sahara', 'swift', + 'trove', 'zaqar'] def add_application_ips(ctxt): From fbaf48d54106c7b4cf4aa7b316721ecba22f4b83 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Fri, 11 Sep 2020 15:27:15 +0100 Subject: [PATCH 686/898] More fixes for series-upgrade (#413) --- zaza/openstack/charm_tests/series_upgrade/tests.py | 4 ++-- zaza/openstack/utilities/cli.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/series_upgrade/tests.py b/zaza/openstack/charm_tests/series_upgrade/tests.py index c595448..122af11 100644 --- a/zaza/openstack/charm_tests/series_upgrade/tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/tests.py @@ -81,7 +81,7 @@ class SeriesUpgradeTest(unittest.TestCase): logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( - charm_name, + application, 'complete-cluster-series-upgrade', action_params={}) model.block_until_all_units_idle() @@ -90,7 +90,7 @@ class SeriesUpgradeTest(unittest.TestCase): logging.info( "Running complete-cluster-series-upgrade action on leader") model.run_action_on_leader( - charm_name, + application, 'complete-cluster-series-upgrade', action_params={}) model.block_until_all_units_idle() diff --git a/zaza/openstack/utilities/cli.py b/zaza/openstack/utilities/cli.py index 68a659f..f8071be 100644 --- a/zaza/openstack/utilities/cli.py +++ b/zaza/openstack/utilities/cli.py @@ -16,6 +16,7 @@ import logging import os +import sys def parse_arg(options, arg, multiargs=False): @@ -51,6 +52,6 @@ def setup_logging(): rootLogger = logging.getLogger() rootLogger.setLevel('INFO') if not rootLogger.hasHandlers(): - consoleHandler = logging.StreamHandler() + consoleHandler = logging.StreamHandler(sys.stdout) consoleHandler.setFormatter(logFormatter) rootLogger.addHandler(consoleHandler) From 46d426ea307625bae13a400c99cde053c3bbbeac Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 14 Sep 2020 11:28:46 +0200 Subject: [PATCH 687/898] Make kerberos test setup more robust --- zaza/openstack/charm_tests/kerberos/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/kerberos/setup.py b/zaza/openstack/charm_tests/kerberos/setup.py index f58a961..fadd7a9 100644 --- a/zaza/openstack/charm_tests/kerberos/setup.py +++ b/zaza/openstack/charm_tests/kerberos/setup.py @@ -113,6 +113,8 @@ def retrieve_and_attach_keytab(): 'keystone_keytab', tmp_file) + zaza.model.block_until_all_units_idle() + def openstack_setup_kerberos(): """Create a test domain, project, and user for kerberos tests.""" From a1d61c76263c23d1e52042fd61aa7689dab33d4d Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 14 Sep 2020 13:54:47 +0200 Subject: [PATCH 688/898] Let kerberos test setup wait for model to settle --- zaza/openstack/charm_tests/kerberos/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/zaza/openstack/charm_tests/kerberos/setup.py b/zaza/openstack/charm_tests/kerberos/setup.py index fadd7a9..d441f8c 100644 --- a/zaza/openstack/charm_tests/kerberos/setup.py +++ b/zaza/openstack/charm_tests/kerberos/setup.py @@ -113,6 +113,7 @@ def retrieve_and_attach_keytab(): 'keystone_keytab', tmp_file) + zaza.model.wait_for_application_states() zaza.model.block_until_all_units_idle() From 17bad029a5f1df0ca3b41ba8f3f6eb388a8cc4d5 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 15 Sep 2020 09:55:09 +0100 Subject: [PATCH 689/898] Fix network setup for mojo A recent change introduced a configuration option in the ./tests/test.yaml file (default location) which allows juju wait to be used rather than waiting on various workload messages. This, unfortunately, breaks mojo tests as they don't use a tests.yaml. This change refactors that code, and enables a 'use_juju_wait' to be passed into the relevant functions, and a new command line option (default true) to disable using juju wait. --- zaza/openstack/charm_tests/neutron/setup.py | 14 +++++++++++--- zaza/openstack/configure/network.py | 17 ++++++++++++++--- zaza/openstack/utilities/openstack.py | 13 +++++++++---- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/setup.py b/zaza/openstack/charm_tests/neutron/setup.py index e8f2439..f65b87a 100644 --- a/zaza/openstack/charm_tests/neutron/setup.py +++ b/zaza/openstack/charm_tests/neutron/setup.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # Copyright 2018 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,6 +26,8 @@ from zaza.openstack.utilities import ( openstack as openstack_utils, ) +import zaza.charm_lifecycle.utils as lifecycle_utils + # The overcloud network configuration settings are declared. # These are the network configuration settings under test. @@ -81,12 +81,20 @@ def basic_overcloud_network(limit_gws=None): # Get keystone session keystone_session = openstack_utils.get_overcloud_keystone_session() + # Get optional use_juju_wait for netw ork option + options = (lifecycle_utils + .get_charm_config(fatal=False) + .get('configure_options', {})) + use_juju_wait = options.get( + 'configure_gateway_ext_port_use_juju_wait', True) + # Handle network for OpenStack-on-OpenStack scenarios if juju_utils.get_provider_type() == "openstack": undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session() network.setup_gateway_ext_port(network_config, keystone_session=undercloud_ks_sess, - limit_gws=None) + limit_gws=None, + use_juju_wait=use_juju_wait) # Confugre the overcloud network network.setup_sdn(network_config, keystone_session=keystone_session) diff --git a/zaza/openstack/configure/network.py b/zaza/openstack/configure/network.py index 649992c..ad1a07b 100755 --- a/zaza/openstack/configure/network.py +++ b/zaza/openstack/configure/network.py @@ -184,7 +184,8 @@ def setup_sdn(network_config, keystone_session=None): def setup_gateway_ext_port(network_config, keystone_session=None, - limit_gws=None): + limit_gws=None, + use_juju_wait=True): """Perform setup external port on Neutron Gateway. For OpenStack on OpenStack scenarios. @@ -195,6 +196,8 @@ def setup_gateway_ext_port(network_config, keystone_session=None, :type keystone_session: keystoneauth1.session.Session object :param limit_gws: Limit the number of gateways that get a port attached :type limit_gws: Optional[int] + :param use_juju_wait: Use juju wait (default True) for model to settle + :type use_juju_wait: boolean :returns: None :rtype: None """ @@ -230,7 +233,8 @@ def setup_gateway_ext_port(network_config, keystone_session=None, neutron_client, net_id=net_id, add_dataport_to_netplan=add_dataport_to_netplan, - limit_gws=limit_gws) + limit_gws=limit_gws, + use_juju_wait=use_juju_wait) def run_from_cli(**kwargs): @@ -269,6 +273,11 @@ def run_from_cli(**kwargs): default="network.yaml") parser.add_argument("--cacert", help="Path to CA certificate bundle file", default=None) + parser.add_argument("--no-use-juju-wait", + help=("don't use juju wait for the model to settle " + "(default true)"), + action="store_false", + default=True) # Handle CLI options options = parser.parse_args() net_topology = (kwargs.get('net_toplogoy') or @@ -289,7 +298,9 @@ def run_from_cli(**kwargs): undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session( verify=cacert) setup_gateway_ext_port(network_config, - keystone_session=undercloud_ks_sess) + keystone_session=undercloud_ks_sess, + use_juju_wait=cli_utils.parse_arg( + options, 'no_use_juju_wait')) overcloud_ks_sess = openstack_utils.get_overcloud_keystone_session( verify=cacert) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 7ab203d..0c31c05 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -64,7 +64,6 @@ import textwrap import urllib import zaza -import zaza.charm_lifecycle.utils as lifecycle_utils from zaza import model from zaza.openstack.utilities import ( @@ -703,7 +702,8 @@ def add_interface_to_netplan(server_name, mac_address): def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, add_dataport_to_netplan=False, - limit_gws=None): + limit_gws=None, + use_juju_wait=True): """Configure the neturong-gateway external port. :param novaclient: Authenticated novaclient @@ -714,6 +714,9 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, :type net_id: string :param limit_gws: Limit the number of gateways that get a port attached :type limit_gws: Optional[int] + :param use_juju_wait: Whether to use juju wait to wait for the model to + settle once the gateway has been configured. Default is True + :type use_juju_wait: boolean """ deprecated_extnet_mode = deprecated_external_networking() @@ -821,11 +824,13 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, # NOTE(fnordahl): We are stuck with juju_wait until we figure out how # to deal with all the non ['active', 'idle', 'Unit is ready.'] # workload/agent states and msgs that our mojo specs are exposed to. - if lifecycle_utils.get_config_options().get( - 'configure_gateway_ext_port_use_juju_wait', True): + if use_juju_wait: juju_wait.wait(wait_for_workload=True) else: zaza.model.wait_for_agent_status() + # TODO: shouldn't access get_charm_config() here as it relies on + # ./tests/tests.yaml existing by default (regardless of the + # fatal=False) ... it's not great design. test_config = zaza.charm_lifecycle.utils.get_charm_config( fatal=False) zaza.model.wait_for_application_states( From 52a2aabf57e7c563827fe55f5a779934c42f5251 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 28 Aug 2020 11:28:42 +0200 Subject: [PATCH 690/898] utilities/ceph: Add helper for retrieving pools from broker req --- .../utilities/test_zaza_utilities_ceph.py | 17 +++++++++ zaza/openstack/utilities/ceph.py | 38 ++++++++++++++++++- 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/unit_tests/utilities/test_zaza_utilities_ceph.py b/unit_tests/utilities/test_zaza_utilities_ceph.py index 0855e41..ea718b7 100644 --- a/unit_tests/utilities/test_zaza_utilities_ceph.py +++ b/unit_tests/utilities/test_zaza_utilities_ceph.py @@ -116,3 +116,20 @@ class TestCephUtils(ut_utils.BaseTestCase): with self.assertRaises(model.CommandRunFailed): ceph_utils.get_rbd_hash('aunit', 'apool', 'aimage', model_name='amodel') + + def test_pools_from_broker_req(self): + self.patch_object(ceph_utils.zaza_juju, 'get_relation_from_unit') + self.get_relation_from_unit.return_value = { + 'broker_req': ( + '{"api-version": 1, "ops": [' + '{"op": "create-pool", "name": "cinder-ceph", ' + '"compression-mode": null},' + '{"op": "create-pool", "name": "cinder-ceph", ' + '"compression-mode": "aggressive"}]}'), + } + self.assertEquals( + ceph_utils.get_pools_from_broker_req( + 'anApplication', 'aModelName'), + ['cinder-ceph']) + self.get_relation_from_unit.assert_called_once_with( + 'ceph-mon', 'anApplication', None, model_name='aModelName') diff --git a/zaza/openstack/utilities/ceph.py b/zaza/openstack/utilities/ceph.py index 893ea5e..6613154 100644 --- a/zaza/openstack/utilities/ceph.py +++ b/zaza/openstack/utilities/ceph.py @@ -2,8 +2,10 @@ import json import logging -import zaza.openstack.utilities.openstack as openstack_utils import zaza.model as zaza_model +import zaza.utilities.juju as zaza_juju + +import zaza.openstack.utilities.openstack as openstack_utils REPLICATED_POOL_TYPE = 'replicated' ERASURE_POOL_TYPE = 'erasure-coded' @@ -204,3 +206,37 @@ def get_rbd_hash(unit_name, pool, image, model_name=None): if result.get('Code') != '0': raise zaza_model.CommandRunFailed(cmd, result) return result.get('Stdout').rstrip() + + +def get_pools_from_broker_req(application_or_unit, model_name=None): + """Get pools requested by application or unit. + + By retrieving and parsing broker request from relation data we can get a + list of pools a unit has requested. + + :param application_or_unit: Name of application or unit that is at the + other end of a ceph-mon relation. + :type application_or_unit: str + :param model_name: Name of Juju model to operate on + :type model_name: Optional[str] + :returns: List of pools requested. + :rtype: List[str] + :raises: KeyError + """ + # NOTE: we do not pass on a name for the remote_interface_name as that + # varies between the Ceph consuming applications. + relation_data = zaza_juju.get_relation_from_unit( + 'ceph-mon', application_or_unit, None, model_name=model_name) + + # NOTE: we probably should consume the Ceph broker code from c-h but c-h is + # such a beast of a dependency so let's defer adding it to Zaza if we can. + broker_req = json.loads(relation_data['broker_req']) + + # A charm may request modifications to an existing pool by adding multiple + # 'create-pool' broker requests so we need to deduplicate the list before + # returning it. + return list(set([ + op['name'] + for op in broker_req['ops'] + if op['op'] == 'create-pool' + ])) From 5b2a2fee9ca3a1b36808677eec45f1e686d1c2d0 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 31 Aug 2020 15:49:14 +0200 Subject: [PATCH 691/898] unit_tests: Replace patch decorators with BaseTestCase helpers --- unit_tests/charm_tests/test_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/unit_tests/charm_tests/test_utils.py b/unit_tests/charm_tests/test_utils.py index 2a494d6..8e0661b 100644 --- a/unit_tests/charm_tests/test_utils.py +++ b/unit_tests/charm_tests/test_utils.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import zaza.openstack.charm_tests.test_utils as test_utils -from unittest.mock import patch +import unit_tests.utils as ut_utils class TestBaseCharmTest(unittest.TestCase): @@ -38,15 +37,16 @@ class TestBaseCharmTest(unittest.TestCase): }), 'aValue') -class TestOpenStackBaseTest(unittest.TestCase): +class TestOpenStackBaseTest(ut_utils.BaseTestCase): - @patch.object(test_utils.openstack_utils, 'get_cacert') - @patch.object(test_utils.openstack_utils, 'get_overcloud_keystone_session') - @patch.object(test_utils.BaseCharmTest, 'setUpClass') - def test_setUpClass(self, _setUpClass, _get_ovcks, _get_cacert): + def test_setUpClass(self): + self.patch_object(test_utils.openstack_utils, 'get_cacert') + self.patch_object(test_utils.openstack_utils, + 'get_overcloud_keystone_session') + self.patch_object(test_utils.BaseCharmTest, 'setUpClass') class MyTestClass(test_utils.OpenStackBaseTest): model_name = 'deadbeef' MyTestClass.setUpClass('foo', 'bar') - _setUpClass.assert_called_with('foo', 'bar') + self.setUpClass.assert_called_with('foo', 'bar') From 8a8178738d05a0003a6784acea6ba3e7f58a19d2 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 31 Aug 2020 16:26:14 +0200 Subject: [PATCH 692/898] Add support for resetting back to charm default on config_change Due to python-libjuju's requirement for coercing every config value into strings, some configuration type/values are not possible to specify the default for. Allow the config_change helper to optionally use reset back to charm default instead of specifying default config values in a dictionary. --- unit_tests/charm_tests/test_utils.py | 73 +++++++++++++++++++++++- zaza/openstack/charm_tests/test_utils.py | 27 +++++++-- 2 files changed, 93 insertions(+), 7 deletions(-) diff --git a/unit_tests/charm_tests/test_utils.py b/unit_tests/charm_tests/test_utils.py index 8e0661b..7ca3ba5 100644 --- a/unit_tests/charm_tests/test_utils.py +++ b/unit_tests/charm_tests/test_utils.py @@ -12,12 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +from unittest import mock + import zaza.openstack.charm_tests.test_utils as test_utils import unit_tests.utils as ut_utils -class TestBaseCharmTest(unittest.TestCase): +class TestBaseCharmTest(ut_utils.BaseTestCase): + + def setUp(self): + super(TestBaseCharmTest, self).setUp() + self.target = test_utils.BaseCharmTest() + + def patch_target(self, attr, return_value=None): + mocked = mock.patch.object(self.target, attr) + self._patches[attr] = mocked + started = mocked.start() + started.return_value = return_value + self._patches_start[attr] = started + setattr(self, attr, started) def test_get_my_tests_options(self): @@ -36,6 +50,63 @@ class TestBaseCharmTest(unittest.TestCase): }, }), 'aValue') + def test_config_change(self): + default_config = {'fakeKey': 'testProvidedDefault'} + alterna_config = {'fakeKey': 'testProvidedAlterna'} + self.target.model_name = 'aModel' + self.target.test_config = {} + self.patch_target('config_current') + self.config_current.return_value = default_config + self.patch_object(test_utils.model, 'set_application_config') + self.patch_object(test_utils.model, 'wait_for_agent_status') + self.patch_object(test_utils.model, 'wait_for_application_states') + self.patch_object(test_utils.model, 'block_until_all_units_idle') + with self.target.config_change( + default_config, alterna_config, application_name='anApp'): + self.set_application_config.assert_called_once_with( + 'anApp', alterna_config, model_name='aModel') + self.wait_for_agent_status.assert_called_once_with( + model_name='aModel') + self.wait_for_application_states.assert_called_once_with( + model_name='aModel', states={}) + self.block_until_all_units_idle.assert_called_once_with() + # after yield we will have different calls than the above, measure both + self.set_application_config.assert_has_calls([ + mock.call('anApp', alterna_config, model_name='aModel'), + mock.call('anApp', default_config, model_name='aModel'), + ]) + self.wait_for_application_states.assert_has_calls([ + mock.call(model_name='aModel', states={}), + mock.call(model_name='aModel', states={}), + ]) + self.block_until_all_units_idle.assert_has_calls([ + mock.call(), + mock.call(), + ]) + # confirm operation with `reset_to_charm_default` + self.set_application_config.reset_mock() + self.wait_for_agent_status.reset_mock() + self.wait_for_application_states.reset_mock() + self.patch_object(test_utils.model, 'reset_application_config') + with self.target.config_change( + default_config, alterna_config, application_name='anApp', + reset_to_charm_default=True): + self.set_application_config.assert_called_once_with( + 'anApp', alterna_config, model_name='aModel') + # we want to assert this not to be called after yield + self.set_application_config.reset_mock() + self.assertFalse(self.set_application_config.called) + self.reset_application_config.assert_called_once_with( + 'anApp', alterna_config.keys(), model_name='aModel') + self.wait_for_application_states.assert_has_calls([ + mock.call(model_name='aModel', states={}), + mock.call(model_name='aModel', states={}), + ]) + self.block_until_all_units_idle.assert_has_calls([ + mock.call(), + mock.call(), + ]) + class TestOpenStackBaseTest(ut_utils.BaseTestCase): diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index ee61336..9ba6955 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -182,7 +182,7 @@ class BaseCharmTest(unittest.TestCase): @contextlib.contextmanager def config_change(self, default_config, alternate_config, - application_name=None): + application_name=None, reset_to_charm_default=False): """Run change config tests. Change config to `alternate_config`, wait for idle workload status, @@ -202,6 +202,12 @@ class BaseCharmTest(unittest.TestCase): by a charm under test other than the object's application. :type application_name: str + :param reset_to_charm_default: When True we will ask Juju to reset each + configuration option mentioned in the + `alternate_config` dictionary back to + the charm default and ignore the + `default_config` dictionary. + :type reset_to_charm_default: bool """ if not application_name: application_name = self.application_name @@ -246,11 +252,20 @@ class BaseCharmTest(unittest.TestCase): yield - logging.debug('Restoring charm setting to {}'.format(default_config)) - model.set_application_config( - application_name, - self._stringed_value_config(default_config), - model_name=self.model_name) + if reset_to_charm_default: + logging.debug('Resetting these charm configuration options to the ' + 'charm default: "{}"' + .format(alternate_config.keys())) + model.reset_application_config(application_name, + alternate_config.keys(), + model_name=self.model_name) + else: + logging.debug('Restoring charm setting to {}' + .format(default_config)) + model.set_application_config( + application_name, + self._stringed_value_config(default_config), + model_name=self.model_name) logging.debug( 'Waiting for units to reach target states') From 89ea43d0d51e6a245cbe20b92ee7228ed4a4df1c Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 15 Sep 2020 20:29:48 +0200 Subject: [PATCH 693/898] Log resource cleanup failure and carry on Fixes #417 --- zaza/openstack/charm_tests/test_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index ee61336..250c183 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -495,6 +495,11 @@ class OpenStackBaseTest(BaseCharmTest): self.nova_client.servers, server.id, msg="server") + except AssertionError as e: + # Resource failed to be removed within the expected time frame, + # log this fact and carry on. + logging.warning('Gave up waiting for resource cleanup: "{}"' + .format(str(e))) except AttributeError: # Test did not define self.RESOURCE_PREFIX, ignore. pass From 33ea7acb7e2ebdbdf21bd5096afff9f73f35e5e6 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 16 Sep 2020 11:24:16 +0200 Subject: [PATCH 694/898] Make test_manila_share more robust by retrying launching test instances on failure. https://bugs.launchpad.net/charm-manila-ganesha/+bug/1895636 --- zaza/openstack/charm_tests/manila_ganesha/tests.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 2f9f8be..9d9db70 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -62,13 +62,7 @@ packages: share_proto="nfs", size=1) # Spawn Servers - instance_1 = guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name='{}-ins-1'.format(self.RESOURCE_PREFIX), - userdata=self.INSTANCE_USERDATA) - instance_2 = guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name='{}-ins-2'.format(self.RESOURCE_PREFIX), + instance_1, instance_2 = self.launch_guests( userdata=self.INSTANCE_USERDATA) fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] From c5a3f832d014b28dd997356416af3ca082434d4c Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 16 Sep 2020 12:04:42 +0200 Subject: [PATCH 695/898] test_utils/config_change: Wait for agent to start executing Avoid returning prior to the config change has even started. Fixes: #419 --- zaza/openstack/charm_tests/test_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 9ba6955..e6eca75 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -267,6 +267,9 @@ class BaseCharmTest(unittest.TestCase): self._stringed_value_config(default_config), model_name=self.model_name) + logging.debug( + 'Waiting for units to execute config-changed hook') + model.wait_for_agent_status(model_name=self.model_name) logging.debug( 'Waiting for units to reach target states') model.wait_for_application_states( From 1ff71db4de7d3d4422efd2ab3e3e18060481fba0 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 17 Sep 2020 16:17:00 +0200 Subject: [PATCH 696/898] Make linter happy --- zaza/openstack/charm_tests/manila_ganesha/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 9d9db70..27009bf 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -20,7 +20,6 @@ from tenacity import Retrying, stop_after_attempt, wait_exponential from manilaclient import client as manilaclient -import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.neutron.tests as neutron_tests import zaza.openstack.charm_tests.nova.utils as nova_utils import zaza.openstack.charm_tests.test_utils as test_utils From a1f3a8710fe916025f2c9aadbf26415d6d155074 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 14 Sep 2020 12:45:11 +0200 Subject: [PATCH 697/898] Teach Zaza to determine release pair for Ceph deployments As Ceph is distributed as part of Ubuntu Cloud Archive, the pockets lend their name from the accompanying OpenStack release. --- zaza/openstack/utilities/openstack.py | 5 +++++ zaza/openstack/utilities/os_versions.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 0c31c05..cc07afb 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -119,6 +119,10 @@ CHARM_TYPES = { 'pkg': 'ovn-common', 'origin_setting': 'source' }, + 'ceph-mon': { + 'pkg': 'ceph-common', + 'origin_setting': 'source' + }, } # Older tests use the order the services appear in the list to imply @@ -138,6 +142,7 @@ UPGRADE_SERVICES = [ {'name': 'openstack-dashboard', 'type': CHARM_TYPES['openstack-dashboard']}, {'name': 'ovn-central', 'type': CHARM_TYPES['ovn-central']}, + {'name': 'ceph-mon', 'type': CHARM_TYPES['ceph-mon']}, ] diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index b46fc21..75dc723 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -249,4 +249,11 @@ PACKAGE_CODENAMES = { ('2', 'train'), ('20', 'ussuri'), ]), + 'ceph-common': OrderedDict([ + ('10', 'mitaka'), # jewel + ('12', 'queens'), # luminous + ('13', 'rocky'), # mimic + ('14', 'train'), # nautilus + ('15', 'ussuri'), # octopus + ]), } From 4db2202b1ec0818c91463759bacb25b834da54f3 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 28 Aug 2020 14:19:26 +0200 Subject: [PATCH 698/898] ceph: Add functional tests for BlueStore compression --- zaza/openstack/charm_tests/ceph/tests.py | 138 +++++++++++++++++++++++ 1 file changed, 138 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 99285a3..19ce1ec 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -872,3 +872,141 @@ def _get_mon_count_from_prometheus(prometheus_ip): response = client.get(url) logging.debug("Prometheus response: {}".format(response.json())) return response.json()['data']['result'][0]['value'][1] + + +class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest): + """Test charm handling of bluestore compression configuration options.""" + + @classmethod + def setUpClass(cls): + """Perform class one time initialization.""" + super(BlueStoreCompressionCharmOperation, cls).setUpClass() + cls.current_release = zaza_openstack.get_os_release( + zaza_openstack.get_current_os_release_pair( + application='ceph-mon')) + cls.bionic_rocky = zaza_openstack.get_os_release('bionic_rocky') + + def setUp(self): + """Perform common per test initialization steps.""" + super(BlueStoreCompressionCharmOperation, self).setUp() + + # determine if the tests should be run or not + logging.debug('os_release: {} >= {} = {}' + .format(self.current_release, + self.bionic_rocky, + self.current_release >= self.bionic_rocky)) + self.mimic_or_newer = self.current_release >= self.bionic_rocky + + def _assert_pools_properties(self, pools, pools_detail, + expected_properties, log_func=logging.info): + """Check properties on a set of pools. + + :param pools: List of pool names to check. + :type pools: List[str] + :param pools_detail: List of dictionaries with pool detail + :type pools_detail List[Dict[str,any]] + :param expected_properties: Properties to check and their expected + values. + :type expected_properties: Dict[str,any] + :returns: Nothing + :raises: AssertionError + """ + for pool in pools: + for pd in pools_detail: + if pd['pool_name'] == pool: + if 'options' in expected_properties: + for k, v in expected_properties['options'].items(): + self.assertEquals(pd['options'][k], v) + log_func("['options']['{}'] == {}".format(k, v)) + for k, v in expected_properties.items(): + if k == 'options': + continue + self.assertEquals(pd[k], v) + log_func("{} == {}".format(k, v)) + + def test_configure_compression(self): + """Enable compression and validate properties flush through to pool.""" + if not self.mimic_or_newer: + logging.info('Skipping test, Mimic or newer required.') + return + if self.application_name == 'ceph-osd': + # The ceph-osd charm itself does not request pools, neither does + # the BlueStore Compression configuration options it have affect + # pool properties. + logging.info('test does not apply to ceph-osd charm.') + return + elif self.application_name == 'ceph-radosgw': + # The Ceph RadosGW creates many light weight pools to keep track of + # metadata, we only compress the pool containing actual data. + app_pools = ['.rgw.buckets.data'] + else: + # Retrieve which pools the charm under test has requested skipping + # metadata pools as they are deliberately not compressed. + app_pools = [ + pool + for pool in zaza_ceph.get_pools_from_broker_req( + self.application_name, model_name=self.model_name) + if 'metadata' not in pool + ] + + ceph_pools_detail = zaza_ceph.get_ceph_pool_details( + model_name=self.model_name) + + logging.debug('BEFORE: {}'.format(ceph_pools_detail)) + try: + logging.info('Checking Ceph pool compression_mode prior to change') + self._assert_pools_properties( + app_pools, ceph_pools_detail, + {'options': {'compression_mode': 'none'}}) + except KeyError: + logging.info('property does not exist on pool, which is OK.') + logging.info('Changing "bluestore-compression-mode" to "force" on {}' + .format(self.application_name)) + with self.config_change( + {'bluestore-compression-mode': 'none'}, + {'bluestore-compression-mode': 'force'}): + # Retrieve pool details from Ceph after changing configuration + ceph_pools_detail = zaza_ceph.get_ceph_pool_details( + model_name=self.model_name) + logging.debug('CONFIG_CHANGE: {}'.format(ceph_pools_detail)) + logging.info('Checking Ceph pool compression_mode after to change') + self._assert_pools_properties( + app_pools, ceph_pools_detail, + {'options': {'compression_mode': 'force'}}) + ceph_pools_detail = zaza_ceph.get_ceph_pool_details( + model_name=self.model_name) + logging.debug('AFTER: {}'.format(ceph_pools_detail)) + logging.debug(zaza_juju.get_relation_from_unit( + 'ceph-mon', self.application_name, None, + model_name=self.model_name)) + logging.info('Checking Ceph pool compression_mode after restoring ' + 'config to previous value') + self._assert_pools_properties( + app_pools, ceph_pools_detail, + {'options': {'compression_mode': 'none'}}) + + def test_invalid_compression_configuration(self): + """Set invalid configuration and validate charm response.""" + if not self.mimic_or_newer: + logging.info('Skipping test, Mimic or newer required.') + return + stored_target_deploy_status = self.test_config.get( + 'target_deploy_status', {}) + new_target_deploy_status = stored_target_deploy_status.copy() + new_target_deploy_status[self.application_name] = { + 'workload-status': 'blocked', + 'workload-status-message': 'Invalid configuration', + } + if 'target_deploy_status' in self.test_config: + self.test_config['target_deploy_status'].update( + new_target_deploy_status) + else: + self.test_config['target_deploy_status'] = new_target_deploy_status + + with self.config_change( + {'bluestore-compression-mode': 'none'}, + {'bluestore-compression-mode': 'PEBCAK'}): + logging.info('Charm went into blocked state as expected, restore ' + 'configuration') + self.test_config[ + 'target_deploy_status'] = stored_target_deploy_status From 540bd91cc0a13068103cf9d9ee3c9dd21653a942 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 18 Sep 2020 09:46:08 +0100 Subject: [PATCH 699/898] Add Ceph-iscsi Erasure coding check (#412) * Add Ceph-iscsi Erasure coding check Add a test which created a ceph iscsi target backed by an erasure coded pool. * Update action params inline with charm change * Fix EC pool type --- .../openstack/charm_tests/ceph/iscsi/setup.py | 18 ++--- .../openstack/charm_tests/ceph/iscsi/tests.py | 79 +++++++++++++++---- 2 files changed, 73 insertions(+), 24 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/setup.py b/zaza/openstack/charm_tests/ceph/iscsi/setup.py index 8662f8b..abd6a73 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/setup.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/setup.py @@ -19,12 +19,12 @@ import zaza.model def basic_guest_setup(): """Run basic setup for iscsi guest.""" - unit = zaza.model.get_units('ubuntu')[0] - setup_cmds = [ - "apt install --yes open-iscsi multipath-tools", - "systemctl start iscsi", - "systemctl start iscsid"] - for cmd in setup_cmds: - zaza.model.run_on_unit( - unit.entity_id, - cmd) + for unit in zaza.model.get_units('ubuntu'): + setup_cmds = [ + "apt install --yes open-iscsi multipath-tools", + "systemctl start iscsi", + "systemctl start iscsid"] + for cmd in setup_cmds: + zaza.model.run_on_unit( + unit.entity_id, + cmd) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index 3c904a6..08a7607 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -26,7 +26,10 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): """Class for `ceph-iscsi` tests.""" GW_IQN = "iqn.2003-03.com.canonical.iscsi-gw:iscsi-igw" - DATA_POOL_NAME = 'superssd' + DATA_POOL_NAME = 'zaza_rep_pool' + EC_PROFILE_NAME = 'zaza_iscsi' + EC_DATA_POOL = 'zaza_ec_data_pool' + EC_METADATA_POOL = 'zaza_ec_metadata_pool' def get_client_initiatorname(self, unit): """Return the initiatorname for the given unit.""" @@ -48,18 +51,16 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): initiatorname = line.split('=')[1].rstrip() return initiatorname - def get_ctxt(self): + def get_base_ctxt(self): """Generate a context for running gwcli commands to create a target.""" gw_units = zaza.model.get_units('ceph-iscsi') - client_units = zaza.model.get_units('ubuntu') - client = client_units[0] - self.get_client_initiatorname(client.entity_id) primary_gw = gw_units[0] secondary_gw = gw_units[1] host_names = generic_utils.get_unit_hostnames(gw_units, fqdn=True) + client_entity_ids = [ + u.entity_id for u in zaza.model.get_units('ubuntu')] ctxt = { - 'pool_name': self.DATA_POOL_NAME, - 'client_entity_id': client.entity_id, + 'client_entity_ids': sorted(client_entity_ids), 'gw_iqn': self.GW_IQN, 'gw1_ip': primary_gw.public_address, 'gw1_hostname': host_names[primary_gw.entity_id], @@ -67,13 +68,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): 'gw2_ip': secondary_gw.public_address, 'gw2_hostname': host_names[secondary_gw.entity_id], 'gw2_entity_id': secondary_gw.entity_id, - 'img_size': '1G', - 'img_name': 'disk_1', - 'chap_username': 'myiscsiusername', - 'chap_password': 'myiscsipassword', 'chap_creds': 'username={chap_username} password={chap_password}', - 'client_initiatorname': self.get_client_initiatorname( - client.entity_id), 'gwcli_gw_dir': '/iscsi-targets/{gw_iqn}/gateways', 'gwcli_hosts_dir': '/iscsi-targets/{gw_iqn}/hosts', 'gwcli_disk_dir': '/disks', @@ -103,7 +98,8 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): ctxt['gw1_entity_id'], ctxt['gw2_entity_id']), 'iqn': self.GW_IQN, - 'pool-name': self.DATA_POOL_NAME, + 'rbd-pool-name': ctxt['pool_name'], + 'ec-rbd-metadata-pool': ctxt.get('ec_meta_pool_name', ''), 'image-size': ctxt['img_size'], 'image-name': ctxt['img_name'], 'client-initiatorname': ctxt['client_initiatorname'], @@ -140,10 +136,63 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): action_params={ 'name': self.DATA_POOL_NAME})) + def create_ec_data_pool(self): + """Create data pool to back iscsi targets.""" + generic_utils.assertActionRanOK(zaza.model.run_action_on_leader( + 'ceph-mon', + 'create-erasure-profile', + action_params={ + 'name': self.EC_PROFILE_NAME, + 'coding-chunks': 2, + 'data-chunks': 4, + 'plugin': 'jerasure'})) + generic_utils.assertActionRanOK(zaza.model.run_action_on_leader( + 'ceph-mon', + 'create-pool', + action_params={ + 'name': self.EC_DATA_POOL, + 'pool-type': 'erasure-coded', + 'allow-ec-overwrites': True, + 'erasure-profile-name': self.EC_PROFILE_NAME})) + generic_utils.assertActionRanOK(zaza.model.run_action_on_leader( + 'ceph-mon', + 'create-pool', + action_params={ + 'name': self.EC_METADATA_POOL})) + def test_create_and_mount_volume(self): """Test creating a target and mounting it on a client.""" self.create_data_pool() - ctxt = self.get_ctxt() + ctxt = self.get_base_ctxt() + client_entity_id = ctxt['client_entity_ids'][0] + ctxt.update({ + 'client_entity_id': client_entity_id, + 'client_initiatorname': self.get_client_initiatorname( + client_entity_id), + 'pool_name': self.DATA_POOL_NAME, + 'chap_username': 'myiscsiusername1', + 'chap_password': 'myiscsipassword1', + 'img_size': '1G', + 'img_name': 'disk_rep_1'}) + self.create_iscsi_target(ctxt) + self.mount_iscsi_target(ctxt) + self.check_client_device(ctxt) + + def test_create_and_mount_ec_backed_volume(self): + """Test creating an EC backed target and mounting it on a client.""" + self.create_ec_data_pool() + ctxt = self.get_base_ctxt() + client_entity_id = ctxt['client_entity_ids'][1] + ctxt.update({ + 'client_entity_id': client_entity_id, + 'client_initiatorname': self.get_client_initiatorname( + client_entity_id), + 'pool_name': self.EC_DATA_POOL, + 'ec_meta_pool_name': self.EC_METADATA_POOL, + 'chap_username': 'myiscsiusername2', + 'chap_password': 'myiscsipassword2', + 'img_size': '2G', + 'img_name': 'disk_ec_1'}) self.create_iscsi_target(ctxt) self.mount_iscsi_target(ctxt) self.check_client_device(ctxt) From e9215f1d73186cc7246ad7f5c74941435a72e89f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 16 Sep 2020 08:17:25 +0200 Subject: [PATCH 700/898] Retry load balancer resource creation Creating a load balancer consist of many independent API calls. If any one of them fail today the whole job will be killed. On failure to create a lb resoruce, clean up and retry. --- zaza/openstack/charm_tests/octavia/tests.py | 44 ++++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index e84ac3e..1bdc5e4 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -18,6 +18,7 @@ import logging import subprocess import tenacity +from keystoneauth1 import exceptions as keystone_exceptions import osc_lib.exceptions import zaza.openstack.charm_tests.test_utils as test_utils @@ -80,8 +81,15 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): # List of floating IPs created by this test cls.fips = [] - def resource_cleanup(self): - """Remove resources created during test execution.""" + @tenacity.retry(stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)) + def resource_cleanup(self, only_local=False): + """Remove resources created during test execution. + + :param only_local: When set to true do not call parent method + :type only_local: bool + """ for lb in self.loadbalancers: self.octavia_client.load_balancer_delete(lb['id'], cascade=True) try: @@ -90,8 +98,16 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): provisioning_status='DELETED') except osc_lib.exceptions.NotFound: pass + # allow resource cleanup to be run multiple times + self.loadbalancers = [] for fip in self.fips: self.neutron_client.delete_floatingip(fip) + # allow resource cleanup to be run multiple times + self.fips = [] + + if only_local: + return + # we run the parent resource_cleanup last as it will remove instances # referenced as members in the above cleaned up load balancers super(LBAASv2Test, self).resource_cleanup() @@ -157,6 +173,7 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): 'provider': provider, }}) lb = result['loadbalancer'] + self.loadbalancers.append(lb) lb_id = lb['id'] logging.info('Awaiting loadbalancer to reach provisioning_status ' @@ -283,10 +300,25 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): for provider in self.get_lb_providers(self.octavia_client).keys(): logging.info('Creating loadbalancer with provider {}' .format(provider)) - lb = self._create_lb_resources(self.octavia_client, provider, - vip_subnet_id, subnet_id, - payload_ips) - self.loadbalancers.append(lb) + final_exc = None + # NOTE: we cannot use tenacity here as the method we call into + # already uses it to wait for operations to complete. + for retry in range(0, 3): + try: + lb = self._create_lb_resources(self.octavia_client, + provider, + vip_subnet_id, + subnet_id, + payload_ips) + break + except (AssertionError, + keystone_exceptions.connection.ConnectFailure) as e: + logging.info('Retrying load balancer creation, last ' + 'failure: "{}"'.format(str(e))) + self.resource_cleanup(only_local=True) + final_exc = e + else: + raise final_exc lb_fp = openstack_utils.create_floating_ip( self.neutron_client, 'ext_net', port={'id': lb['vip_port_id']}) From 430704ef80c40f8a02ab0643e83449e35a6d70f0 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 16 Sep 2020 11:01:53 +0200 Subject: [PATCH 701/898] Retry retrofit action The retrofit process involves downloading packages from the internet and is as such susceptible to random failures due to internet gremlins. --- .../octavia/diskimage_retrofit/setup.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/diskimage_retrofit/setup.py b/zaza/openstack/charm_tests/octavia/diskimage_retrofit/setup.py index 7fbca5e..73d1aad 100644 --- a/zaza/openstack/charm_tests/octavia/diskimage_retrofit/setup.py +++ b/zaza/openstack/charm_tests/octavia/diskimage_retrofit/setup.py @@ -15,6 +15,7 @@ """Code for configuring octavia-diskimage-retrofit.""" import logging +import tenacity import zaza.model @@ -39,12 +40,20 @@ def retrofit_amphora_image(unit='octavia-diskimage-retrofit/0', if image_id: params.update({'source-image': image_id}) - # NOTE(fnordahl) ``zaza.model.run_action_on_leader`` fails here, - # apparently has to do with handling of subordinates in ``libjuju`` or - # ``juju`` itself. - action = zaza.model.run_action( - unit, - 'retrofit-image', - action_params=params, - raise_on_failure=True) + # NOTE(fnordahl) the retrofit process involves downloading packages from + # the internet and is as such susceptible to random failures due to + # internet gremlins. + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)): + with attempt: + # NOTE(fnordahl) ``zaza.model.run_action_on_leader`` fails here, + # apparently has to do with handling of subordinates in ``libjuju`` + # or ``juju`` itself. + action = zaza.model.run_action( + unit, + 'retrofit-image', + action_params=params, + raise_on_failure=True) return action From ced5b3dd13f84b23371fde76e309e6af4a56f048 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 17 Sep 2020 14:31:57 +0200 Subject: [PATCH 702/898] Retry scp/ssh operations in `add_interface_to_netplan` helper Fixes #420 --- zaza/openstack/utilities/openstack.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 0c31c05..1e8f625 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -690,14 +690,20 @@ def add_interface_to_netplan(server_name, mac_address): "{}\nserver_name: {}".format(body_value, unit_name, interface, mac_address, server_name)) - with tempfile.NamedTemporaryFile(mode="w") as netplan_file: - netplan_file.write(body_value) - netplan_file.flush() - model.scp_to_unit(unit_name, netplan_file.name, - '/home/ubuntu/60-dataport.yaml', user="ubuntu") - run_cmd_mv = "sudo mv /home/ubuntu/60-dataport.yaml /etc/netplan/" - model.run_on_unit(unit_name, run_cmd_mv) - model.run_on_unit(unit_name, "sudo netplan apply") + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)): + with attempt: + with tempfile.NamedTemporaryFile(mode="w") as netplan_file: + netplan_file.write(body_value) + netplan_file.flush() + model.scp_to_unit( + unit_name, netplan_file.name, + '/home/ubuntu/60-dataport.yaml', user="ubuntu") + run_cmd_mv = "sudo mv /home/ubuntu/60-dataport.yaml /etc/netplan/" + model.run_on_unit(unit_name, run_cmd_mv) + model.run_on_unit(unit_name, "sudo netplan apply") def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, From c5786d9184629cd7c6b60aae63e6b1e946e4360b Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 21 Sep 2020 13:40:48 +0100 Subject: [PATCH 703/898] Ceph iscsi default pool test (#415) * Add Ceph-iscsi Erasure coding check Add a test which created a ceph iscsi target backed by an erasure coded pool. * Fix EC pool type * Test ceph iscsi target from default pool --- .../openstack/charm_tests/ceph/iscsi/tests.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index 08a7607..a377f0f 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -98,7 +98,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): ctxt['gw1_entity_id'], ctxt['gw2_entity_id']), 'iqn': self.GW_IQN, - 'rbd-pool-name': ctxt['pool_name'], + 'rbd-pool-name': ctxt.get('pool_name', ''), 'ec-rbd-metadata-pool': ctxt.get('ec_meta_pool_name', ''), 'image-size': ctxt['img_size'], 'image-name': ctxt['img_name'], @@ -197,6 +197,23 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): self.mount_iscsi_target(ctxt) self.check_client_device(ctxt) + def test_create_and_mount_volume_default_pool(self): + """Test creating a target and mounting it on a client.""" + self.create_data_pool() + ctxt = self.get_base_ctxt() + client_entity_id = ctxt['client_entity_ids'][2] + ctxt.update({ + 'client_entity_id': client_entity_id, + 'client_initiatorname': self.get_client_initiatorname( + client_entity_id), + 'chap_username': 'myiscsiusername3', + 'chap_password': 'myiscsipassword3', + 'img_size': '3G', + 'img_name': 'disk_default_1'}) + self.create_iscsi_target(ctxt) + self.mount_iscsi_target(ctxt) + self.check_client_device(ctxt) + def test_pause_resume(self): """Test pausing and resuming a unit.""" with self.pause_resume( From be59370476eaeb8ca0d6e1adca58c9acfa8faa77 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 22 Sep 2020 14:50:09 +0200 Subject: [PATCH 704/898] Forcefully remove amphorae when Octavia is unable to remove --- zaza/openstack/charm_tests/octavia/tests.py | 42 ++++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 1bdc5e4..eb0b217 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -19,6 +19,7 @@ import subprocess import tenacity from keystoneauth1 import exceptions as keystone_exceptions +import octaviaclient.api.v2.octavia import osc_lib.exceptions import zaza.openstack.charm_tests.test_utils as test_utils @@ -81,6 +82,27 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): # List of floating IPs created by this test cls.fips = [] + def _remove_amphorae_instances(self): + """Remove amphorae instances forcefully. + + In some situations Octavia is unable to remove load balancer resources. + This helper can be used to remove the underlying instances. + """ + result = self.octavia_client.amphora_list() + for amphora in result.get('amphorae', []): + for server in self.nova_client.servers.list(): + if 'compute_id' in amphora and server.id == amphora[ + 'compute_id']: + try: + openstack_utils.delete_resource( + self.nova_client.servers, + server.id, + msg="server") + except AssertionError as e: + logging.warning( + 'Gave up waiting for resource cleanup: "{}"' + .format(str(e))) + @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential( multiplier=1, min=2, max=10)) @@ -91,13 +113,21 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): :type only_local: bool """ for lb in self.loadbalancers: - self.octavia_client.load_balancer_delete(lb['id'], cascade=True) try: - self.wait_for_lb_resource( - self.octavia_client.load_balancer_show, lb['id'], - provisioning_status='DELETED') - except osc_lib.exceptions.NotFound: - pass + self.octavia_client.load_balancer_delete( + lb['id'], cascade=True) + except octaviaclient.api.v2.octavia.OctaviaClientException as e: + logging.info('Octavia is unable to delete load balancer: "{}"' + .format(e)) + logging.info('Attempting to forcefully remove amphorae') + self._remove_amphorae_instances() + else: + try: + self.wait_for_lb_resource( + self.octavia_client.load_balancer_show, lb['id'], + provisioning_status='DELETED') + except osc_lib.exceptions.NotFound: + pass # allow resource cleanup to be run multiple times self.loadbalancers = [] for fip in self.fips: From 0a458766c25662f85ae383d29b005f036a76c1fa Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 18 Sep 2020 13:34:26 +0000 Subject: [PATCH 705/898] Extend iSCSI tests. Extend iSCSI tests to mount multipath device and write data to it Then unmount it and logout of the iscsi target. Finally log back into the iscsi target, mount device and check data is present. --- .../openstack/charm_tests/ceph/iscsi/tests.py | 79 +++++++++++++++---- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index a377f0f..19357f6 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -107,8 +107,9 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): 'client-password': ctxt['chap_password'] })) - def mount_iscsi_target(self, ctxt): + def login_iscsi_target(self, ctxt): """Mount iscsi target on client.""" + logging.info("Logging in to iscsi target") base_op_cmd = ('iscsiadm --mode node --targetname {gw_iqn} ' '--op=update ').format(**ctxt) setup_cmds = [ @@ -119,14 +120,56 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): 'iscsiadm --mode node --targetname {gw_iqn} --login'] self.run_commands(ctxt['client_entity_id'], setup_cmds, ctxt) - def check_client_device(self, ctxt): - """Wait for multipath device to appear on client.""" + def logout_iscsi_targets(self, ctxt): + """Logout of iscsi target on client.""" + logging.info("Logging out of iscsi target") + logout_cmds = [ + 'iscsiadm --mode node --logoutall=all'] + self.run_commands(ctxt['client_entity_id'], logout_cmds, ctxt) + + def check_client_device(self, ctxt, init_client=True): + """Wait for multipath device to appear on client and test access.""" + logging.info("Checking multipath device is present.") + device_ctxt = { + 'bdevice': '/dev/dm-0', + 'mount_point': '/mnt/iscsi', + 'test_file': '/mnt/iscsi/test.data'} + ls_bdevice_cmd = 'ls -l {bdevice}' + mkfs_cmd = 'mke2fs {bdevice}' + mkdir_cmd = 'mkdir {mount_point}' + mount_cmd = 'mount {bdevice} {mount_point}' + umount_cmd = 'umount {mount_point}' + check_mounted_cmd = 'mountpoint {mount_point}' + write_cmd = 'truncate -s 1M {test_file}' + check_file = 'ls -l {test_file}' + if init_client: + commands = [ + mkfs_cmd, + mkdir_cmd, + mount_cmd, + check_mounted_cmd, + write_cmd, + check_file, + umount_cmd] + else: + commands = [ + mount_cmd, + check_mounted_cmd, + check_file, + umount_cmd] + async def check_device_present(): run = await zaza.model.async_run_on_unit( ctxt['client_entity_id'], - 'ls -l /dev/dm-0') - return '/dev/dm-0' in run['Stdout'] + ls_bdevice_cmd.format(bdevice=device_ctxt['bdevice'])) + return device_ctxt['bdevice'] in run['Stdout'] + + logging.info("Checking {} is present on {}".format( + device_ctxt['bdevice'], + ctxt['client_entity_id'])) zaza.model.block_until(check_device_present) + logging.info("Checking mounting device and access") + self.run_commands(ctxt['client_entity_id'], commands, device_ctxt) def create_data_pool(self): """Create data pool to back iscsi targets.""" @@ -160,6 +203,20 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): action_params={ 'name': self.EC_METADATA_POOL})) + def run_client_check(self, ctxt): + """Check access to mulipath device. + + Write a filesystem to device, mount it and write data. Then unmount + and logout the iscsi target, finally reconnect and remount checking + data is still present. + """ + self.create_iscsi_target(ctxt) + self.login_iscsi_target(ctxt) + self.check_client_device(ctxt, init_client=True) + self.logout_iscsi_targets(ctxt) + self.login_iscsi_target(ctxt) + self.check_client_device(ctxt, init_client=False) + def test_create_and_mount_volume(self): """Test creating a target and mounting it on a client.""" self.create_data_pool() @@ -174,9 +231,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): 'chap_password': 'myiscsipassword1', 'img_size': '1G', 'img_name': 'disk_rep_1'}) - self.create_iscsi_target(ctxt) - self.mount_iscsi_target(ctxt) - self.check_client_device(ctxt) + self.run_client_checks(ctxt) def test_create_and_mount_ec_backed_volume(self): """Test creating an EC backed target and mounting it on a client.""" @@ -193,9 +248,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): 'chap_password': 'myiscsipassword2', 'img_size': '2G', 'img_name': 'disk_ec_1'}) - self.create_iscsi_target(ctxt) - self.mount_iscsi_target(ctxt) - self.check_client_device(ctxt) + self.run_client_checks(ctxt) def test_create_and_mount_volume_default_pool(self): """Test creating a target and mounting it on a client.""" @@ -210,9 +263,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): 'chap_password': 'myiscsipassword3', 'img_size': '3G', 'img_name': 'disk_default_1'}) - self.create_iscsi_target(ctxt) - self.mount_iscsi_target(ctxt) - self.check_client_device(ctxt) + self.run_client_checks(ctxt) def test_pause_resume(self): """Test pausing and resuming a unit.""" From eda2dfb450768da4cdb2dd86f7408145b9acb60a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 23 Sep 2020 10:22:36 +0000 Subject: [PATCH 706/898] Typo fix --- zaza/openstack/charm_tests/ceph/iscsi/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index 19357f6..2665c10 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -203,7 +203,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): action_params={ 'name': self.EC_METADATA_POOL})) - def run_client_check(self, ctxt): + def run_client_checks(self, ctxt): """Check access to mulipath device. Write a filesystem to device, mount it and write data. Then unmount From c3105657d4623c8b5f1314daff8ae59aac1d2fab Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 23 Sep 2020 11:09:31 +0000 Subject: [PATCH 707/898] Fix doc strings --- .../openstack/charm_tests/ceph/iscsi/tests.py | 70 +++++++++++++++---- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index 2665c10..17775b7 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -32,7 +32,13 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): EC_METADATA_POOL = 'zaza_ec_metadata_pool' def get_client_initiatorname(self, unit): - """Return the initiatorname for the given unit.""" + """Return the initiatorname for the given unit. + + :param unit_name: Name of unit to match + :type unit: str + :returns: Initiator name + :rtype: str + """ generic_utils.assertRemoteRunOK(zaza.model.run_on_unit( unit, ('cp /etc/iscsi/initiatorname.iscsi /tmp; ' @@ -52,7 +58,11 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): return initiatorname def get_base_ctxt(self): - """Generate a context for running gwcli commands to create a target.""" + """Generate a context for running gwcli commands to create a target. + + :returns: Base gateway context + :rtype: Dict + """ gw_units = zaza.model.get_units('ceph-iscsi') primary_gw = gw_units[0] secondary_gw = gw_units[1] @@ -79,8 +89,16 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): def run_commands(self, unit_name, commands, ctxt): """Run commands on unit. - Apply context to commands until all variables have been replaced, then - run the command on the given unit. + Iterate over each command and apply the context to the command, then + run the command on the suppliend unit. + + :param unit_name: Name of unit to match + :type unit: str + :param commands: List of commands to run. + :type commands: List[str] + :param ctxt: Context to apply to each command. + :type ctxt: Dict + :raises: AssertionError """ for _cmd in commands: cmd = _cmd.format(**ctxt) @@ -89,7 +107,11 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): cmd)) def create_iscsi_target(self, ctxt): - """Create target on gateway.""" + """Create target on gateway. + + :param ctxt: Base gateway context + :type ctxt: Dict + """ generic_utils.assertActionRanOK(zaza.model.run_action_on_leader( 'ceph-iscsi', 'create-target', @@ -108,7 +130,11 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): })) def login_iscsi_target(self, ctxt): - """Mount iscsi target on client.""" + """Login to the iscsi target on client. + + :param ctxt: Base gateway context + :type ctxt: Dict + """ logging.info("Logging in to iscsi target") base_op_cmd = ('iscsiadm --mode node --targetname {gw_iqn} ' '--op=update ').format(**ctxt) @@ -121,14 +147,25 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): self.run_commands(ctxt['client_entity_id'], setup_cmds, ctxt) def logout_iscsi_targets(self, ctxt): - """Logout of iscsi target on client.""" + """Logout of iscsi target on client. + + :param ctxt: Base gateway context + :type ctxt: Dict + """ logging.info("Logging out of iscsi target") logout_cmds = [ 'iscsiadm --mode node --logoutall=all'] self.run_commands(ctxt['client_entity_id'], logout_cmds, ctxt) def check_client_device(self, ctxt, init_client=True): - """Wait for multipath device to appear on client and test access.""" + """Wait for multipath device to appear on client and test access. + + :param ctxt: Base gateway context + :type ctxt: Dict + :param init_client: Initialise client if this is the first time it has + been used. + :type init_client: bool + """ logging.info("Checking multipath device is present.") device_ctxt = { 'bdevice': '/dev/dm-0', @@ -203,19 +240,22 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): action_params={ 'name': self.EC_METADATA_POOL})) - def run_client_checks(self, ctxt): + def run_client_checks(self, test_ctxt): """Check access to mulipath device. Write a filesystem to device, mount it and write data. Then unmount and logout the iscsi target, finally reconnect and remount checking data is still present. + + :param test_ctxt: Test context. + :type test_ctxt: Dict """ - self.create_iscsi_target(ctxt) - self.login_iscsi_target(ctxt) - self.check_client_device(ctxt, init_client=True) - self.logout_iscsi_targets(ctxt) - self.login_iscsi_target(ctxt) - self.check_client_device(ctxt, init_client=False) + self.create_iscsi_target(test_ctxt) + self.login_iscsi_target(test_ctxt) + self.check_client_device(test_ctxt, init_client=True) + self.logout_iscsi_targets(test_ctxt) + self.login_iscsi_target(test_ctxt) + self.check_client_device(test_ctxt, init_client=False) def test_create_and_mount_volume(self): """Test creating a target and mounting it on a client.""" From 76e7666f9c7ebd5bc2b34b67959a1bbee6a7c703 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 23 Sep 2020 11:25:00 +0000 Subject: [PATCH 708/898] Fix typo --- zaza/openstack/charm_tests/ceph/iscsi/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index 17775b7..b1a456d 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -90,7 +90,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): """Run commands on unit. Iterate over each command and apply the context to the command, then - run the command on the suppliend unit. + run the command on the supplied unit. :param unit_name: Name of unit to match :type unit: str From ad1ea7667db4a1cd83f44239b5b1152c94c4b377 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 23 Sep 2020 17:04:19 +0000 Subject: [PATCH 709/898] handle stdout v Stdout --- zaza/openstack/charm_tests/ceph/iscsi/tests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index b1a456d..bb9e2b5 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -199,7 +199,13 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): run = await zaza.model.async_run_on_unit( ctxt['client_entity_id'], ls_bdevice_cmd.format(bdevice=device_ctxt['bdevice'])) - return device_ctxt['bdevice'] in run['Stdout'] + # Juju have moved to using 'stdout' rather than 'Stdout' so handle + # either. + try: + stdout = run['stdout'] + except KeyError: + stdout = run['Stdout'] + return device_ctxt['bdevice'] in stdout logging.info("Checking {} is present on {}".format( device_ctxt['bdevice'], From 6b93fb3acb8cd8df1e13bf90b93458b384b67f18 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 24 Sep 2020 10:33:11 +0200 Subject: [PATCH 710/898] Do not hide real exception on test failure --- zaza/openstack/charm_tests/heat/tests.py | 10 +++------- zaza/openstack/charm_tests/policyd/tests.py | 4 ++-- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/heat/tests.py b/zaza/openstack/charm_tests/heat/tests.py index d69217f..24062b7 100644 --- a/zaza/openstack/charm_tests/heat/tests.py +++ b/zaza/openstack/charm_tests/heat/tests.py @@ -74,13 +74,9 @@ class HeatBasicDeployment(test_utils.OpenStackBaseTest): def test_400_heat_resource_types_list(self): """Check default resource list behavior and confirm functionality.""" logging.info('Checking default heat resource list...') - try: - types = self.heat_client.resource_types.list() - self.assertIsInstance(types, list, "Resource type is not a list!") - self.assertGreater(len(types), 0, "Resource type list len is zero") - except Exception as e: - msg = 'Resource type list failed: {}'.format(e) - self.fail(msg) + types = self.heat_client.resource_types.list() + self.assertIsInstance(types, list, "Resource type is not a list!") + self.assertGreater(len(types), 0, "Resource type list len is zero") def test_410_heat_stack_create_delete(self): """Create stack, confirm nova compute resource, delete stack.""" diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index e69aaff..e498804 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -244,7 +244,7 @@ class BasePolicydSpecialization(PolicydTest, by the policy override in the `_rule` class variable. The method should pass cleanly without the override in place. - The test_003_test_overide_is_observed will then apply the override and then + The test_003_test_override_is_observed will then apply the override and then call get_client_and_attempt_operation() again, and this time it should detect the failure and raise the PolicydOperationFailedException() exception. This will be detected as the override working and thus the test @@ -399,7 +399,7 @@ class BasePolicydSpecialization(PolicydTest, return openstack_utils.get_keystone_session( openstack_utils.get_overcloud_auth(address=ip)) - def test_003_test_overide_is_observed(self): + def test_003_test_override_is_observed(self): """Test that the override is observed by the underlying service.""" if (openstack_utils.get_os_release() < openstack_utils.get_os_release('groovy_victoria')): From c09a9af3b2b9bcf4f926cc82389e62e26144fbbe Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 24 Sep 2020 10:54:14 +0200 Subject: [PATCH 711/898] Make linter happy --- zaza/openstack/charm_tests/policyd/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index e498804..eca316d 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -244,8 +244,8 @@ class BasePolicydSpecialization(PolicydTest, by the policy override in the `_rule` class variable. The method should pass cleanly without the override in place. - The test_003_test_override_is_observed will then apply the override and then - call get_client_and_attempt_operation() again, and this time it should + The test_003_test_override_is_observed will then apply the override and + then call get_client_and_attempt_operation() again, and this time it should detect the failure and raise the PolicydOperationFailedException() exception. This will be detected as the override working and thus the test will pass. From 7e0eaf919f685766e070fbf842263ca037b5dab4 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 24 Sep 2020 08:59:46 +0000 Subject: [PATCH 712/898] Stdout v stdout now handled by zaza --- zaza/openstack/charm_tests/ceph/iscsi/tests.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index bb9e2b5..a986833 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -199,13 +199,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): run = await zaza.model.async_run_on_unit( ctxt['client_entity_id'], ls_bdevice_cmd.format(bdevice=device_ctxt['bdevice'])) - # Juju have moved to using 'stdout' rather than 'Stdout' so handle - # either. - try: - stdout = run['stdout'] - except KeyError: - stdout = run['Stdout'] - return device_ctxt['bdevice'] in stdout + return device_ctxt['bdevice'] in run['stdout'] logging.info("Checking {} is present on {}".format( device_ctxt['bdevice'], From 70e758be05ebb4ef5bb711d775d5e2fa67d2b9ba Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 24 Sep 2020 15:18:49 +0200 Subject: [PATCH 713/898] Break out required environment variables Instead of printing out the list of all required environment variables when one is missing, only print out the missing variables. --- unit_tests/charm_tests/test_tempest.py | 66 +++++++++++++++++++++ zaza/openstack/charm_tests/tempest/setup.py | 9 ++- 2 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 unit_tests/charm_tests/test_tempest.py diff --git a/unit_tests/charm_tests/test_tempest.py b/unit_tests/charm_tests/test_tempest.py new file mode 100644 index 0000000..9c4fc9a --- /dev/null +++ b/unit_tests/charm_tests/test_tempest.py @@ -0,0 +1,66 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +import unittest + +import zaza.openstack.charm_tests.tempest.setup as tempest_setup + + +class TestTempestSetup(unittest.TestCase): + """Test class to encapsulate testing Mysql test utils.""" + + def setUp(self): + super(TestTempestSetup, self).setUp() + + def test_add_environment_var_config_with_missing_variable(self): + ctxt = {} + with self.assertRaises(Exception) as context: + tempest_setup.add_environment_var_config(ctxt, ['swift']) + self.assertEqual( + ('Environment variables [TEST_SWIFT_IP] must all be ' + 'set to run this test'), + str(context.exception)) + + @mock.patch.object(tempest_setup.deployment_env, 'get_deployment_context') + def test_add_environment_var_config_with_all_variables( + self, + get_deployment_context): + ctxt = {} + get_deployment_context.return_value = { + 'TEST_GATEWAY': 'test', + 'TEST_CIDR_EXT': 'test', + 'TEST_FIP_RANGE': 'test', + 'TEST_NAMESERVER': 'test', + 'TEST_CIDR_PRIV': 'test', + } + tempest_setup.add_environment_var_config(ctxt, ['neutron']) + self.assertEqual(ctxt['test_gateway'], 'test') + + @mock.patch.object(tempest_setup.deployment_env, 'get_deployment_context') + def test_add_environment_var_config_with_some_variables( + self, + get_deployment_context): + ctxt = {} + get_deployment_context.return_value = { + 'TEST_GATEWAY': 'test', + 'TEST_NAMESERVER': 'test', + 'TEST_CIDR_PRIV': 'test', + } + with self.assertRaises(Exception) as context: + tempest_setup.add_environment_var_config(ctxt, ['neutron']) + self.assertEqual( + ('Environment variables [TEST_CIDR_EXT, TEST_FIP_RANGE] must ' + 'all be set to run this test'), + str(context.exception)) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index e7a1c41..98ea327 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -190,6 +190,7 @@ def add_environment_var_config(ctxt, services): :rtype: None """ deploy_env = deployment_env.get_deployment_context() + missing_vars = [] for svc, env_vars in SETUP_ENV_VARS.items(): if svc in services: for var in env_vars: @@ -197,9 +198,11 @@ def add_environment_var_config(ctxt, services): if value: ctxt[var.lower()] = value else: - raise ValueError( - ("Environment variables {} must all be set to run this" - " test").format(', '.join(env_vars))) + missing_vars.append(var) + if missing_vars: + raise ValueError( + ("Environment variables [{}] must all be set to run this" + " test").format(', '.join(missing_vars))) def add_auth_config(ctxt): From 18e8df8eac86bc02156de632efe2e94481126bc3 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 25 Sep 2020 08:20:04 +0200 Subject: [PATCH 714/898] Do not attempt to restore configuration when equal The fix for the config_change helpers short cut on config restore in c5a3f832d014b28dd997356416af3ca082434d4c introduced a regression for consumers that use the helper to set but not restore config. Re-introduce the shortcut when the caller passes in equal settings for default and alternate configuration. Fixes #427 --- unit_tests/charm_tests/test_utils.py | 18 ++++++++++++++++++ zaza/openstack/charm_tests/test_utils.py | 4 ++++ 2 files changed, 22 insertions(+) diff --git a/unit_tests/charm_tests/test_utils.py b/unit_tests/charm_tests/test_utils.py index 7ca3ba5..ffb7f83 100644 --- a/unit_tests/charm_tests/test_utils.py +++ b/unit_tests/charm_tests/test_utils.py @@ -106,6 +106,24 @@ class TestBaseCharmTest(ut_utils.BaseTestCase): mock.call(), mock.call(), ]) + # confirm operation where both default and alternate config passed in + # are the same. This is used to set config and not change it back. + self.set_application_config.reset_mock() + self.wait_for_agent_status.reset_mock() + self.wait_for_application_states.reset_mock() + self.reset_application_config.reset_mock() + with self.target.config_change( + alterna_config, alterna_config, application_name='anApp'): + self.set_application_config.assert_called_once_with( + 'anApp', alterna_config, model_name='aModel') + # we want to assert these not to be called after yield + self.set_application_config.reset_mock() + self.wait_for_agent_status.reset_mock() + self.wait_for_application_states.reset_mock() + self.assertFalse(self.set_application_config.called) + self.assertFalse(self.reset_application_config.called) + self.assertFalse(self.wait_for_agent_status.called) + self.assertFalse(self.wait_for_application_states.called) class TestOpenStackBaseTest(ut_utils.BaseTestCase): diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 055ea66..a3614f0 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -259,6 +259,10 @@ class BaseCharmTest(unittest.TestCase): model.reset_application_config(application_name, alternate_config.keys(), model_name=self.model_name) + elif default_config == alternate_config: + logging.debug('default_config == alternate_config, not attempting ' + ' to restore configuration') + return else: logging.debug('Restoring charm setting to {}' .format(default_config)) From 522d3e53b8d87c05d64fbbb8722c861cdd52638d Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 25 Sep 2020 10:39:13 +0200 Subject: [PATCH 715/898] Only configure private Neutron CIDR if it is configured --- zaza/openstack/charm_tests/tempest/setup.py | 7 +++++-- zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 | 2 ++ zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 98ea327..5c85d9c 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -26,10 +26,12 @@ import zaza.openstack.charm_tests.glance.setup as glance_setup SETUP_ENV_VARS = { 'neutron': ['TEST_GATEWAY', 'TEST_CIDR_EXT', 'TEST_FIP_RANGE', - 'TEST_NAMESERVER', 'TEST_CIDR_PRIV'], + 'TEST_NAME_SERVER', 'TEST_CIDR_PRIV'], 'swift': ['TEST_SWIFT_IP'], } +IGNORABLE_VARS = ['TEST_CIDR_PRIV'] + TEMPEST_FLAVOR_NAME = 'm1.tempest' TEMPEST_ALT_FLAVOR_NAME = 'm2.tempest' TEMPEST_SVC_LIST = ['ceilometer', 'cinder', 'glance', 'heat', 'horizon', @@ -198,7 +200,8 @@ def add_environment_var_config(ctxt, services): if value: ctxt[var.lower()] = value else: - missing_vars.append(var) + if var not in IGNORABLE_VARS: + missing_vars.append(var) if missing_vars: raise ValueError( ("Environment variables [{}] must all be set to run this" diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 index 83a05ca..8c9b6c6 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 @@ -52,7 +52,9 @@ http_image = http://{{ test_swift_ip }}:80/swift/v1/images/cirros-0.3.4-x86_64-u {% if 'neutron' in enabled_services %} [network] +{% if test_cidr_priv %} project_network_cidr = {{ test_cidr_priv }} +{% endif %} public_network_id = {{ ext_net }} dns_servers = {{ test_nameserver }} project_networks_reachable = false diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 index 0ed401a..fc92dd2 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 @@ -54,7 +54,9 @@ http_image = http://{{ test_swift_ip }}:80/swift/v1/images/cirros-0.3.4-x86_64-u {% if 'neutron' in enabled_services %} [network] +{% if test_cidr_priv %} project_network_cidr = {{ test_cidr_priv }} +{% endif %} public_network_id = {{ ext_net }} dns_servers = {{ test_nameserver }} project_networks_reachable = false From 09789767f301e1b7133d76b441e4d15ca7c5251e Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 25 Sep 2020 11:19:32 +0200 Subject: [PATCH 716/898] Enhance common launch instance helper Move retry logic to the launch single guest method. Add optional arguments that is passed on to the downstream launch guest helper. --- zaza/openstack/charm_tests/test_utils.py | 35 ++++++++++++++---------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index a3614f0..1f24aa9 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -526,7 +526,8 @@ class OpenStackBaseTest(BaseCharmTest): # Test did not define self.RESOURCE_PREFIX, ignore. pass - def launch_guest(self, guest_name, userdata=None): + def launch_guest(self, guest_name, userdata=None, use_boot_volume=False, + instance_key=None): """Launch two guests to use in tests. Note that it is up to the caller to have set the RESOURCE_PREFIX class @@ -539,9 +540,14 @@ class OpenStackBaseTest(BaseCharmTest): :type guest_name: str :param userdata: Userdata to attach to instance :type userdata: Optional[str] + :param use_boot_volume: Whether to boot guest from a shared volume. + :type use_boot_volume: boolean + :param instance_key: Key to collect associated config data with. + :type instance_key: Optional[str] :returns: Nova instance objects :rtype: Server """ + instance_key = instance_key or glance_setup.LTS_IMAGE_NAME instance_name = '{}-{}'.format(self.RESOURCE_PREFIX, guest_name) instance = self.retrieve_guest(instance_name) @@ -554,10 +560,16 @@ class OpenStackBaseTest(BaseCharmTest): instance.id, msg="server") - return configure_guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name=instance_name, - userdata=userdata) + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)): + with attempt: + return configure_guest.launch_instance( + instance_key, + vm_name=instance_name, + use_boot_volume=use_boot_volume, + userdata=userdata) def launch_guests(self, userdata=None): """Launch two guests to use in tests. @@ -572,15 +584,10 @@ class OpenStackBaseTest(BaseCharmTest): """ launched_instances = [] for guest_number in range(1, 2+1): - for attempt in tenacity.Retrying( - stop=tenacity.stop_after_attempt(3), - wait=tenacity.wait_exponential( - multiplier=1, min=2, max=10)): - with attempt: - launched_instances.append( - self.launch_guest( - guest_name='ins-{}'.format(guest_number), - userdata=userdata)) + launched_instances.append( + self.launch_guest( + guest_name='ins-{}'.format(guest_number), + userdata=userdata)) return launched_instances def retrieve_guest(self, guest_name): From 786b40fc53bf3a13d2df1faafd2c15c21b17aca5 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 25 Sep 2020 11:21:15 +0200 Subject: [PATCH 717/898] nova: Use common launch guest helper The common helper adds automatic retry handling and resource cleanup. --- zaza/openstack/charm_tests/nova/tests.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index c4dd5b4..16c9337 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -38,32 +38,35 @@ class BaseGuestCreateTest(unittest.TestCase): zaza.openstack.configure.guest.launch_instance(instance_key) -class CirrosGuestCreateTest(BaseGuestCreateTest): +class CirrosGuestCreateTest(test_utils.OpenStackBaseTest): """Tests to launch a cirros image.""" def test_launch_small_instance(self): """Launch a cirros instance and test connectivity.""" - zaza.openstack.configure.guest.launch_instance( - glance_setup.CIRROS_IMAGE_NAME) + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'cirros', instance_key=glance_setup.CIRROS_IMAGE_NAME) -class LTSGuestCreateTest(BaseGuestCreateTest): +class LTSGuestCreateTest(test_utils.OpenStackBaseTest): """Tests to launch a LTS image.""" def test_launch_small_instance(self): """Launch a Bionic instance and test connectivity.""" - zaza.openstack.configure.guest.launch_instance( - glance_setup.LTS_IMAGE_NAME) + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME) -class LTSGuestCreateVolumeBackedTest(BaseGuestCreateTest): +class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): """Tests to launch a LTS image.""" def test_launch_small_instance(self): """Launch a Bionic instance and test connectivity.""" - zaza.openstack.configure.guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - use_boot_volume=True) + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'volume-backed-ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME, + use_boot_volmue=True) class NovaCompute(test_utils.OpenStackBaseTest): From 7bf624b007f4b65b4ce0709abc967906a86ee9ed Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Fri, 25 Sep 2020 12:49:06 +0200 Subject: [PATCH 718/898] Update tempest config files to use newly renamed test_name_server variable --- unit_tests/charm_tests/test_tempest.py | 4 ++-- zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 | 2 +- zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/unit_tests/charm_tests/test_tempest.py b/unit_tests/charm_tests/test_tempest.py index 9c4fc9a..3c4c161 100644 --- a/unit_tests/charm_tests/test_tempest.py +++ b/unit_tests/charm_tests/test_tempest.py @@ -42,7 +42,7 @@ class TestTempestSetup(unittest.TestCase): 'TEST_GATEWAY': 'test', 'TEST_CIDR_EXT': 'test', 'TEST_FIP_RANGE': 'test', - 'TEST_NAMESERVER': 'test', + 'TEST_NAME_SERVER': 'test', 'TEST_CIDR_PRIV': 'test', } tempest_setup.add_environment_var_config(ctxt, ['neutron']) @@ -55,7 +55,7 @@ class TestTempestSetup(unittest.TestCase): ctxt = {} get_deployment_context.return_value = { 'TEST_GATEWAY': 'test', - 'TEST_NAMESERVER': 'test', + 'TEST_NAME_SERVER': 'test', 'TEST_CIDR_PRIV': 'test', } with self.assertRaises(Exception) as context: diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 index 8c9b6c6..d4b8810 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v2.j2 @@ -56,7 +56,7 @@ http_image = http://{{ test_swift_ip }}:80/swift/v1/images/cirros-0.3.4-x86_64-u project_network_cidr = {{ test_cidr_priv }} {% endif %} public_network_id = {{ ext_net }} -dns_servers = {{ test_nameserver }} +dns_servers = {{ test_name_server }} project_networks_reachable = false [network-feature-enabled] diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 index fc92dd2..b441f2d 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 @@ -58,7 +58,7 @@ http_image = http://{{ test_swift_ip }}:80/swift/v1/images/cirros-0.3.4-x86_64-u project_network_cidr = {{ test_cidr_priv }} {% endif %} public_network_id = {{ ext_net }} -dns_servers = {{ test_nameserver }} +dns_servers = {{ test_name_server }} project_networks_reachable = false floating_network_name = {{ ext_net }} From 49268243ba11b8cb8c1510ffb2eea28943f5cb3b Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 22 Sep 2020 16:18:09 +0200 Subject: [PATCH 719/898] Improve logging --- zaza/openstack/charm_tests/cinder_backup/tests.py | 8 ++++---- zaza/openstack/utilities/openstack.py | 13 ++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 7d3442e..1cdd887 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -63,7 +63,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): self.cinder_client.volumes, vol_new.id, expected_status="available", - msg="Volume status wait") + msg="Extended volume") def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self): """Create, backup, delete, restore a ceph-backed cinder volume. @@ -99,7 +99,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): wait_iteration_max_time=180, stop_after_attempt=30, expected_status='available', - msg='Volume status wait') + msg='ceph-backed cinder volume') # Backup the volume vol_backup = self.cinder_client.backups.create( @@ -111,7 +111,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): wait_iteration_max_time=180, stop_after_attempt=30, expected_status='available', - msg='Volume status wait') + msg='Backup volume') # Delete the volume openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) # Restore the volume @@ -122,7 +122,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): wait_iteration_max_time=180, stop_after_attempt=15, expected_status='available', - msg='Backup status wait') + msg='Restored backup volume') # Delete the backup openstack_utils.delete_volume_backup( self.cinder_client, diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 35972c6..35e188d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1896,14 +1896,13 @@ def _resource_reaches_status(resource, resource_id, :param expected_status: status to expect resource to reach :type expected_status: str :param msg: text to identify purpose in logging - :type msy: str + :type msg: str :raises: AssertionError """ resource_status = resource.get(resource_id).status - logging.info(resource_status) - assert resource_status == expected_status, ( - "Resource in {} state, waiting for {}" .format(resource_status, - expected_status,)) + logging.info("{}: resource {} in {} state, waiting for {}".format( + msg, resource_id, resource_status, expected_status)) + assert resource_status == expected_status def resource_reaches_status(resource, @@ -1965,8 +1964,8 @@ def _resource_removed(resource, resource_id, msg="resource"): :raises: AssertionError """ matching = [r for r in resource.list() if r.id == resource_id] - logging.debug("Resource {} still present".format(resource_id)) - assert len(matching) == 0, "Resource {} still present".format(resource_id) + logging.debug("{}: resource {} still present".format(msg, resource_id)) + assert len(matching) == 0 def resource_removed(resource, From e70be304e5d1cc45ab07b8251dfad393260e486d Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Fri, 25 Sep 2020 18:22:09 +1200 Subject: [PATCH 720/898] Add tests for OVN charm NRPE checks The ovn-* charms have added NRPE service checks, this ensures that the service check files have been added to the units as expected. --- zaza/openstack/charm_tests/ovn/tests.py | 63 ++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index d12911e..30abddd 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -15,11 +15,14 @@ """Encapsulate OVN testing.""" import logging -import tenacity import juju +import tenacity + import zaza + +import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -36,10 +39,19 @@ class BaseCharmOperationTest(test_utils.BaseCharmTest): """Run class setup for OVN charm operation tests.""" super(BaseCharmOperationTest, cls).setUpClass() cls.services = ['NotImplemented'] # This must be overridden + cls.nrpe_checks = ['NotImplemented'] # This must be overridden cls.current_release = openstack_utils.get_os_release( openstack_utils.get_current_os_release_pair( cls.release_application or cls.application_name)) + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + # sleep for 2mins to allow 1min cron job to run... + wait=tenacity.wait_fixed(120), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_commands_on_units(self, cmds, units): + return generic_utils.check_commands_on_units(cmds, units) + def test_pause_resume(self): """Run pause and resume tests. @@ -50,6 +62,19 @@ class BaseCharmOperationTest(test_utils.BaseCharmTest): logging.info('Testing pause resume (services="{}")' .format(self.services)) + def test_nrpe_configured(self): + """Confirm that the NRPE service check files are created.""" + units = zaza.model.get_units(self.application_name) + cmds = [] + for check_name in self.nrpe_checks: + cmds.append( + 'egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_{}.cfg'.format(check_name) + ) + ret = self._retry_check_commands_on_units(cmds, units) + logging.info(ret) + self.assertIsNone(ret, msg=ret) + class CentralCharmOperationTest(BaseCharmOperationTest): """OVN Central Charm operation tests.""" @@ -62,6 +87,22 @@ class CentralCharmOperationTest(BaseCharmOperationTest): 'ovn-northd', 'ovsdb-server', ] + source = zaza.model.get_application_config( + cls.application_name)['source']['value'] + logging.info(source) + if 'train' in source: + cls.nrpe_checks = [ + 'ovn-northd', + 'ovn-nb-ovsdb', + 'ovn-sb-ovsdb', + ] + else: + # Ussuri or later (distro or cloudarchive) + cls.nrpe_checks = [ + 'ovn-northd', + 'ovn-ovsdb-server-sb', + 'ovn-ovsdb-server-nb', + ] class ChassisCharmOperationTest(BaseCharmOperationTest): @@ -76,6 +117,26 @@ class ChassisCharmOperationTest(BaseCharmOperationTest): cls.services = [ 'ovn-controller', ] + if cls.application_name == 'ovn-chassis': + principal_app_name = 'magpie' + else: + principal_app_name = cls.application_name + source = zaza.model.get_application_config( + principal_app_name)['source']['value'] + logging.info(source) + if 'train' in source: + cls.nrpe_checks = [ + 'ovn-host', + 'ovs-vswitchd', + 'ovsdb-server', + ] + else: + # Ussuri or later (distro or cloudarchive) + cls.nrpe_checks = [ + 'ovn-controller', + 'ovsdb-server', + 'ovs-vswitchd', + ] class OVSOVNMigrationTest(test_utils.BaseCharmTest): From a71a0f38ff4a8491c1c8fcd94f52eec2cedfa230 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 25 Sep 2020 14:57:18 +0200 Subject: [PATCH 721/898] Add some tenacity around cinder backup creation --- .../charm_tests/cinder_backup/tests.py | 63 +++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 1cdd887..1acbf9e 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -89,29 +89,43 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): self.assertEqual(pool_name, expected_pool) - # Create ceph-backed cinder volume - cinder_vol = self.cinder_client.volumes.create( - name='{}-410-vol'.format(self.RESOURCE_PREFIX), - size=1) - openstack_utils.resource_reaches_status( - self.cinder_client.volumes, - cinder_vol.id, - wait_iteration_max_time=180, - stop_after_attempt=30, - expected_status='available', - msg='ceph-backed cinder volume') + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3)): + with attempt: + # Create ceph-backed cinder volume + cinder_vol_name = '{}-410-vol-{}'.format( + self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) + cinder_vol = self.cinder_client.volumes.create( + name=cinder_vol_name, size=1) + openstack_utils.resource_reaches_status( + self.cinder_client.volumes, + cinder_vol.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='ceph-backed cinder volume') + + # Back up the volume + # NOTE(lourot): sometimes, especially on Mitaka, the backup + # remains stuck forever in 'creating' state and the volume in + # 'backing-up' state. See lp:1877076 + # Attempting to create another volume and another backup + # usually then succeeds. Release notes and bug trackers show + # that many things have been fixed and are still left to be + # fixed in this area. + # When the backup creation succeeds, it usually does within + # 12 minutes. + vol_backup_name = cinder_vol_name + '-backup' + vol_backup = self.cinder_client.backups.create( + cinder_vol.id, vol_backup_name) + openstack_utils.resource_reaches_status( + self.cinder_client.backups, + vol_backup.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='Backup volume') - # Backup the volume - vol_backup = self.cinder_client.backups.create( - cinder_vol.id, - name='{}-410-backup-vol'.format(self.RESOURCE_PREFIX)) - openstack_utils.resource_reaches_status( - self.cinder_client.backups, - vol_backup.id, - wait_iteration_max_time=180, - stop_after_attempt=30, - expected_status='available', - msg='Backup volume') # Delete the volume openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) # Restore the volume @@ -143,12 +157,11 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): obj_count_samples.append(obj_count) pool_size_samples.append(kb_used) - name = '{}-410-vol'.format(self.RESOURCE_PREFIX) vols = self.cinder_client.volumes.list() try: - cinder_vols = [v for v in vols if v.name == name] + cinder_vols = [v for v in vols if v.name == cinder_vol_name] except AttributeError: - cinder_vols = [v for v in vols if v.display_name == name] + cinder_vols = [v for v in vols if v.display_name == cinder_vol_name] if not cinder_vols: # NOTE(hopem): it appears that at some point cinder-backup stopped # restoring volume metadata properly so revert to default name if From f9d28f16b421699a478554f74b6b3a5895864610 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 28 Sep 2020 11:03:33 +0200 Subject: [PATCH 722/898] Make linter happy --- zaza/openstack/charm_tests/cinder_backup/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 1acbf9e..2281cc1 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -161,7 +161,8 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): try: cinder_vols = [v for v in vols if v.name == cinder_vol_name] except AttributeError: - cinder_vols = [v for v in vols if v.display_name == cinder_vol_name] + cinder_vols = [v for v in vols if + v.display_name == cinder_vol_name] if not cinder_vols: # NOTE(hopem): it appears that at some point cinder-backup stopped # restoring volume metadata properly so revert to default name if From 7a67d98b4585756398e47f745a445e91677062fb Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Fri, 2 Oct 2020 11:00:51 +1300 Subject: [PATCH 723/898] Only log OVN NRPE failure when it fails --- zaza/openstack/charm_tests/ovn/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 30abddd..c5eae5e 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -72,7 +72,8 @@ class BaseCharmOperationTest(test_utils.BaseCharmTest): 'check_{}.cfg'.format(check_name) ) ret = self._retry_check_commands_on_units(cmds, units) - logging.info(ret) + if ret: + logging.info(ret) self.assertIsNone(ret, msg=ret) From efcb7a7e0653a1b5a475064ced25608f1d94ee2f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 2 Oct 2020 10:50:10 +0200 Subject: [PATCH 724/898] Fix typo --- zaza/openstack/charm_tests/nova/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 16c9337..79e0e70 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -66,7 +66,7 @@ class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): self.RESOURCE_PREFIX = 'zaza-nova' self.launch_guest( 'volume-backed-ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME, - use_boot_volmue=True) + use_boot_volume=True) class NovaCompute(test_utils.OpenStackBaseTest): From 7c2f5cdf2493d39505b28d1b3243c6ab76b59e01 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Sat, 3 Oct 2020 00:23:05 +0000 Subject: [PATCH 725/898] Add Ironic tests --- requirements.txt | 1 + .../test_zaza_utilities_openstack.py | 14 +- zaza/openstack/charm_tests/glance/setup.py | 49 ++++- zaza/openstack/charm_tests/ironic/__init__.py | 15 ++ zaza/openstack/charm_tests/ironic/setup.py | 167 ++++++++++++++++++ zaza/openstack/charm_tests/ironic/tests.py | 83 +++++++++ zaza/openstack/utilities/openstack.py | 13 +- 7 files changed, 329 insertions(+), 13 deletions(-) create mode 100644 zaza/openstack/charm_tests/ironic/__init__.py create mode 100644 zaza/openstack/charm_tests/ironic/setup.py create mode 100644 zaza/openstack/charm_tests/ironic/tests.py diff --git a/requirements.txt b/requirements.txt index 8b15edc..cec0286 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ python-ceilometerclient python-cinderclient python-glanceclient python-heatclient +python-ironicclient python-keystoneclient python-manilaclient python-neutronclient diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 02bd578..5ad0b9b 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -504,7 +504,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): glance_mock.images.upload.assert_called_once_with( '9d1125af', f(), - ) + backend=None) self.resource_reaches_status.assert_called_once_with( glance_mock.images, '9d1125af', @@ -529,7 +529,11 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.upload_image_to_glance.assert_called_once_with( glance_mock, 'wibbly/c.img', - 'bob') + 'bob', + backend=None, + disk_format='qcow2', + visibility='public', + container_format='bare') def test_create_image_pass_directory(self): glance_mock = mock.MagicMock() @@ -549,7 +553,11 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.upload_image_to_glance.assert_called_once_with( glance_mock, 'tests/c.img', - 'bob') + 'bob', + backend=None, + disk_format='qcow2', + visibility='public', + container_format='bare') self.gettempdir.assert_not_called() def test_create_ssh_key(self): diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index ab0a3b3..6686d70 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -32,8 +32,37 @@ def basic_setup(): """ +def _get_default_glance_client(): + """Create default Glance client using overcloud credentials.""" + keystone_session = openstack_utils.get_overcloud_keystone_session() + glance_client = openstack_utils.get_glance_session_client(keystone_session) + return glance_client + + +def get_stores_info(glance_client=None): + """Retrieve glance backing store info. + + :param glance_client: Authenticated glanceclient + :type glance_client: glanceclient.Client + """ + glance_client = glance_client or _get_default_glance_client() + stores = glance_client.images.get_stores_info().get("stores", []) + return stores + + +def get_store_ids(glance_client=None): + """Retrieve glance backing store ids. + + :param glance_client: Authenticated glanceclient + :type glance_client: glanceclient.Client + """ + stores = get_stores_info(glance_client) + return [store["id"] for store in stores] + + def add_image(image_url, glance_client=None, image_name=None, tags=[], - properties=None): + properties=None, backend=None, disk_format='qcow2', + visibility='public', container_format='bare'): """Retrieve image from ``image_url`` and add it to glance. :param image_url: Retrievable URL with image data @@ -47,10 +76,14 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[], :param properties: Properties to add to image :type properties: dict """ - if not glance_client: - keystone_session = openstack_utils.get_overcloud_keystone_session() - glance_client = openstack_utils.get_glance_session_client( - keystone_session) + glance_client = glance_client or _get_default_glance_client() + if backend is not None: + stores = get_store_ids(glance_client) + if backend not in stores: + raise ValueError("Invalid backend: %(backend)s " + "(available: %(available)s)" % { + "backend": backend, + "available": ", ".join(stores)}) if image_name: image = openstack_utils.get_images_by_name( glance_client, image_name) @@ -65,7 +98,11 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[], image_url, image_name, tags=tags, - properties=properties) + properties=properties, + backend=backend, + disk_format=disk_format, + visibility=visibility, + container_format=container_format) def add_cirros_image(glance_client=None, image_name=None): diff --git a/zaza/openstack/charm_tests/ironic/__init__.py b/zaza/openstack/charm_tests/ironic/__init__.py new file mode 100644 index 0000000..aedad05 --- /dev/null +++ b/zaza/openstack/charm_tests/ironic/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ironic.""" diff --git a/zaza/openstack/charm_tests/ironic/setup.py b/zaza/openstack/charm_tests/ironic/setup.py new file mode 100644 index 0000000..b03c8cb --- /dev/null +++ b/zaza/openstack/charm_tests/ironic/setup.py @@ -0,0 +1,167 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for configuring ironic.""" + +import copy +import os + +import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.utilities.openstack as openstack_utils +from zaza.openstack.utilities import ( + cli as cli_utils, +) +import zaza.model as zaza_model + + +FLAVORS = { + 'bm1.small': { + 'flavorid': 2, + 'ram': 2048, + 'disk': 20, + 'vcpus': 1, + 'properties': { + "resources:CUSTOM_BAREMETAL1_SMALL": 1, + }, + }, + 'bm1.medium': { + 'flavorid': 3, + 'ram': 4096, + 'disk': 40, + 'vcpus': 2, + 'properties': { + "resources:CUSTOM_BAREMETAL1_MEDIUM": 1, + }, + }, + 'bm1.large': { + 'flavorid': 4, + 'ram': 8192, + 'disk': 40, + 'vcpus': 4, + 'properties': { + "resources:CUSTOM_BAREMETAL1_LARGE": 1, + }, + }, + 'bm1.tempest': { + 'flavorid': 6, + 'ram': 256, + 'disk': 1, + 'vcpus': 1, + 'properties': { + "resources:CUSTOM_BAREMETAL1_TEMPEST": 1, + }, + }, + 'bm2.tempest': { + 'flavorid': 7, + 'ram': 512, + 'disk': 1, + 'vcpus': 1, + 'properties': { + "resources:CUSTOM_BAREMETAL2_TEMPEST": 1, + }, + }, +} + + +def add_ironic_deployment_image(initrd_url=None, kernel_url=None): + """Add Ironic deploy images to glance. + + :param initrd_url: URL where the ari image resides + :type initrd_url: str + :param kernel_url: URL where the aki image resides + :type kernel_url: str + """ + base_name = 'ironic-deploy' + initrd_name = "{}-initrd".format(base_name) + vmlinuz_name = "{}-vmlinuz".format(base_name) + if not initrd_url: + initrd_url = os.environ.get('TEST_IRONIC_DEPLOY_INITRD', None) + if not kernel_url: + kernel_url = os.environ.get('TEST_IRONIC_DEPLOY_VMLINUZ', None) + if not all([initrd_url, kernel_url]): + raise ValueError("Missing required deployment image URLs") + glance_setup.add_image( + initrd_url, + image_name=initrd_name, + backend="swift", + disk_format="ari", + container_format="ari") + glance_setup.add_image( + kernel_url, + image_name=vmlinuz_name, + backend="swift", + disk_format="aki", + container_format="aki") + + +def add_ironic_os_image(image_url=None): + """Upload the operating system images built for bare metal deployments. + + :param image_url: URL where the image resides + :type image_url: str + """ + image_url = image_url or os.environ.get( + 'TEST_IRONIC_RAW_BM_IMAGE', None) + image_name = "baremetal-ubuntu-image" + if image_url is None: + raise ValueError("Missing image_url") + + glance_setup.add_image( + image_url, + image_name=image_name, + backend="swift", + disk_format="raw", + container_format="bare") + + +def set_temp_url_secret(): + """Run the set-temp-url-secret on the ironic-conductor leader. + + This is needed if direct boot method is enabled. + """ + zaza_model.run_action_on_leader( + 'ironic-conductor', + 'set-temp-url-secret', + action_params={}) + + +def create_bm_flavors(nova_client=None): + """Create baremetal flavors. + + :param nova_client: Authenticated nova client + :type nova_client: novaclient.v2.client.Client + """ + if not nova_client: + keystone_session = openstack_utils.get_overcloud_keystone_session() + nova_client = openstack_utils.get_nova_session_client( + keystone_session) + cli_utils.setup_logging() + names = [flavor.name for flavor in nova_client.flavors.list()] + # Disable scheduling based on standard flavor properties + default_properties = { + "resources:VCPU": 0, + "resources:MEMORY_MB": 0, + "resources:DISK_GB": 0, + } + for flavor in FLAVORS.keys(): + if flavor not in names: + properties = copy.deepcopy(default_properties) + properties.update(FLAVORS[flavor]["properties"]) + bm_flavor = nova_client.flavors.create( + name=flavor, + ram=FLAVORS[flavor]['ram'], + vcpus=FLAVORS[flavor]['vcpus'], + disk=FLAVORS[flavor]['disk'], + flavorid=FLAVORS[flavor]['flavorid']) + bm_flavor.set_keys(properties) diff --git a/zaza/openstack/charm_tests/ironic/tests.py b/zaza/openstack/charm_tests/ironic/tests.py new file mode 100644 index 0000000..9cfb85a --- /dev/null +++ b/zaza/openstack/charm_tests/ironic/tests.py @@ -0,0 +1,83 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate ironic testing.""" + +import logging + +import ironicclient.client as ironic_client +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +def _get_ironic_client(ironic_api_version="1.58"): + keystone_session = openstack_utils.get_overcloud_keystone_session() + ironic = ironic_client.Client(1, session=keystone_session, + os_ironic_api_version=ironic_api_version) + return ironic + + +class IronicTest(test_utils.OpenStackBaseTest): + """Run Ironic specific tests.""" + + _SERVICES = ['ironic-api'] + + def test_110_catalog_endpoints(self): + """Verify that the endpoints are present in the catalog.""" + overcloud_auth = openstack_utils.get_overcloud_auth() + keystone_client = openstack_utils.get_keystone_client( + overcloud_auth) + actual_endpoints = keystone_client.service_catalog.get_endpoints() + actual_interfaces = [endpoint['interface'] for endpoint in + actual_endpoints["baremetal"]] + for expected_interface in ('internal', 'admin', 'public'): + assert(expected_interface in actual_interfaces) + + def test_400_api_connection(self): + """Simple api calls to check service is up and responding.""" + ironic = _get_ironic_client() + + logging.info('listing conductors') + conductors = ironic.conductor.list() + assert(len(conductors) > 0) + + # By default, only IPMI HW type is enabled. iDrac and Redfish + # can optionally be enabled + drivers = ironic.driver.list() + driver_names = [drv.name for drv in drivers] + + expected = ['intel-ipmi', 'ipmi'] + for exp in expected: + assert(exp in driver_names) + assert(len(driver_names) == 2) + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change. + + Change debug mode and assert that change propagates to the correct + file and that services are restarted as a result + """ + self.restart_on_changed_debug_oslo_config_file( + '/etc/ironic/ironic.conf', self._SERVICES) + + def test_910_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + logging.info('Skipping pause resume test LP: #1886202...') + return + with self.pause_resume(self._SERVICES): + logging.info("Testing pause resume") diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 35972c6..b62885e 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2060,7 +2060,8 @@ def delete_volume_backup(cinder, vol_backup_id): def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', - visibility='public', container_format='bare'): + visibility='public', container_format='bare', + backend=None): """Upload the given image to glance and apply the given label. :param glance: Authenticated glanceclient @@ -2086,7 +2087,7 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', disk_format=disk_format, visibility=visibility, container_format=container_format) - glance.images.upload(image.id, open(local_path, 'rb')) + glance.images.upload(image.id, open(local_path, 'rb'), backend=backend) resource_reaches_status( glance.images, @@ -2098,7 +2099,8 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], - properties=None): + properties=None, backend=None, disk_format='qcow2', + visibility='public', container_format='bare'): """Download the image and upload it to glance. Download an image from image_url and upload it to glance labelling @@ -2132,7 +2134,10 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], if not os.path.exists(local_path): download_image(image_url, local_path) - image = upload_image_to_glance(glance, local_path, image_name) + image = upload_image_to_glance( + glance, local_path, image_name, backend=backend, + disk_format=disk_format, visibility=visibility, + container_format=container_format) for tag in tags: result = glance.image_tags.update(image.id, tag) logging.debug( From 181bb98d2060f1edbacb8beb0555918f3e58c5e1 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 9 Oct 2020 09:39:00 +0200 Subject: [PATCH 726/898] Clean up newly created instances if faulty (#439) --- zaza/openstack/charm_tests/test_utils.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 1f24aa9..9157bd3 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -550,21 +550,23 @@ class OpenStackBaseTest(BaseCharmTest): instance_key = instance_key or glance_setup.LTS_IMAGE_NAME instance_name = '{}-{}'.format(self.RESOURCE_PREFIX, guest_name) - instance = self.retrieve_guest(instance_name) - if instance: - logging.info('Removing already existing instance ({}) with ' - 'requested name ({})' - .format(instance.id, instance_name)) - openstack_utils.delete_resource( - self.nova_client.servers, - instance.id, - msg="server") - for attempt in tenacity.Retrying( stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential( multiplier=1, min=2, max=10)): with attempt: + old_instance_with_same_name = self.retrieve_guest( + instance_name) + if old_instance_with_same_name: + logging.info( + 'Removing already existing instance ({}) with ' + 'requested name ({})' + .format(old_instance_with_same_name.id, instance_name)) + openstack_utils.delete_resource( + self.nova_client.servers, + old_instance_with_same_name.id, + msg="server") + return configure_guest.launch_instance( instance_key, vm_name=instance_name, From f27ee7c1f6972dbaa6b3e22b0259ac3dda8140a2 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 9 Oct 2020 11:44:03 +0200 Subject: [PATCH 727/898] Make determination of app under test more robust --- zaza/openstack/charm_tests/test_utils.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 9157bd3..74b76b2 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -127,10 +127,28 @@ class BaseCharmTest(unittest.TestCase): else: cls.model_name = model.get_juju_model() cls.test_config = lifecycle_utils.get_charm_config(fatal=False) + charm_under_test_name = cls.test_config['charm_name'] + if application_name: cls.application_name = application_name else: - cls.application_name = cls.test_config['charm_name'] + deployed_app_names = model.sync_deployed(model_name=cls.model_name) + if charm_under_test_name in deployed_app_names: + # There is an application named like the charm under test. + # Let's consider it the application under test: + cls.application_name = charm_under_test_name + else: + # Let's search for any application whose name starts with the + # name of the charm under test and assume it's the application + # under test: + for app_name in deployed_app_names: + if app_name.startswith(charm_under_test_name): + cls.application_name = app_name + break + else: + logging.warning('Could not find application under test') + return + cls.lead_unit = model.get_lead_unit_name( cls.application_name, model_name=cls.model_name) From 299fae65dbc561ad766cef3da2a3b2c2e0f84ba4 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 12 Oct 2020 12:03:44 +0000 Subject: [PATCH 728/898] Fix test runs with no charm_name Fix z-o-t so that tests that specify an application name can be run irrespective of whether the charm_name has been set in tests.yaml. Closes issue #442 --- zaza/openstack/charm_tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 74b76b2..493a868 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -127,11 +127,11 @@ class BaseCharmTest(unittest.TestCase): else: cls.model_name = model.get_juju_model() cls.test_config = lifecycle_utils.get_charm_config(fatal=False) - charm_under_test_name = cls.test_config['charm_name'] if application_name: cls.application_name = application_name else: + charm_under_test_name = cls.test_config['charm_name'] deployed_app_names = model.sync_deployed(model_name=cls.model_name) if charm_under_test_name in deployed_app_names: # There is an application named like the charm under test. From 1889d0db530ecc03f61976815d0704040b62e0e1 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Mon, 12 Oct 2020 17:51:04 +0200 Subject: [PATCH 729/898] Add func-tests for ceph-osd 'service' action --- zaza/openstack/charm_tests/ceph/osd/tests.py | 177 +++++++++++++++++++ 1 file changed, 177 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index e3f7be1..b8080c0 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -16,6 +16,10 @@ import logging import unittest +import re + +from copy import deepcopy +from typing import List import zaza.openstack.charm_tests.test_utils as test_utils import zaza.model as zaza_model @@ -47,3 +51,176 @@ class SecurityTest(unittest.TestCase): expected_passes, expected_failures, expected_to_pass=True) + + +class OsdService: + """Simple representation of ceph-osd systemd service.""" + + def __init__(self, id_: int): + """ + Init service using its ID. + + e.g.: id_=1 -> ceph.-osd@1.service + """ + self.id = id_ + self.name = 'ceph-osd@{}'.format(id_) + + +class ServiceTest(unittest.TestCase): + """ceph-osd systemd service tests.""" + + TESTED_UNIT = 'ceph-osd/0' + SERVICE_PATTERN = re.compile(r'ceph-osd@(?P\d+)\.service') + + def __init__(self, methodName='runTest'): + """Initialize Test Case.""" + super(ServiceTest, self).__init__(methodName) + self._available_services = None + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph service tests.""" + super(ServiceTest, cls).setUpClass() + + def setUp(self): + """Run test setup.""" + # Note: This counter reset is needed because ceph-osd service is + # limited to 3 restarts per 30 mins which is insufficient + # when running functional tests for 'service' action. This + # limitation is defined in /lib/systemd/system/ceph-osd@.service + # in section [Service] with options 'StartLimitInterval' and + # 'StartLimitBurst' + reset_counter = 'systemctl reset-failed' + zaza_model.run_on_unit(self.TESTED_UNIT, reset_counter) + + def tearDown(self): + """Start ceph-osd services after each test. + + This ensures that the environment is ready for the next tests. + """ + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': 'all'}, + raise_on_failure=True) + + @property + def available_services(self) -> List[OsdService]: + """Return list of all ceph-osd services present on the TESTED_UNIT.""" + if self._available_services is None: + self._available_services = self._fetch_osd_services() + return self._available_services + + def _fetch_osd_services(self) -> List[OsdService]: + """Fetch all ceph-osd services present on the TESTED_UNIT.""" + service_list = [] + service_list_cmd = 'systemctl list-units --full --all ' \ + '--no-pager -t service' + result = zaza_model.run_on_unit(self.TESTED_UNIT, service_list_cmd) + for line in result['Stdout'].split('\n'): + service_name = self.SERVICE_PATTERN.search(line) + if service_name: + service_id = int(service_name.group('service_id')) + service_list.append(OsdService(service_id)) + return service_list + + def test_start_stop_all_by_keyword(self): + """Start and Stop all ceph-osd services using keyword 'all'.""" + service_list = [service.name for service in self.available_services] + + logging.info("Running 'service stop=all' action on {} " + "unit".format(self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'stop': 'all'}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') + + logging.info("Running 'service start=all' action on {} " + "unit".format(self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': 'all'}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running', + pgrep_full=True) + + def test_start_stop_all_by_list(self): + """Start and Stop all ceph-osd services using explicit list.""" + service_list = [service.name for service in self.available_services] + service_ids = [str(service.id) for service in self.available_services] + action_params = ','.join(service_ids) + + logging.info("Running 'service stop={}' action on {} " + "unit".format(action_params, self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'stop': action_params}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') + + logging.info("Running 'service start={}' action on {} " + "unit".format(action_params, self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': action_params}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running', + pgrep_full=True) + + def test_stop_specific(self): + """Stop only specified ceph-osd service.""" + if len(self.available_services) < 2: + raise unittest.SkipTest('This test can be performed only if ' + 'there\'s more than one ceph-osd service ' + 'present on the tested unit') + + should_run = deepcopy(self.available_services) + to_stop = should_run.pop() + should_run = [service.name for service in should_run] + + logging.info("Running 'service stop={} on {} " + "unit".format(to_stop.id, self.TESTED_UNIT)) + + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'stop': to_stop.id}) + + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=[to_stop.name, ], + target_status='stopped') + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=should_run, + target_status='running', + pgrep_full=True) + + def test_start_specific(self): + """Start only specified ceph-osd service.""" + if len(self.available_services) < 2: + raise unittest.SkipTest('This test can be performed only if ' + 'there\'s more than one ceph-osd service ' + 'present on the tested unit') + + service_names = [service.name for service in self.available_services] + should_stop = deepcopy(self.available_services) + to_start = should_stop.pop() + should_stop = [service.name for service in should_stop] + + logging.info("Stopping all running ceph-osd services") + service_stop_cmd = 'systemctl stop ceph-osd.target' + zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd) + + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_names, + target_status='stopped') + + logging.info("Running 'service start={} on {} " + "unit".format(to_start.id, self.TESTED_UNIT)) + + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': to_start.id}) + + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=[to_start.name, ], + target_status='running', + pgrep_full=True) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=should_stop, + target_status='stopped') From d7893798cafbc2a6346b9b924ea6fad5cf5f3a1b Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Mon, 31 Aug 2020 12:08:43 +0530 Subject: [PATCH 730/898] Add new keystone-ldap charm options in LDAP tests New charm options for keystone-ldap are getting introdcued as part of bug [1]. Modify the keystone ldap tests to modify ldap options as per the new charm options. Add ldap options to verify ldap-config-flags takes precendece over explicit config flags. [1] https://bugs.launchpad.net/charm-keystone-ldap/+bug/1832765 --- zaza/openstack/charm_tests/keystone/tests.py | 35 ++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index ed5f42a..9abe78c 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -455,3 +455,38 @@ class LdapTests(BaseKeystoneTest): self.assertIsNotNone(johndoe, "user 'john doe' was unknown") janedoe = self._find_keystone_v3_user('jane doe', 'userdomain') self.assertIsNotNone(janedoe, "user 'jane doe' was unknown") + + +class LdapExplicitCharmConfigTests(LdapTests): + """Keystone ldap tests tests.""" + + def _get_ldap_config(self): + """Generate ldap config for current model. + + :return: tuple of whether ldap-server is running and if so, config + for the keystone-ldap application. + :rtype: Tuple[bool, Dict[str,str]] + """ + ldap_ips = zaza.model.get_app_ips("ldap-server") + self.assertTrue(ldap_ips, "Should be at least one ldap server") + return { + 'ldap-server': "ldap://{}".format(ldap_ips[0]), + 'ldap-user': 'cn=admin,dc=test,dc=com', + 'ldap-password': 'crapper', + 'ldap-suffix': 'dc=test,dc=com', + 'domain-name': 'userdomain', + 'ldap-query-scope': 'one', + 'ldap-user-objectclass': 'inetOrgPerson', + 'ldap-user-id-attribute': 'cn', + 'ldap-user-name-attribute': 'sn', + 'ldap-user-enabled-attribute': 'enabled', + 'ldap-user-enabled-invert': False, + 'ldap-user-enabled-mask': 0, + 'ldap-user-enabled-default': 'True', + 'ldap-group-tree-dn': 'ou=groups', + 'ldap-group-objectclass': 'groupOfNames', + 'ldap-group-id-attribute': 'cn', + 'ldap-group-member-attribute': 'memberUid', + 'ldap-group-members-are-ids': True, + 'ldap-config-flags': '{group_objectclass: "posixGroup"}', + } From b44e556d0aa08abd0c252f64d5d201db46c117a2 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Tue, 13 Oct 2020 20:21:46 +1300 Subject: [PATCH 731/898] Add tests for NRPE checks in aodh Adds a test to ensure that Aodh units related to nrpe get service check files created. --- zaza/openstack/charm_tests/aodh/tests.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index e0178df..3b966e5 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -25,6 +25,7 @@ import zaza.model import zaza.openstack.configure.guest import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.configure.telemetry as telemetry_utils @@ -70,6 +71,13 @@ class AodhTest(test_utils.OpenStackBaseTest): """Check that aodh api is responding.""" self.aodh_client.alarm.list() + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + wait=tenacity.wait_fixed(120), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_commands_on_units(self, cmds, units): + return generic_utils.check_commands_on_units(cmds, units) + @property def services(self): """Return a list of the service that should be running.""" @@ -139,6 +147,26 @@ class AodhTest(test_utils.OpenStackBaseTest): logging.info("Testing pause resume") self.query_aodh_api() + def test_902_nrpe_service_checks(self): + """Confirm that the NRPE service check files are created.""" + units = zaza.model.get_units('aodh') + cmds = [] + if self.release >= self.xenial_ocata: + services = ['aodh-evaluator', 'aodh-notifier', + 'aodh-listener', 'apache2'] + else: + services = ['aodh-api', 'aodh-evaluator', + 'aodh-notifier', 'aodh-listener'] + for check_name in services: + cmds.append( + 'egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_{}.cfg'.format(check_name) + ) + ret = self._retry_check_commands_on_units(cmds, units) + if ret: + logging.info(ret) + self.assertIsNone(ret, msg=ret) + class AodhServerAlarmTest(test_utils.OpenStackBaseTest): """Test server events trigger Aodh alarms.""" From 24755c23820167dbcc45b09eb0ec922da9f74a7c Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Wed, 14 Oct 2020 13:47:20 +0200 Subject: [PATCH 732/898] Add function that waits on desired service status using `systemctl` --- zaza/openstack/charm_tests/ceph/osd/tests.py | 108 +++++++++++++------ 1 file changed, 77 insertions(+), 31 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index b8080c0..58bc5bc 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -66,6 +66,55 @@ class OsdService: self.name = 'ceph-osd@{}'.format(id_) +async def async_wait_for_service_status(unit_name, services, target_status, + model_name=None, timeout=2700): + """Wait for all services on the unit are in the desired state. + + Note:This function emulates the + `zaza.model.async_block_until_service_status` function, but it's using + `systemctl is-active` command istead of `pidof/pgrep` of the original + function. + + :param unit_name: Name of unit to run action on + :type unit_name: str + :param services: List of services to check + :type services: [] + :param target_status: State services must be in (stopped or running) + :type target_status: str + :param model_name: Name of model to query. + :type model_name: str + :param timeout: Time to wait for status to be achieved + :type timeout: int + """ + async def _check_service(): + for service in services: + command = r"systemctl is-active '{}'".format(service) + out = await zaza_model.async_run_on_unit( + unit_name, + command, + model_name=model_name, + timeout=timeout) + response = out['Stdout'].strip() + + if target_status == "running" and response == 'active': + return True + elif target_status == "stopped" and response == 'inactive': + return True + else: + return False + logging.info("Unsing fancy method") + accepted_states = ('stopped', 'running') + if target_status not in accepted_states: + raise RuntimeError('Invalid target state "{}". Accepted states: ' + '{}'.format(target_status, accepted_states)) + + async with zaza_model.run_in_model(model_name): + await zaza_model.async_block_until(_check_service, timeout=timeout) + + +wait_for_service = zaza_model.sync_wrapper(async_wait_for_service_status) + + class ServiceTest(unittest.TestCase): """ceph-osd systemd service tests.""" @@ -130,18 +179,17 @@ class ServiceTest(unittest.TestCase): "unit".format(self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'stop': 'all'}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') logging.info("Running 'service start=all' action on {} " "unit".format(self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'start': 'all'}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='running', - pgrep_full=True) + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running') def test_start_stop_all_by_list(self): """Start and Stop all ceph-osd services using explicit list.""" @@ -153,18 +201,17 @@ class ServiceTest(unittest.TestCase): "unit".format(action_params, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'stop': action_params}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') logging.info("Running 'service start={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'start': action_params}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='running', - pgrep_full=True) + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running') def test_stop_specific(self): """Stop only specified ceph-osd service.""" @@ -183,13 +230,12 @@ class ServiceTest(unittest.TestCase): zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'stop': to_stop.id}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=[to_stop.name, ], - target_status='stopped') - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=should_run, - target_status='running', - pgrep_full=True) + wait_for_service(unit_name=self.TESTED_UNIT, + services=[to_stop.name, ], + target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=should_run, + target_status='running') def test_start_specific(self): """Start only specified ceph-osd service.""" @@ -207,9 +253,9 @@ class ServiceTest(unittest.TestCase): service_stop_cmd = 'systemctl stop ceph-osd.target' zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_names, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_names, + target_status='stopped') logging.info("Running 'service start={} on {} " "unit".format(to_start.id, self.TESTED_UNIT)) @@ -217,10 +263,10 @@ class ServiceTest(unittest.TestCase): zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'start': to_start.id}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=[to_start.name, ], - target_status='running', - pgrep_full=True) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=should_stop, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=[to_start.name, ], + target_status='running') + + wait_for_service(unit_name=self.TESTED_UNIT, + services=should_stop, + target_status='stopped') From 12db27016eb348394cc8fd64d98924985da4b0a2 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Wed, 14 Oct 2020 13:55:23 +0200 Subject: [PATCH 733/898] Fix typos --- zaza/openstack/charm_tests/ceph/osd/tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 58bc5bc..35b6de4 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -68,11 +68,11 @@ class OsdService: async def async_wait_for_service_status(unit_name, services, target_status, model_name=None, timeout=2700): - """Wait for all services on the unit are in the desired state. + """Wait for all services on the unit to be in the desired state. - Note:This function emulates the + Note: This function emulates the `zaza.model.async_block_until_service_status` function, but it's using - `systemctl is-active` command istead of `pidof/pgrep` of the original + `systemctl is-active` command instead of `pidof/pgrep` of the original function. :param unit_name: Name of unit to run action on @@ -102,7 +102,7 @@ async def async_wait_for_service_status(unit_name, services, target_status, return True else: return False - logging.info("Unsing fancy method") + accepted_states = ('stopped', 'running') if target_status not in accepted_states: raise RuntimeError('Invalid target state "{}". Accepted states: ' From 440ee4b274f6ea71c312dad332cd1f08daf42267 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sun, 18 Oct 2020 22:28:32 +0200 Subject: [PATCH 734/898] octavia: Disable check for member operational status Temporarily disable this check until we figure out why operational_status sometimes does not become 'ONLINE' while the member does indeed work and the subsequent retrieval of payload through loadbalancer is successful ref LP: #1896729. --- zaza/openstack/charm_tests/octavia/tests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index eb0b217..e95a33f 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -280,7 +280,13 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): lambda x: octavia_client.member_show( pool_id=pool_id, member_id=x), member_id, - operating_status='ONLINE' if monitor else '') + operating_status='') + # Temporarily disable this check until we figure out why + # operational_status sometimes does not become 'ONLINE' + # while the member does indeed work and the subsequent + # retrieval of payload through loadbalancer is successful + # ref LP: #1896729. + # operating_status='ONLINE' if monitor else '') logging.info(resp) return lb From 4f83645b1c51857eb9bf073f564d00ff773df4c4 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Mon, 19 Oct 2020 08:40:23 +0000 Subject: [PATCH 735/898] Add python-ironicclient to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f4e31f8..b18cd9a 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ install_require = [ 'python-barbicanclient>=4.0.1,<5.0.0', 'python-designateclient>=1.5,<3.0.0', 'python-heatclient<2.0.0', + 'python-ironicclient', 'python-glanceclient<3.0.0', 'python-keystoneclient<3.22.0', 'python-manilaclient<2.0.0', From a75d49d9595f9d34c1a96e2371517f140306ffa2 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Mon, 19 Oct 2020 11:44:13 +0000 Subject: [PATCH 736/898] Add tenacity to image uploads --- zaza/openstack/charm_tests/ironic/setup.py | 29 +++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/ironic/setup.py b/zaza/openstack/charm_tests/ironic/setup.py index b03c8cb..fe7d034 100644 --- a/zaza/openstack/charm_tests/ironic/setup.py +++ b/zaza/openstack/charm_tests/ironic/setup.py @@ -16,6 +16,7 @@ import copy import os +import tenacity import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.utilities.openstack as openstack_utils @@ -74,6 +75,20 @@ FLAVORS = { } +def _add_image(url, image_name, backend="swift", + disk_format="raw", container_format="bare"): + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + reraise=True): + with attempt: + glance_setup.add_image( + url, + image_name=image_name, + backend=backend, + disk_format=disk_format, + container_format=container_format) + + def add_ironic_deployment_image(initrd_url=None, kernel_url=None): """Add Ironic deploy images to glance. @@ -91,15 +106,17 @@ def add_ironic_deployment_image(initrd_url=None, kernel_url=None): kernel_url = os.environ.get('TEST_IRONIC_DEPLOY_VMLINUZ', None) if not all([initrd_url, kernel_url]): raise ValueError("Missing required deployment image URLs") - glance_setup.add_image( + + _add_image( initrd_url, - image_name=initrd_name, + initrd_name, backend="swift", disk_format="ari", container_format="ari") - glance_setup.add_image( + + _add_image( kernel_url, - image_name=vmlinuz_name, + vmlinuz_name, backend="swift", disk_format="aki", container_format="aki") @@ -117,9 +134,9 @@ def add_ironic_os_image(image_url=None): if image_url is None: raise ValueError("Missing image_url") - glance_setup.add_image( + _add_image( image_url, - image_name=image_name, + image_name, backend="swift", disk_format="raw", container_format="bare") From 67aaa10ad6f15d9be513a5205b7871a18f90ebf2 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 20 Oct 2020 14:43:47 +0200 Subject: [PATCH 737/898] Retry Nova flavor- and keypair-creation Fixes #452 --- zaza/openstack/charm_tests/nova/setup.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/nova/setup.py b/zaza/openstack/charm_tests/nova/setup.py index c556c97..7f8ee53 100644 --- a/zaza/openstack/charm_tests/nova/setup.py +++ b/zaza/openstack/charm_tests/nova/setup.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Code for configureing nova.""" +"""Code for configuring nova.""" + +import tenacity import zaza.openstack.utilities.openstack as openstack_utils from zaza.openstack.utilities import ( @@ -21,6 +23,9 @@ from zaza.openstack.utilities import ( import zaza.openstack.charm_tests.nova.utils as nova_utils +@tenacity.retry(stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)) def create_flavors(nova_client=None): """Create basic flavors. @@ -43,6 +48,9 @@ def create_flavors(nova_client=None): flavorid=nova_utils.FLAVORS[flavor]['flavorid']) +@tenacity.retry(stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)) def manage_ssh_key(nova_client=None): """Create basic flavors. From 79e43c7f684f18839a5ff59457cfc8e9116b3b59 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 22 Oct 2020 15:28:16 +0100 Subject: [PATCH 738/898] bionic-train -> ussuri needs to upgrade placement For the openstack upgrade of train to ussuri, the placement charm needs to be upgraded. Add it to the UPGRADE_SERVICES. Related launchpad bug: LP:1724174 [1] [1]: https://bugs.launchpad.net/openstack-mojo-specs/+bug/1724174 --- zaza/openstack/utilities/openstack.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index b62885e..8d25433 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -123,6 +123,10 @@ CHARM_TYPES = { 'pkg': 'ceph-common', 'origin_setting': 'source' }, + 'placement': { + 'pkg': 'placement-common', + 'origin_setting': 'openstack-origin' + }, } # Older tests use the order the services appear in the list to imply @@ -143,6 +147,7 @@ UPGRADE_SERVICES = [ 'type': CHARM_TYPES['openstack-dashboard']}, {'name': 'ovn-central', 'type': CHARM_TYPES['ovn-central']}, {'name': 'ceph-mon', 'type': CHARM_TYPES['ceph-mon']}, + {'name': 'placement', 'type': CHARM_TYPES['placement']}, ] From 990be121717396802a496de1fd8453de2ef67029 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Wed, 28 Oct 2020 13:58:49 +0100 Subject: [PATCH 739/898] Fix async_wait_for_service_status() checking only first service in the list --- zaza/openstack/charm_tests/ceph/osd/tests.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 35b6de4..6f1bfac 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -60,7 +60,7 @@ class OsdService: """ Init service using its ID. - e.g.: id_=1 -> ceph.-osd@1.service + e.g.: id_=1 -> ceph-osd@1 """ self.id = id_ self.name = 'ceph-osd@{}'.format(id_) @@ -87,6 +87,7 @@ async def async_wait_for_service_status(unit_name, services, target_status, :type timeout: int """ async def _check_service(): + services_ok = True for service in services: command = r"systemctl is-active '{}'".format(service) out = await zaza_model.async_run_on_unit( @@ -97,11 +98,14 @@ async def async_wait_for_service_status(unit_name, services, target_status, response = out['Stdout'].strip() if target_status == "running" and response == 'active': - return True + continue elif target_status == "stopped" and response == 'inactive': - return True + continue else: - return False + services_ok = False + break + + return services_ok accepted_states = ('stopped', 'running') if target_status not in accepted_states: From 991d7ea7a41fe86ed5c5abe36966214514c02cce Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Tue, 3 Nov 2020 15:47:18 +0100 Subject: [PATCH 740/898] Removed `typing` usage --- zaza/openstack/charm_tests/ceph/osd/tests.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 6f1bfac..03e212f 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -19,7 +19,6 @@ import unittest import re from copy import deepcopy -from typing import List import zaza.openstack.charm_tests.test_utils as test_utils import zaza.model as zaza_model @@ -56,7 +55,7 @@ class SecurityTest(unittest.TestCase): class OsdService: """Simple representation of ceph-osd systemd service.""" - def __init__(self, id_: int): + def __init__(self, id_): """ Init service using its ID. @@ -78,7 +77,7 @@ async def async_wait_for_service_status(unit_name, services, target_status, :param unit_name: Name of unit to run action on :type unit_name: str :param services: List of services to check - :type services: [] + :type services: List[str] :param target_status: State services must be in (stopped or running) :type target_status: str :param model_name: Name of model to query. @@ -122,7 +121,7 @@ wait_for_service = zaza_model.sync_wrapper(async_wait_for_service_status) class ServiceTest(unittest.TestCase): """ceph-osd systemd service tests.""" - TESTED_UNIT = 'ceph-osd/0' + TESTED_UNIT = 'ceph-osd/0' # This can be any ceph-osd unit in the model SERVICE_PATTERN = re.compile(r'ceph-osd@(?P\d+)\.service') def __init__(self, methodName='runTest'): @@ -156,13 +155,13 @@ class ServiceTest(unittest.TestCase): raise_on_failure=True) @property - def available_services(self) -> List[OsdService]: + def available_services(self): """Return list of all ceph-osd services present on the TESTED_UNIT.""" if self._available_services is None: self._available_services = self._fetch_osd_services() return self._available_services - def _fetch_osd_services(self) -> List[OsdService]: + def _fetch_osd_services(self): """Fetch all ceph-osd services present on the TESTED_UNIT.""" service_list = [] service_list_cmd = 'systemctl list-units --full --all ' \ @@ -181,7 +180,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop=all' action on {} " "unit".format(self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + zaza_model.run_action_on_units([self.TESTED_UNIT], 'service', action_params={'stop': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, From a91ae76975d40dd94fc211368af964b9084ee0fa Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 6 Nov 2020 14:12:59 +0000 Subject: [PATCH 741/898] Stop assuming there are two iscsi gateways The tests assume there are two iscsi gateways which is not a safe assumption. --- .../openstack/charm_tests/ceph/iscsi/tests.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index a986833..0766e4a 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -64,26 +64,25 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): :rtype: Dict """ gw_units = zaza.model.get_units('ceph-iscsi') - primary_gw = gw_units[0] - secondary_gw = gw_units[1] host_names = generic_utils.get_unit_hostnames(gw_units, fqdn=True) client_entity_ids = [ u.entity_id for u in zaza.model.get_units('ubuntu')] ctxt = { 'client_entity_ids': sorted(client_entity_ids), 'gw_iqn': self.GW_IQN, - 'gw1_ip': primary_gw.public_address, - 'gw1_hostname': host_names[primary_gw.entity_id], - 'gw1_entity_id': primary_gw.entity_id, - 'gw2_ip': secondary_gw.public_address, - 'gw2_hostname': host_names[secondary_gw.entity_id], - 'gw2_entity_id': secondary_gw.entity_id, 'chap_creds': 'username={chap_username} password={chap_password}', 'gwcli_gw_dir': '/iscsi-targets/{gw_iqn}/gateways', 'gwcli_hosts_dir': '/iscsi-targets/{gw_iqn}/hosts', 'gwcli_disk_dir': '/disks', 'gwcli_client_dir': '{gwcli_hosts_dir}/{client_initiatorname}', } + ctxt['gateway_units'] = [ + { + 'entity_id': u.entity_id, + 'ip': u.public_address, + 'hostname': host_names[u.entity_id]} + for u in zaza.model.get_units('ceph-iscsi')] + ctxt['gw_ip'] = sorted([g['ip'] for g in ctxt['gateway_units']])[0] return ctxt def run_commands(self, unit_name, commands, ctxt): @@ -116,9 +115,8 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): 'ceph-iscsi', 'create-target', action_params={ - 'gateway-units': '{} {}'.format( - ctxt['gw1_entity_id'], - ctxt['gw2_entity_id']), + 'gateway-units': ' '.join([g['entity_id'] + for g in ctxt['gateway_units']]), 'iqn': self.GW_IQN, 'rbd-pool-name': ctxt.get('pool_name', ''), 'ec-rbd-metadata-pool': ctxt.get('ec_meta_pool_name', ''), @@ -139,7 +137,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): base_op_cmd = ('iscsiadm --mode node --targetname {gw_iqn} ' '--op=update ').format(**ctxt) setup_cmds = [ - 'iscsiadm -m discovery -t st -p {gw1_ip}', + 'iscsiadm -m discovery -t st -p {gw_ip}', base_op_cmd + '-n node.session.auth.authmethod -v CHAP', base_op_cmd + '-n node.session.auth.username -v {chap_username}', base_op_cmd + '-n node.session.auth.password -v {chap_password}', From c91ebb5ec42d12acfbd12af73fc124378dc5805e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 9 Nov 2020 10:38:23 +0100 Subject: [PATCH 742/898] Remove unused security tests (#441) --- .../charm_tests/security/__init__.py | 15 ---- zaza/openstack/charm_tests/security/tests.py | 81 ------------------- 2 files changed, 96 deletions(-) delete mode 100644 zaza/openstack/charm_tests/security/__init__.py delete mode 100644 zaza/openstack/charm_tests/security/tests.py diff --git a/zaza/openstack/charm_tests/security/__init__.py b/zaza/openstack/charm_tests/security/__init__.py deleted file mode 100644 index ec47696..0000000 --- a/zaza/openstack/charm_tests/security/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2018 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test security checklist.""" diff --git a/zaza/openstack/charm_tests/security/tests.py b/zaza/openstack/charm_tests/security/tests.py deleted file mode 100644 index 741b53b..0000000 --- a/zaza/openstack/charm_tests/security/tests.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2018 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Encapsulate general security testing.""" - -import unittest - -import zaza.model as model -import zaza.charm_lifecycle.utils as utils -from zaza.openstack.utilities.file_assertions import ( - assert_path_glob, - assert_single_file, -) - - -def _make_test_function(application, file_details, paths=None): - """Generate a test function given the specified inputs. - - :param application: Application name to assert file ownership on - :type application: str - :param file_details: Dictionary of file details to test - :type file_details: dict - :param paths: List of paths to test in this application - :type paths: Optional[list(str)] - :returns: Test function - :rtype: unittest.TestCase - """ - def test(self): - for unit in model.get_units(application): - unit = unit.entity_id - if '*' in file_details['path']: - assert_path_glob(self, unit, file_details, paths) - else: - assert_single_file(self, unit, file_details) - return test - - -def _add_tests(): - """Add tests to the unittest.TestCase.""" - def class_decorator(cls): - """Add tests based on input yaml to `cls`.""" - files = utils.get_charm_config('./file-assertions.yaml') - deployed_applications = model.sync_deployed() - for name, attributes in files.items(): - # Lets make sure to only add tests for deployed applications - if name in deployed_applications: - paths = [ - file['path'] for - file in attributes['files'] - if "*" not in file["path"] - ] - for file in attributes['files']: - test_func = _make_test_function(name, file, paths=paths) - setattr( - cls, - 'test_{}_{}'.format(name, file['path']), - test_func) - return cls - return class_decorator - - -class FileOwnershipTest(unittest.TestCase): - """Encapsulate File ownership tests.""" - - pass - - -FileOwnershipTest = _add_tests()(FileOwnershipTest) From 82c1a5321b0cf6498b161059f4d1f8d5b8e719fe Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 10 Nov 2020 15:34:32 +0100 Subject: [PATCH 743/898] Fix volume and backup name --- zaza/openstack/charm_tests/cinder_backup/tests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 2281cc1..41da8ab 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -93,7 +93,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): stop=tenacity.stop_after_attempt(3)): with attempt: # Create ceph-backed cinder volume - cinder_vol_name = '{}-410-vol-{}'.format( + cinder_vol_name = '{}-410{}-vol'.format( self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) cinder_vol = self.cinder_client.volumes.create( name=cinder_vol_name, size=1) @@ -115,7 +115,12 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): # fixed in this area. # When the backup creation succeeds, it usually does within # 12 minutes. - vol_backup_name = cinder_vol_name + '-backup' + # NOTE(lourot): it seems like we have to stick to the + # `-backup-vol` naming convention otherwise + # cinder-backup fails with a ceph/rados error about not being + # able to find the pool. See lp:1897587 + vol_backup_name = '{}-410{}-backup-vol'.format( + self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) vol_backup = self.cinder_client.backups.create( cinder_vol.id, vol_backup_name) openstack_utils.resource_reaches_status( From ca40adc07878543896ef369f28b5eafe8a3a6f83 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 11 Nov 2020 08:44:51 +0100 Subject: [PATCH 744/898] Fix comment --- zaza/openstack/charm_tests/cinder_backup/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 41da8ab..c16e207 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -116,7 +116,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): # When the backup creation succeeds, it usually does within # 12 minutes. # NOTE(lourot): it seems like we have to stick to the - # `-backup-vol` naming convention otherwise + # `--backup-vol` naming convention otherwise # cinder-backup fails with a ceph/rados error about not being # able to find the pool. See lp:1897587 vol_backup_name = '{}-410{}-backup-vol'.format( From 64fd8026975837282f335790ea1f8af5ac7882df Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 11 Nov 2020 14:55:09 +0100 Subject: [PATCH 745/898] Fix call to cinder_client.backups.create() --- zaza/openstack/charm_tests/cinder_backup/tests.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index c16e207..97b3658 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -93,7 +93,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): stop=tenacity.stop_after_attempt(3)): with attempt: # Create ceph-backed cinder volume - cinder_vol_name = '{}-410{}-vol'.format( + cinder_vol_name = '{}-410-{}-vol'.format( self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) cinder_vol = self.cinder_client.volumes.create( name=cinder_vol_name, size=1) @@ -115,14 +115,10 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): # fixed in this area. # When the backup creation succeeds, it usually does within # 12 minutes. - # NOTE(lourot): it seems like we have to stick to the - # `--backup-vol` naming convention otherwise - # cinder-backup fails with a ceph/rados error about not being - # able to find the pool. See lp:1897587 - vol_backup_name = '{}-410{}-backup-vol'.format( + vol_backup_name = '{}-410-{}-backup-vol'.format( self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) vol_backup = self.cinder_client.backups.create( - cinder_vol.id, vol_backup_name) + cinder_vol.id, name=vol_backup_name) openstack_utils.resource_reaches_status( self.cinder_client.backups, vol_backup.id, From e8e16a353cc1011c7e315e798e2569544b409bb2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 12 Nov 2020 12:14:42 +0000 Subject: [PATCH 746/898] Add magpie tests --- zaza/openstack/charm_tests/magpie/__init__.py | 15 ++++ zaza/openstack/charm_tests/magpie/tests.py | 82 +++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 zaza/openstack/charm_tests/magpie/__init__.py create mode 100644 zaza/openstack/charm_tests/magpie/tests.py diff --git a/zaza/openstack/charm_tests/magpie/__init__.py b/zaza/openstack/charm_tests/magpie/__init__.py new file mode 100644 index 0000000..7fd0805 --- /dev/null +++ b/zaza/openstack/charm_tests/magpie/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing Magpie.""" diff --git a/zaza/openstack/charm_tests/magpie/tests.py b/zaza/openstack/charm_tests/magpie/tests.py new file mode 100644 index 0000000..0ac25a6 --- /dev/null +++ b/zaza/openstack/charm_tests/magpie/tests.py @@ -0,0 +1,82 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Magpie testing.""" + +import logging + +import zaza + +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils + + +class MagpieTest(test_utils.BaseCharmTest): + """Base Magpie tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for Magpie charm operation tests.""" + super(MagpieTest, cls).setUpClass() + unit_names = sorted( + [i.entity_id + for i in zaza.model.get_units('magpie')]) + cls.test_unit_0 = unit_names[0] + cls.test_unit_1 = unit_names[1] + + def test_break_dns_single(self): + """Check DNS failure is reflected in workload status.""" + zaza.model.run_on_unit( + self.test_unit_0, + 'mv /etc/resolv.conf /etc/resolv.conf.bak') + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + zaza.model.block_until_unit_wl_message_match( + self.test_unit_0, + '.*rev dns failed.*') + logging.info('Restoring /etc/resolv.conf') + zaza.model.run_on_unit( + self.test_unit_0, + 'mv /etc/resolv.conf.bak /etc/resolv.conf') + logging.info('Updating status') + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + + def test_break_ping_single(self): + """Check ping failure is reflected in workload status.""" + icmp = "iptables {} INPUT -p icmp --icmp-type echo-request -j REJECT" + logging.info('Blocking ping on {}'.format(self.test_unit_1)) + zaza.model.run_on_unit( + self.test_unit_1, + icmp.format('--append')) + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + logging.info('Checking status on {}'.format(self.test_unit_0)) + zaza.model.block_until_unit_wl_message_match( + self.test_unit_0, + '.*icmp failed.*') + logging.info('Allowing ping on {}'.format(self.test_unit_1)) + zaza.model.run_on_unit( + self.test_unit_1, + icmp.format('--delete')) + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + logging.info('Checking status on {}'.format(self.test_unit_0)) + zaza.model.block_until_unit_wl_message_match( + self.test_unit_0, + '.*icmp ok.*') From b23d9e285b68315ba80bf85656a9661ba4d3f06c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 18 Nov 2020 09:36:44 +0000 Subject: [PATCH 747/898] Support nova client micro versions Support micro versions when requesting a nova client. --- unit_tests/utilities/test_zaza_utilities_openstack.py | 9 +++++++++ zaza/openstack/utilities/openstack.py | 7 +++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 5ad0b9b..a047d9d 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -289,6 +289,15 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): openstack_utils.get_undercloud_keystone_session() self.get_keystone_session.assert_called_once_with(_auth, verify=None) + def test_get_nova_session_client(self): + session_mock = mock.MagicMock() + self.patch_object(openstack_utils.novaclient_client, "Client") + openstack_utils.get_nova_session_client(session_mock) + self.Client.assert_called_once_with(2, session=session_mock) + self.Client.reset_mock() + openstack_utils.get_nova_session_client(session_mock, version=2.56) + self.Client.assert_called_once_with(2.56, session=session_mock) + def test_get_urllib_opener(self): self.patch_object(openstack_utils.urllib.request, "ProxyHandler") self.patch_object(openstack_utils.urllib.request, "HTTPHandler") diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 8d25433..5d2dcca 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -253,15 +253,18 @@ def get_designate_session_client(**kwargs): **kwargs) -def get_nova_session_client(session): +def get_nova_session_client(session, version=None): """Return novaclient authenticated by keystone session. :param session: Keystone session object :type session: keystoneauth1.session.Session object + :param version: Version of client to request. + :type version: float :returns: Authenticated novaclient :rtype: novaclient.Client object """ - return novaclient_client.Client(2, session=session) + version = version or 2 + return novaclient_client.Client(version, session=session) def get_neutron_session_client(session): From dfecc58f5af22b726d7b799b9b332e8a00428afe Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 18 Nov 2020 12:13:59 +0000 Subject: [PATCH 748/898] Extend resource_reaches_status The resource_reaches_status assumes that the attribute to be checked is called 'status' but this can vary between resource types so allow the name of the attribute to specified. --- .../utilities/test_zaza_utilities_openstack.py | 9 ++++++--- zaza/openstack/utilities/openstack.py | 13 ++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 5ad0b9b..75509aa 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -369,12 +369,15 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'e01df65a') def test__resource_reaches_status_bespoke(self): + client_mock = mock.MagicMock() resource_mock = mock.MagicMock() - resource_mock.get.return_value = mock.MagicMock(status='readyish') + resource_mock.special_status = 'readyish' + client_mock.get.return_value = resource_mock openstack_utils._resource_reaches_status( - resource_mock, + client_mock, 'e01df65a', - 'readyish') + 'readyish', + resource_attribute='special_status') def test__resource_reaches_status_bespoke_fail(self): resource_mock = mock.MagicMock() diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index c309116..0b25093 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1886,7 +1886,8 @@ def download_image(image_url, target_file): def _resource_reaches_status(resource, resource_id, expected_status='available', - msg='resource'): + msg='resource', + resource_attribute='status'): """Wait for an openstack resources status to reach an expected status. Wait for an openstack resources status to reach an expected status @@ -1902,9 +1903,11 @@ def _resource_reaches_status(resource, resource_id, :type expected_status: str :param msg: text to identify purpose in logging :type msg: str + :param resource_attribute: Resource attribute to check against + :type resource_attribute: str :raises: AssertionError """ - resource_status = resource.get(resource_id).status + resource_status = getattr(resource.get(resource_id), resource_attribute) logging.info("{}: resource {} in {} state, waiting for {}".format( msg, resource_id, resource_status, expected_status)) assert resource_status == expected_status @@ -1914,6 +1917,7 @@ def resource_reaches_status(resource, resource_id, expected_status='available', msg='resource', + resource_attribute='status', wait_exponential_multiplier=1, wait_iteration_max_time=60, stop_after_attempt=8, @@ -1933,6 +1937,8 @@ def resource_reaches_status(resource, :type expected_status: str :param msg: text to identify purpose in logging :type msg: str + :param resource_attribute: Resource attribute to check against + :type resource_attribute: str :param wait_exponential_multiplier: Wait 2^x * wait_exponential_multiplier seconds between each retry :type wait_exponential_multiplier: int @@ -1954,7 +1960,8 @@ def resource_reaches_status(resource, resource, resource_id, expected_status, - msg) + msg, + resource_attribute) def _resource_removed(resource, resource_id, msg="resource"): From c5c11194a1a5032a954308889d0f76872ffe8d6f Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 17 Nov 2020 16:02:29 +0100 Subject: [PATCH 749/898] Support 'major.minor' when determining OS release of OVN packages --- zaza/openstack/utilities/openstack.py | 22 ++++++++++++++++++++-- zaza/openstack/utilities/os_versions.py | 14 +++++++++----- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index c309116..67f5f02 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -19,6 +19,7 @@ This module contains a number of functions for interacting with OpenStack. from .os_versions import ( OPENSTACK_CODENAMES, SWIFT_CODENAMES, + OVN_CODENAMES, PACKAGE_CODENAMES, OPENSTACK_RELEASES_PAIRS, ) @@ -1479,8 +1480,23 @@ def get_swift_codename(version): :returns: Codename for swift :rtype: string """ - codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] - return codenames[0] + return _get_special_codename(version, SWIFT_CODENAMES) + + +def get_ovn_codename(version): + """Determine OpenStack codename that corresponds to OVN version. + + :param version: Version of OVN + :type version: string + :returns: Codename for OVN + :rtype: string + """ + return _get_special_codename(version, OVN_CODENAMES) + + +def _get_special_codename(version, codenames): + found = [k for k, v in six.iteritems(codenames) if version in v] + return found[0] def get_os_code_info(package, pkg_version): @@ -1517,6 +1533,8 @@ def get_os_code_info(package, pkg_version): # < Liberty co-ordinated project versions if 'swift' in package: return get_swift_codename(vers) + elif 'ovn' in package: + return get_ovn_codename(vers) else: return OPENSTACK_CODENAMES[vers] diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 75dc723..33e2fac 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -71,7 +71,6 @@ OPENSTACK_RELEASES_PAIRS = [ 'eoan_train', 'bionic_ussuri', 'focal_ussuri', 'focal_victoria', 'groovy_victoria'] -# The ugly duckling - must list releases oldest to newest SWIFT_CODENAMES = OrderedDict([ ('diablo', ['1.4.3']), @@ -113,6 +112,15 @@ SWIFT_CODENAMES = OrderedDict([ ['2.25.0']), ]) +OVN_CODENAMES = OrderedDict([ + ('train', + ['2.12']), + ('ussuri', + ['20.03']), + ('victoria', + ['20.06']), +]) + # >= Liberty version->codename mapping PACKAGE_CODENAMES = { 'nova-common': OrderedDict([ @@ -245,10 +253,6 @@ PACKAGE_CODENAMES = { ('10', 'ussuri'), ('11', 'victoria'), ]), - 'ovn-common': OrderedDict([ - ('2', 'train'), - ('20', 'ussuri'), - ]), 'ceph-common': OrderedDict([ ('10', 'mitaka'), # jewel ('12', 'queens'), # luminous From 817473675d8eb06673b5bb64fb7699cad9cc51c0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 18 Nov 2020 14:12:13 +0000 Subject: [PATCH 750/898] Numbers are imutable so safe to have as default argument --- zaza/openstack/utilities/openstack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 5d2dcca..112c572 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -253,7 +253,7 @@ def get_designate_session_client(**kwargs): **kwargs) -def get_nova_session_client(session, version=None): +def get_nova_session_client(session, version=2): """Return novaclient authenticated by keystone session. :param session: Keystone session object @@ -263,7 +263,6 @@ def get_nova_session_client(session, version=None): :returns: Authenticated novaclient :rtype: novaclient.Client object """ - version = version or 2 return novaclient_client.Client(version, session=session) From b07afac21ac1e95a62fb656b19928c7d61ad4ede Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 18 Nov 2020 14:33:37 +0000 Subject: [PATCH 751/898] Add placement-common versions for train->ussuri upgrade --- zaza/openstack/utilities/openstack.py | 1 - zaza/openstack/utilities/os_versions.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index c309116..ae4d8c4 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1493,7 +1493,6 @@ def get_os_code_info(package, pkg_version): :returns: Codename for package :rtype: string """ - # {'code_num': entry, 'code_name': OPENSTACK_CODENAMES[entry]} # Remove epoch if it exists if ':' in pkg_version: pkg_version = pkg_version.split(':')[1:][0] diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 75dc723..2c806d5 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -256,4 +256,9 @@ PACKAGE_CODENAMES = { ('14', 'train'), # nautilus ('15', 'ussuri'), # octopus ]), + 'placement-common': OrderedDict([ + ('2', 'train'), + ('3', 'ussuri'), + ('4', 'victoria'), + ]), } From 7404b0cb9b1a4e757842e836b35d2d34bfd4403a Mon Sep 17 00:00:00 2001 From: Rodrigo Barbieri Date: Thu, 12 Nov 2020 14:42:50 -0300 Subject: [PATCH 752/898] Update Keystone LDAP tests - Reversed ldap-config-flags vs charm configs precedence for https://review.opendev.org/748364 - Added new functional test to validate the precedence - Added zaza helper function to assist handling charm configs that do not accept "" (like booleans and ints), and reverting to unset values through config_change() - Fixed zaza reset helper function that apparently never worked ({}.keys() returns dict_keys which is not a sequence) --- unit_tests/charm_tests/test_utils.py | 62 +++++++++- zaza/openstack/charm_tests/keystone/tests.py | 114 +++++++++++++++---- zaza/openstack/charm_tests/test_utils.py | 42 ++++++- 3 files changed, 194 insertions(+), 24 deletions(-) diff --git a/unit_tests/charm_tests/test_utils.py b/unit_tests/charm_tests/test_utils.py index ffb7f83..c1e8166 100644 --- a/unit_tests/charm_tests/test_utils.py +++ b/unit_tests/charm_tests/test_utils.py @@ -97,7 +97,7 @@ class TestBaseCharmTest(ut_utils.BaseTestCase): self.set_application_config.reset_mock() self.assertFalse(self.set_application_config.called) self.reset_application_config.assert_called_once_with( - 'anApp', alterna_config.keys(), model_name='aModel') + 'anApp', list(alterna_config.keys()), model_name='aModel') self.wait_for_application_states.assert_has_calls([ mock.call(model_name='aModel', states={}), mock.call(model_name='aModel', states={}), @@ -125,6 +125,66 @@ class TestBaseCharmTest(ut_utils.BaseTestCase): self.assertFalse(self.wait_for_agent_status.called) self.assertFalse(self.wait_for_application_states.called) + def test_separate_non_string_config(self): + intended_cfg_keys = ['foo2', 'foo3', 'foo4', 'foo5'] + current_config_mock = { + 'foo2': None, + 'foo3': 'old_bar3', + 'foo4': None, + 'foo5': 'old_bar5', + } + self.patch_target('config_current') + self.config_current.return_value = current_config_mock + non_string_type_keys = ['foo2', 'foo3', 'foo4'] + expected_result_filtered = { + 'foo3': 'old_bar3', + 'foo5': 'old_bar5', + } + expected_result_special = { + 'foo2': None, + 'foo4': None, + } + current, non_string = ( + self.target.config_current_separate_non_string_type_keys( + non_string_type_keys, intended_cfg_keys, 'application_name') + ) + + self.assertEqual(expected_result_filtered, current) + self.assertEqual(expected_result_special, non_string) + + self.config_current.assert_called_once_with( + 'application_name', intended_cfg_keys) + + def test_separate_special_config_None_params(self): + current_config_mock = { + 'foo1': 'old_bar1', + 'foo2': None, + 'foo3': 'old_bar3', + 'foo4': None, + 'foo5': 'old_bar5', + } + self.patch_target('config_current') + self.config_current.return_value = current_config_mock + non_string_type_keys = ['foo2', 'foo3', 'foo4'] + expected_result_filtered = { + 'foo1': 'old_bar1', + 'foo3': 'old_bar3', + 'foo5': 'old_bar5', + } + expected_result_special = { + 'foo2': None, + 'foo4': None, + } + current, non_string = ( + self.target.config_current_separate_non_string_type_keys( + non_string_type_keys) + ) + + self.assertEqual(expected_result_filtered, current) + self.assertEqual(expected_result_special, non_string) + + self.config_current.assert_called_once_with(None, None) + class TestOpenStackBaseTest(ut_utils.BaseTestCase): diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 9abe78c..fb0c2f3 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -380,7 +380,12 @@ class SecurityTests(BaseKeystoneTest): class LdapTests(BaseKeystoneTest): - """Keystone ldap tests tests.""" + """Keystone ldap tests.""" + + non_string_type_keys = ('ldap-user-enabled-mask', + 'ldap-user-enabled-invert', + 'ldap-group-members-are-ids', + 'ldap-use-pool') @classmethod def setUpClass(cls): @@ -434,31 +439,44 @@ class LdapTests(BaseKeystoneTest): def test_100_keystone_ldap_users(self): """Validate basic functionality of keystone API with ldap.""" application_name = 'keystone-ldap' - config = self._get_ldap_config() + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) with self.config_change( - self.config_current(application_name, config.keys()), - config, - application_name=application_name): - logging.info( - 'Waiting for users to become available in keystone...' - ) - test_config = lifecycle_utils.get_charm_config(fatal=False) - zaza.model.wait_for_application_states( - states=test_config.get("target_deploy_status", {}) - ) + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Waiting for users to become available in keystone...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) - with self.v3_keystone_preferred(): - # NOTE(jamespage): Test fixture should have johndoe and janedoe - # accounts - johndoe = self._find_keystone_v3_user('john doe', 'userdomain') - self.assertIsNotNone(johndoe, "user 'john doe' was unknown") - janedoe = self._find_keystone_v3_user('jane doe', 'userdomain') - self.assertIsNotNone(janedoe, "user 'jane doe' was unknown") + with self.v3_keystone_preferred(): + # NOTE(jamespage): Test fixture should have + # johndoe and janedoe accounts + johndoe = self._find_keystone_v3_user( + 'john doe', 'userdomain') + self.assertIsNotNone( + johndoe, "user 'john doe' was unknown") + janedoe = self._find_keystone_v3_user( + 'jane doe', 'userdomain') + self.assertIsNotNone( + janedoe, "user 'jane doe' was unknown") class LdapExplicitCharmConfigTests(LdapTests): - """Keystone ldap tests tests.""" + """Keystone ldap tests.""" def _get_ldap_config(self): """Generate ldap config for current model. @@ -484,9 +502,61 @@ class LdapExplicitCharmConfigTests(LdapTests): 'ldap-user-enabled-mask': 0, 'ldap-user-enabled-default': 'True', 'ldap-group-tree-dn': 'ou=groups', - 'ldap-group-objectclass': 'groupOfNames', + 'ldap-group-objectclass': '', 'ldap-group-id-attribute': 'cn', 'ldap-group-member-attribute': 'memberUid', 'ldap-group-members-are-ids': True, - 'ldap-config-flags': '{group_objectclass: "posixGroup"}', + 'ldap-config-flags': '{group_objectclass: "posixGroup",' + ' use_pool: True,' + ' group_tree_dn: "group_tree_dn_foobar"}', } + + def test_200_config_flags_precedence(self): + """Validates precedence when the same config options are used.""" + application_name = 'keystone-ldap' + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) + + with self.config_change( + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Performing LDAP settings validation in keystone.conf...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) + units = zaza.model.get_units("keystone-ldap", + model_name=self.model_name) + result = zaza.model.run_on_unit( + units[0].name, + "cat /etc/keystone/domains/keystone.userdomain.conf") + # not present in charm config, but present in config flags + self.assertIn("use_pool = True", result['stdout'], + "use_pool value is expected to be present and " + "set to True in the config file") + # ldap-config-flags overriding empty charm config value + self.assertIn("group_objectclass = posixGroup", + result['stdout'], + "group_objectclass is expected to be present and" + " set to posixGroup in the config file") + # overridden by charm config, not written to file + self.assertNotIn( + "group_tree_dn_foobar", + result['stdout'], + "user_tree_dn ldap-config-flags value needs to be " + "overridden by ldap-user-tree-dn in config file") + # complementing the above, value used is from charm setting + self.assertIn("group_tree_dn = ou=groups", result['stdout'], + "user_tree_dn value is expected to be present " + "and set to dc=test,dc=com in the config file") diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 493a868..14ee0f3 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -154,6 +154,46 @@ class BaseCharmTest(unittest.TestCase): model_name=cls.model_name) logging.debug('Leader unit is {}'.format(cls.lead_unit)) + def config_current_separate_non_string_type_keys( + self, non_string_type_keys, config_keys=None, + application_name=None): + """Obtain current config and the non-string type config separately. + + If the charm config option is not string, it will not accept being + reverted back in "config_change()" method if the current value is None. + Therefore, obtain the current config and separate those out, so they + can be used for a separate invocation of "config_change()" with + reset_to_charm_default set to True. + + :param config_keys: iterable of strs to index into the current config. + If None, return all keys from the config + :type config_keys: Optional[Iterable[str]] + :param non_string_type_keys: list of non-string type keys to be + separated out only if their current value + is None + :type non_string_type_keys: list + :param application_name: String application name for use when called + by a charm under test other than the object's + application. + :type application_name: Optional[str] + :return: Dictionary of current charm configs without the + non-string type keys provided, and dictionary of the + non-string keys found in the supplied config_keys list. + :rtype: Dict[str, Any], Dict[str, None] + """ + current_config = self.config_current(application_name, config_keys) + non_string_type_config = {} + if config_keys is None: + config_keys = list(current_config.keys()) + for key in config_keys: + # We only care if the current value is None, otherwise it will + # not face issues being reverted by "config_change()" + if key in non_string_type_keys and current_config[key] is None: + non_string_type_config[key] = None + current_config.pop(key) + + return current_config, non_string_type_config + def config_current(self, application_name=None, keys=None): """Get Current Config of an application normalized into key-values. @@ -275,7 +315,7 @@ class BaseCharmTest(unittest.TestCase): 'charm default: "{}"' .format(alternate_config.keys())) model.reset_application_config(application_name, - alternate_config.keys(), + list(alternate_config.keys()), model_name=self.model_name) elif default_config == alternate_config: logging.debug('default_config == alternate_config, not attempting ' From 90aca8be5eead4e2af4b24a56bdfdfb2668776cc Mon Sep 17 00:00:00 2001 From: Arif Ali Date: Wed, 7 Oct 2020 17:39:35 +0100 Subject: [PATCH 753/898] Add ldap group/membership tests Adds a test to check for groups that are coming from LDAP. Adds a test to ensure that openstack is able to check the membership of a user in the group. Signed-off-by: Arif Ali --- zaza/openstack/charm_tests/keystone/tests.py | 130 ++++++++++++++++++- 1 file changed, 125 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index fb0c2f3..b2830ef 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -407,13 +407,22 @@ class LdapTests(BaseKeystoneTest): 'ldap-password': 'crapper', 'ldap-suffix': 'dc=test,dc=com', 'domain-name': 'userdomain', + 'ldap-config-flags': + { + 'group_tree_dn': 'ou=groups,dc=test,dc=com', + 'group_objectclass': 'posixGroup', + 'group_name_attribute': 'cn', + 'group_member_attribute': 'memberUid', + 'group_members_are_ids': 'true', + } } - def _find_keystone_v3_user(self, username, domain): + def _find_keystone_v3_user(self, username, domain, group=None): """Find a user within a specified keystone v3 domain. :param str username: Username to search for in keystone :param str domain: username selected from which domain + :param str group: group to search for in keystone for group membership :return: return username if found :rtype: Optional[str] """ @@ -423,9 +432,15 @@ class LdapTests(BaseKeystoneTest): openstack_utils.get_overcloud_auth(address=ip)) client = openstack_utils.get_keystone_session_client(session) - domain_users = client.users.list( - domain=client.domains.find(name=domain).id - ) + if group is None: + domain_users = client.users.list( + domain=client.domains.find(name=domain).id, + ) + else: + domain_users = client.users.list( + domain=client.domains.find(name=domain).id, + group=self._find_keystone_v3_group(group, domain).id, + ) usernames = [u.name.lower() for u in domain_users] if username.lower() in usernames: @@ -436,6 +451,33 @@ class LdapTests(BaseKeystoneTest): ) return None + def _find_keystone_v3_group(self, group, domain): + """Find a group within a specified keystone v3 domain. + + :param str group: Group to search for in keystone + :param str domain: group selected from which domain + :return: return group if found + :rtype: Optional[str] + """ + for ip in self.keystone_ips: + logging.info('Keystone IP {}'.format(ip)) + session = openstack_utils.get_keystone_session( + openstack_utils.get_overcloud_auth(address=ip)) + client = openstack_utils.get_keystone_session_client(session) + + domain_groups = client.groups.list( + domain=client.domains.find(name=domain).id + ) + + for searched_group in domain_groups: + if searched_group.name.lower() == group.lower(): + return searched_group + + logging.debug( + "Group {} was not found. Returning None.".format(group) + ) + return None + def test_100_keystone_ldap_users(self): """Validate basic functionality of keystone API with ldap.""" application_name = 'keystone-ldap' @@ -474,6 +516,83 @@ class LdapTests(BaseKeystoneTest): self.assertIsNotNone( janedoe, "user 'jane doe' was unknown") + def test_101_keystone_ldap_groups(self): + """Validate basic functionality of keystone API with ldap.""" + application_name = 'keystone-ldap' + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) + + with self.config_change( + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Waiting for groups to become available in keystone...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) + + with self.v3_keystone_preferred(): + # NOTE(arif-ali): Test fixture should have openstack and + # admin groups + openstack_group = self._find_keystone_v3_group( + 'openstack', 'userdomain') + self.assertIsNotNone( + openstack_group.name, "group 'openstack' was unknown") + admin_group = self._find_keystone_v3_group( + 'admin', 'userdomain') + self.assertIsNotNone( + admin_group.name, "group 'admin' was unknown") + + def test_102_keystone_ldap_group_membership(self): + """Validate basic functionality of keystone API with ldap.""" + application_name = 'keystone-ldap' + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) + + with self.config_change( + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Waiting for groups to become available in keystone...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) + + with self.v3_keystone_preferred(): + # NOTE(arif-ali): Test fixture should have openstack and + # admin groups + openstack_group = self._find_keystone_v3_user( + 'john doe', 'userdomain', group='openstack') + self.assertIsNotNone( + openstack_group, + "john doe was not in group 'openstack'") + admin_group = self._find_keystone_v3_user( + 'john doe', 'userdomain', group='admin') + self.assertIsNotNone( + admin_group, "'john doe' was not in group 'admin'") + class LdapExplicitCharmConfigTests(LdapTests): """Keystone ldap tests.""" @@ -501,9 +620,10 @@ class LdapExplicitCharmConfigTests(LdapTests): 'ldap-user-enabled-invert': False, 'ldap-user-enabled-mask': 0, 'ldap-user-enabled-default': 'True', - 'ldap-group-tree-dn': 'ou=groups', + 'ldap-group-tree-dn': 'ou=groups,dc=test,dc=com', 'ldap-group-objectclass': '', 'ldap-group-id-attribute': 'cn', + 'ldap-group-name-attribute': 'cn', 'ldap-group-member-attribute': 'memberUid', 'ldap-group-members-are-ids': True, 'ldap-config-flags': '{group_objectclass: "posixGroup",' From d6f719e3313659c3a290e239412887a747f9598c Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Thu, 26 Nov 2020 14:42:12 +0100 Subject: [PATCH 754/898] Skip 'service' action tests on systems without systemd --- zaza/openstack/charm_tests/ceph/osd/tests.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 03e212f..63f9c4e 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -136,8 +136,14 @@ class ServiceTest(unittest.TestCase): def setUp(self): """Run test setup.""" - # Note: This counter reset is needed because ceph-osd service is - # limited to 3 restarts per 30 mins which is insufficient + # Skip 'service' action tests on systems without systemd + result = zaza_model.run_on_unit(self.TESTED_UNIT, 'which systemctl') + if not result['Stdout']: + raise unittest.SkipTest("'service' action is not supported on " + "systems without 'systemd'. Skipping " + "tests.") + # Note(mkalcok): This counter reset is needed because ceph-osd service + # is limited to 3 restarts per 30 mins which is insufficient # when running functional tests for 'service' action. This # limitation is defined in /lib/systemd/system/ceph-osd@.service # in section [Service] with options 'StartLimitInterval' and From a8ca4720a3d9199949a24e61db3129b21b3ea269 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 27 Nov 2020 09:53:12 +0100 Subject: [PATCH 755/898] Fix BlueStoreCompressionCharmOperation on Victoria (#468) Before this fix, the test tried to determine the OpenStack release based on the ceph-mon charm. Unfortunately Ceph has the same version on Ussuri and Victoria. As a consequence the test would wrongly conclude that it's testing against "groovy_ussuri", which isn't a valid Ubuntu/OpenStack pair. With this fix, the test now determines the OpenStack release based on the keystone charm, with which we are able to tell Ussuri and Victoria apart. This test class is being run against the following charm functional tests at the moment, which all have a keystone charm in their test bundles: nova-compute, cinder-ceph, glance, ceph-fs, ceph-radosgw and gnocchi. --- zaza/openstack/charm_tests/ceph/tests.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 19ce1ec..349d514 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -881,9 +881,7 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest): def setUpClass(cls): """Perform class one time initialization.""" super(BlueStoreCompressionCharmOperation, cls).setUpClass() - cls.current_release = zaza_openstack.get_os_release( - zaza_openstack.get_current_os_release_pair( - application='ceph-mon')) + cls.current_release = zaza_openstack.get_os_release() cls.bionic_rocky = zaza_openstack.get_os_release('bionic_rocky') def setUp(self): From 69b61c63f4ca647e597c17a98baa33d1fe9ee0a7 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 27 Nov 2020 12:04:43 +0100 Subject: [PATCH 756/898] Add HaclusterScaleBackAndForthTest --- zaza/openstack/charm_tests/hacluster/tests.py | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 29751e1..99272ed 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -77,7 +77,67 @@ class HaclusterTest(HaclusterBaseTest): class HaclusterScalebackTest(HaclusterBaseTest): - """hacluster scaleback tests.""" + """hacluster scaleback tests. + + Use for testing older releases where lp:1400481 wasn't fixed yet. + Superseded by HaclusterScaleBackAndForthTest. + """ + + @classmethod + def setUpClass(cls): + """Run class setup for running hacluster scaleback tests.""" + super(HaclusterScalebackTest, cls).setUpClass() + test_config = cls.test_config['tests_options']['hacluster'] + cls._principle_app_name = test_config['principle-app-name'] + cls._hacluster_charm_name = test_config['hacluster-charm-name'] + + def test_930_scaleback(self): + """Remove a unit and add a new one.""" + principle_units = sorted(zaza.model.get_status().applications[ + self._principle_app_name]['units'].keys()) + self.assertEqual(len(principle_units), 3) + doomed_principle_unit = principle_units[0] + other_principle_unit = principle_units[1] + doomed_hacluster_unit = juju_utils.get_subordinate_units( + [doomed_principle_unit], charm_name=self._hacluster_charm_name)[0] + other_hacluster_unit = juju_utils.get_subordinate_units( + [other_principle_unit], charm_name=self._hacluster_charm_name)[0] + + logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) + zaza.model.run_action( + doomed_hacluster_unit, + 'pause', + raise_on_failure=True) + logging.info('OK') + + logging.info('Removing {}'.format(doomed_principle_unit)) + zaza.model.destroy_unit( + self._principle_app_name, + doomed_principle_unit, + wait_disappear=True) + logging.info('OK') + + logging.info('Waiting for model to settle') + zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') + zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + zaza.model.block_until_all_units_idle() + logging.info('OK') + + logging.info('Adding an hacluster unit') + zaza.model.add_unit(self._principle_app_name, wait_appear=True) + logging.info('OK') + + logging.info('Waiting for model to settle') + zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active') + # NOTE(lourot): the principle application remains blocked after scaling + # back up until lp:1400481 is solved. + zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + zaza.model.block_until_all_units_idle() + logging.debug('OK') + + +class HaclusterScaleBackAndForthTest(HaclusterBaseTest): + """hacluster tests scaling back and forth.""" @classmethod def setUpClass(cls): From a3248185f147d04774c3e8f2e32c84d386be385c Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 27 Nov 2020 13:52:44 +0100 Subject: [PATCH 757/898] Fix copy-paste mistake --- zaza/openstack/charm_tests/hacluster/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 99272ed..517cde2 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -141,8 +141,8 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): @classmethod def setUpClass(cls): - """Run class setup for running hacluster scaleback tests.""" - super(HaclusterScalebackTest, cls).setUpClass() + """Run class setup for running hacluster tests.""" + super(HaclusterScaleBackAndForthTest, cls).setUpClass() test_config = cls.test_config['tests_options']['hacluster'] cls._principle_app_name = test_config['principle-app-name'] cls._hacluster_charm_name = test_config['hacluster-charm-name'] From 7f45d461e76159c929ce333fc7a9badd8f11d9d7 Mon Sep 17 00:00:00 2001 From: coreycb Date: Mon, 30 Nov 2020 10:42:35 -0500 Subject: [PATCH 758/898] Log stderr when manila-ganesha verify() fails (#461) --- zaza/openstack/charm_tests/manila_ganesha/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 27009bf..02ea825 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -16,6 +16,8 @@ """Encapsulate Manila Ganesha testing.""" +import logging + from tenacity import Retrying, stop_after_attempt, wait_exponential from manilaclient import client as manilaclient @@ -87,6 +89,8 @@ packages: # Write a file on instance_1 def verify_setup(stdin, stdout, stderr): status = stdout.channel.recv_exit_status() + if status: + logging.info("{}".format(stderr.readlines()[0].strip())) self.assertEqual(status, 0) mount_path = share.export_locations[0] @@ -119,6 +123,8 @@ packages: def verify(stdin, stdout, stderr): status = stdout.channel.recv_exit_status() + if status: + logging.info("{}".format(stderr.readlines()[0].strip())) self.assertEqual(status, 0) out = "" for line in iter(stdout.readline, ""): From 7ced54b382adaa2181c1134e17126ac1d485028b Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Wed, 2 Dec 2020 10:22:00 +0000 Subject: [PATCH 759/898] Concurrent series upgrade updates (#466) * Updates to concurrent series upgrade Updates to make it run more in parallel and spend less time waiting on the whole model when updating machines. * Make the concurrent series upgrade tests work This is a number of changes to get the concurrent (here called 'parallel' historically) series upgrade tests to work. A number of changes were required which included limiting the number of concurrent async co-routines (futures) that could be run as with large models it hits the limits of the Py3 runtime. * Fix the tests and change pause order in maybe_pause_things Due to an additional model helper call, an additional model AsyncMock is required. Also the pause order had changed, and this is restored to ensure the original design is retained (for pause order). Clean up some commented out code and sort out a few PEP8 errors. * Update comment to reflect code (3 -> 4) * Fix tests that fail on bionic but pass on focal Essentially, asyncio.gather has different behaviour on bionic that focal. Although this doesn't affect testing, it does affect the unit tests. These changes are simply to normalise the behaviour of unit tests on focal and bionic. --- ..._zaza_utilities_parallel_series_upgrade.py | 26 +++- .../charm_tests/charm_upgrade/tests.py | 1 + zaza/openstack/charm_tests/mysql/utils.py | 18 ++- .../series_upgrade/parallel_tests.py | 41 ++++-- zaza/openstack/charm_tests/test_utils.py | 26 +++- zaza/openstack/charm_tests/vault/setup.py | 37 +++++ .../utilities/parallel_series_upgrade.py | 130 ++++++++++++------ zaza/openstack/utilities/series_upgrade.py | 7 +- 8 files changed, 220 insertions(+), 66 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 457ce1b..0231dbf 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -182,6 +182,8 @@ class TestParallelSeriesUpgrade(AioTestCase): self.model.async_wait_for_unit_idle = mock.AsyncMock() self.async_run_on_machine = mock.AsyncMock() self.model.async_run_on_machine = self.async_run_on_machine + self.model.async_block_until_units_on_machine_are_idle = \ + mock.AsyncMock() @mock.patch.object(upgrade_utils.cl_utils, 'get_class') async def test_run_post_application_upgrade_functions( @@ -492,7 +494,13 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_remove_confdef_file.assert_called_once_with('1') mock_add_confdef_file.assert_called_once_with('1') - async def test_maybe_pause_things_primary(self): + @mock.patch("asyncio.gather") + async def test_maybe_pause_things_primary(self, mock_gather): + async def _gather(*args): + for f in args: + await f + + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], @@ -503,7 +511,13 @@ class TestParallelSeriesUpgrade(AioTestCase): mock.call('app/2', "pause", action_params={}), ]) - async def test_maybe_pause_things_subordinates(self): + @mock.patch("asyncio.gather") + async def test_maybe_pause_things_subordinates(self, mock_gather): + async def _gather(*args): + for f in args: + await f + + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], @@ -514,7 +528,13 @@ class TestParallelSeriesUpgrade(AioTestCase): mock.call('app-hacluster/2', "pause", action_params={}), ]) - async def test_maybe_pause_things_all(self): + @mock.patch("asyncio.gather") + async def test_maybe_pause_things_all(self, mock_gather): + async def _gather(*args): + for f in args: + await f + + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], diff --git a/zaza/openstack/charm_tests/charm_upgrade/tests.py b/zaza/openstack/charm_tests/charm_upgrade/tests.py index f5f301a..f55caf0 100644 --- a/zaza/openstack/charm_tests/charm_upgrade/tests.py +++ b/zaza/openstack/charm_tests/charm_upgrade/tests.py @@ -35,6 +35,7 @@ class FullCloudCharmUpgradeTest(unittest.TestCase): """Run setup for Charm Upgrades.""" cli_utils.setup_logging() cls.lts = LTSGuestCreateTest() + cls.lts.setUpClass() cls.target_charm_namespace = '~openstack-charmers-next' def get_upgrade_url(self, charm_url): diff --git a/zaza/openstack/charm_tests/mysql/utils.py b/zaza/openstack/charm_tests/mysql/utils.py index 1fe5114..75da5d9 100644 --- a/zaza/openstack/charm_tests/mysql/utils.py +++ b/zaza/openstack/charm_tests/mysql/utils.py @@ -19,8 +19,16 @@ import zaza.model as model async def complete_cluster_series_upgrade(): """Run the complete-cluster-series-upgrade action on the lead unit.""" - # TODO: Make this work across either mysql or percona-cluster names - await model.async_run_action_on_leader( - 'mysql', - 'complete-cluster-series-upgrade', - action_params={}) + # Note that some models use mysql as the application name, and other's use + # percona-cluster. Try mysql first, and if it doesn't exist, then try + # percona-cluster instead. + try: + await model.async_run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) + except KeyError: + await model.async_run_action_on_leader( + 'percona-cluster', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py index a49eae6..e524fa2 100644 --- a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -21,6 +21,7 @@ import logging import os import sys import unittest +import juju from zaza import model from zaza.openstack.utilities import ( @@ -55,6 +56,12 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): @classmethod def setUpClass(cls): """Run setup for Series Upgrades.""" + # NOTE(ajkavanagh): Set the jujulib Connection frame size to 4GB to + # cope with all the outputs from series upgrade; long term, don't send + # that output back, which will require that the upgrade function in the + # charm doesn't capture the output of the upgrade in the action, but + # instead puts it somewhere that can by "juju scp"ed. + juju.client.connection.Connection.MAX_FRAME_SIZE = 2**32 cli_utils.setup_logging() cls.from_series = None cls.to_series = None @@ -89,19 +96,25 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): upgrade_function = \ parallel_series_upgrade.parallel_series_upgrade + # allow up to 4 parallel upgrades at a time. This is to limit the + # amount of data/calls that asyncio is handling as it's gets + # unstable if all the applications are done at the same time. + sem = asyncio.Semaphore(4) for charm_name in apps: charm = applications[charm_name]['charm'] name = upgrade_utils.extract_charm_name_from_url(charm) upgrade_config = parallel_series_upgrade.app_config(name) upgrade_functions.append( - upgrade_function( - charm_name, - **upgrade_config, - from_series=from_series, - to_series=to_series, - completed_machines=completed_machines, - workaround_script=workaround_script, - files=files)) + wrap_coroutine_with_sem( + sem, + upgrade_function( + charm_name, + **upgrade_config, + from_series=from_series, + to_series=to_series, + completed_machines=completed_machines, + workaround_script=workaround_script, + files=files))) asyncio.get_event_loop().run_until_complete( asyncio.gather(*upgrade_functions)) model.block_until_all_units_idle() @@ -109,6 +122,18 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): logging.info("Done!") +async def wrap_coroutine_with_sem(sem, coroutine): + """Wrap a coroutine with a semaphore to limit concurrency. + + :param sem: The semaphore to limit concurrency + :type sem: asyncio.Semaphore + :param coroutine: the corouting to limit concurrency + :type coroutine: types.CoroutineType + """ + async with sem: + await coroutine + + class OpenStackParallelSeriesUpgrade(ParallelSeriesUpgradeTest): """OpenStack Series Upgrade. diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 14ee0f3..5a3e6c1 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -120,7 +120,23 @@ class BaseCharmTest(unittest.TestCase): @classmethod def setUpClass(cls, application_name=None, model_alias=None): - """Run setup for test class to create common resources.""" + """Run setup for test class to create common resources. + + Note: the derived class may not use the application_name; if it's set + to None then this setUpClass() method will attempt to extract the + application name from the charm_config (essentially the test.yaml) + using the key 'charm_name' in the test_config. If that isn't present, + then there will be no application_name set, and this is considered a + generic scenario of a whole model rather than a particular charm under + test. + + :param application_name: the name of the applications that the derived + class is testing. If None, then it's a generic test not connected + to any single charm. + :type application_name: Optional[str] + :param model_alias: the alias to use if needed. + :type model_alias: Optional[str] + """ cls.model_aliases = model.get_juju_model_aliases() if model_alias: cls.model_name = cls.model_aliases[model_alias] @@ -131,7 +147,13 @@ class BaseCharmTest(unittest.TestCase): if application_name: cls.application_name = application_name else: - charm_under_test_name = cls.test_config['charm_name'] + try: + charm_under_test_name = cls.test_config['charm_name'] + except KeyError: + logging.warning("No application_name and no charm config so " + "not setting the application_name. Likely a " + "scenario test.") + return deployed_app_names = model.sync_deployed(model_name=cls.model_name) if charm_under_test_name in deployed_app_names: # There is an application named like the charm under test. diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 4db90b5..adfb92d 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -26,6 +26,7 @@ import zaza.model import zaza.openstack.utilities.cert import zaza.openstack.utilities.openstack import zaza.openstack.utilities.generic +import zaza.openstack.utilities.exceptions as zaza_exceptions import zaza.utilities.juju as juju_utils @@ -73,9 +74,27 @@ def basic_setup_and_unseal(cacert=None): zaza.model.run_on_unit(unit.name, './hooks/update-status') +async def mojo_or_default_unseal_by_unit(): + """Unseal any units reported as sealed using a cacert. + + The mojo cacert is tried first, and if that doesn't exist, then the default + zaza located cacert is used. + """ + try: + await mojo_unseal_by_unit() + except zaza_exceptions.CACERTNotFound: + await unseal_by_unit() + + def mojo_unseal_by_unit(): """Unseal any units reported as sealed using mojo cacert.""" cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path() + unseal_by_unit(cacert) + + +def unseal_by_unit(cacert=None): + """Unseal any units reported as sealed using mojo cacert.""" + cacert = cacert or get_cacert_file() vault_creds = vault_utils.get_credentails() for client in vault_utils.get_clients(cacert=cacert): if client.hvac_client.is_sealed(): @@ -86,9 +105,27 @@ def mojo_unseal_by_unit(): zaza.model.run_on_unit(unit_name, './hooks/update-status') +async def async_mojo_or_default_unseal_by_unit(): + """Unseal any units reported as sealed using a cacert. + + The mojo cacert is tried first, and if that doesn't exist, then the default + zaza located cacert is used. + """ + try: + await async_mojo_unseal_by_unit() + except zaza_exceptions.CACERTNotFound: + await async_unseal_by_unit() + + async def async_mojo_unseal_by_unit(): """Unseal any units reported as sealed using mojo cacert.""" cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path() + await async_unseal_by_unit(cacert) + + +async def async_unseal_by_unit(cacert=None): + """Unseal any units reported as sealed using vault cacert.""" + cacert = cacert or get_cacert_file() vault_creds = vault_utils.get_credentails() for client in vault_utils.get_clients(cacert=cacert): if client.hvac_client.is_sealed(): diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index aa6ab0e..610496d 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -58,7 +58,9 @@ def app_config(charm_name): } exceptions = { 'rabbitmq-server': { - 'origin': 'source', + # NOTE: AJK disable config-changed on rabbitmq-server due to bug: + # #1896520 + 'origin': None, 'pause_non_leader_subordinate': False, 'post_application_upgrade_functions': [ ('zaza.openstack.charm_tests.rabbitmq_server.utils.' @@ -94,7 +96,7 @@ def app_config(charm_name): 'pause_non_leader_subordinate': True, 'post_upgrade_functions': [ ('zaza.openstack.charm_tests.vault.setup.' - 'async_mojo_unseal_by_unit')] + 'async_mojo_or_default_unseal_by_unit')] }, 'mongodb': { 'origin': None, @@ -191,49 +193,64 @@ async def parallel_series_upgrade( status = (await model.async_get_status()).applications[application] logging.info( "Configuring leader / non leaders for {}".format(application)) - leader, non_leaders = get_leader_and_non_leaders(status) - for leader_name, leader_unit in leader.items(): + leaders, non_leaders = get_leader_and_non_leaders(status) + for leader_unit in leaders.values(): leader_machine = leader_unit["machine"] - leader = leader_name - machines = [ - unit["machine"] for name, unit - in non_leaders.items() - if unit['machine'] not in completed_machines] + machines = [unit["machine"] for name, unit in non_leaders.items() + if unit['machine'] not in completed_machines] await maybe_pause_things( status, non_leaders, pause_non_leader_subordinate, pause_non_leader_primary) - await series_upgrade_utils.async_set_series( - application, to_series=to_series) - app_idle = [ + # wait for the entire application set to be idle before starting upgrades + await asyncio.gather(*[ model.async_wait_for_unit_idle(unit, include_subordinates=True) - for unit in status["units"] - ] - await asyncio.gather(*app_idle) + for unit in status["units"]]) await prepare_series_upgrade(leader_machine, to_series=to_series) - prepare_group = [ - prepare_series_upgrade(machine, to_series=to_series) - for machine in machines] - await asyncio.gather(*prepare_group) + await asyncio.gather(*[ + wait_for_idle_then_prepare_series_upgrade( + machine, to_series=to_series) + for machine in machines]) if leader_machine not in completed_machines: machines.append(leader_machine) - upgrade_group = [ + await asyncio.gather(*[ series_upgrade_machine( machine, origin=origin, application=application, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) - for machine in machines - ] - await asyncio.gather(*upgrade_group) + for machine in machines]) completed_machines.extend(machines) + await series_upgrade_utils.async_set_series( + application, to_series=to_series) await run_post_application_upgrade_functions( post_application_upgrade_functions) +async def wait_for_idle_then_prepare_series_upgrade( + machine, to_series, model_name=None): + """Wait for the units to idle the do prepare_series_upgrade. + + We need to be sure that all the units are idle prior to actually calling + prepare_series_upgrade() as otherwise the call will fail. It has to be + checked because when the leader is paused it may kick off relation hooks in + the other units in an HA group. + + :param machine: the machine that is going to be prepared + :type machine: str + :param to_series: The series to which to upgrade + :type to_series: str + :param model_name: Name of model to query. + :type model_name: str + """ + await model.async_block_until_units_on_machine_are_idle( + machine, model_name=model_name) + await prepare_series_upgrade(machine, to_series=to_series) + + async def serial_series_upgrade( application, from_series='xenial', @@ -307,8 +324,10 @@ async def serial_series_upgrade( non_leaders, pause_non_leader_subordinate, pause_non_leader_primary) + logging.info("Finishing pausing application: {}".format(application)) await series_upgrade_utils.async_set_series( application, to_series=to_series) + logging.info("Finished set series for application: {}".format(application)) if not follower_first and leader_machine not in completed_machines: await model.async_wait_for_unit_idle(leader, include_subordinates=True) await prepare_series_upgrade(leader_machine, to_series=to_series) @@ -321,6 +340,8 @@ async def serial_series_upgrade( files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) completed_machines.append(leader_machine) + logging.info("Finished upgrading of leader for application: {}" + .format(application)) # for machine in machines: for unit_name, unit in non_leaders.items(): @@ -339,6 +360,8 @@ async def serial_series_upgrade( files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) + logging.info("Finished upgrading non leaders for application: {}" + .format(application)) if follower_first and leader_machine not in completed_machines: await model.async_wait_for_unit_idle(leader, include_subordinates=True) @@ -354,6 +377,7 @@ async def serial_series_upgrade( completed_machines.append(leader_machine) await run_post_application_upgrade_functions( post_application_upgrade_functions) + logging.info("Done series upgrade for: {}".format(application)) async def series_upgrade_machine( @@ -381,17 +405,16 @@ async def series_upgrade_machine( :returns: None :rtype: None """ - logging.info( - "About to series-upgrade ({})".format(machine)) + logging.info("About to series-upgrade ({})".format(machine)) await run_pre_upgrade_functions(machine, pre_upgrade_functions) await add_confdef_file(machine) await async_dist_upgrade(machine) await async_do_release_upgrade(machine) await remove_confdef_file(machine) await reboot(machine) + await series_upgrade_utils.async_complete_series_upgrade(machine) if origin: await os_utils.async_set_origin(application, origin) - await series_upgrade_utils.async_complete_series_upgrade(machine) await run_post_upgrade_functions(post_upgrade_functions) @@ -484,8 +507,7 @@ async def maybe_pause_things( :returns: Nothing :trype: None """ - subordinate_pauses = [] - leader_pauses = [] + unit_pauses = [] for unit in units: if pause_non_leader_subordinate: if status["units"][unit].get("subordinates"): @@ -495,15 +517,19 @@ async def maybe_pause_things( logging.info("Skipping pausing {} - blacklisted" .format(subordinate)) else: - logging.info("Pausing {}".format(subordinate)) - subordinate_pauses.append(model.async_run_action( - subordinate, "pause", action_params={})) + unit_pauses.append( + _pause_helper("subordinate", subordinate)) if pause_non_leader_primary: - logging.info("Pausing {}".format(unit)) - leader_pauses.append( - model.async_run_action(unit, "pause", action_params={})) - await asyncio.gather(*leader_pauses) - await asyncio.gather(*subordinate_pauses) + unit_pauses.append(_pause_helper("leader", unit)) + if unit_pauses: + await asyncio.gather(*unit_pauses) + + +async def _pause_helper(_type, unit): + """Pause helper to ensure that the log happens nearer to the action.""" + logging.info("Pausing ({}) {}".format(_type, unit)) + await model.async_run_action(unit, "pause", action_params={}) + logging.info("Finished Pausing ({}) {}".format(_type, unit)) def get_leader_and_non_leaders(status): @@ -541,14 +567,14 @@ async def prepare_series_upgrade(machine, to_series): NOTE: This is a new feature in juju behind a feature flag and not yet in libjuju. export JUJU_DEV_FEATURE_FLAGS=upgrade-series - :param machine_num: Machine number - :type machine_num: str + :param machine: Machine number + :type machine: str :param to_series: The series to which to upgrade :type to_series: str :returns: None :rtype: None """ - logging.info("Preparing series upgrade for: {}".format(machine)) + logging.info("Preparing series upgrade for: %s", machine) await series_upgrade_utils.async_prepare_series_upgrade( machine, to_series=to_series) @@ -564,9 +590,8 @@ async def reboot(machine): try: await model.async_run_on_machine(machine, 'sudo init 6 & exit') # await run_on_machine(unit, "sudo reboot && exit") - except subprocess.CalledProcessError as e: - logging.warn("Error doing reboot: {}".format(e)) - pass + except subprocess.CalledProcessError as error: + logging.warning("Error doing reboot: %s", error) async def async_dist_upgrade(machine): @@ -577,16 +602,31 @@ async def async_dist_upgrade(machine): :returns: None :rtype: None """ - logging.info('Updating package db ' + machine) + logging.info('Updating package db %s', machine) update_cmd = 'sudo apt-get update' await model.async_run_on_machine(machine, update_cmd) - logging.info('Updating existing packages ' + machine) + logging.info('Updating existing packages %s', machine) dist_upgrade_cmd = ( """yes | sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes """ """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") await model.async_run_on_machine(machine, dist_upgrade_cmd) + rdict = await model.async_run_on_machine( + machine, + "cat /var/run/reboot-required || true") + if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): + logging.info("dist-upgrade required reboot machine: %s", machine) + await reboot(machine) + logging.info("Waiting for machine to come back afer reboot: %s", + machine) + await model.async_block_until_file_missing_on_machine( + machine, "/var/run/reboot-required") + logging.info("Waiting for machine idleness on %s", machine) + await asyncio.sleep(5.0) + await model.async_block_until_units_on_machine_are_idle(machine) + # TODO: change this to wait on units on the machine + # await model.async_block_until_all_units_idle() async def async_do_release_upgrade(machine): @@ -597,7 +637,7 @@ async def async_do_release_upgrade(machine): :returns: None :rtype: None """ - logging.info('Upgrading ' + machine) + logging.info('Upgrading %s', machine) do_release_upgrade_cmd = ( 'yes | sudo DEBIAN_FRONTEND=noninteractive ' 'do-release-upgrade -f DistUpgradeViewNonInteractive') diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 97ba153..42683f6 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -884,7 +884,8 @@ def dist_upgrade(unit_name): """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") model.run_on_unit(unit_name, dist_upgrade_cmd) - rdict = model.run_on_unit(unit_name, "cat /var/run/reboot-required") + rdict = model.run_on_unit(unit_name, + "cat /var/run/reboot-required || true") if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): logging.info("dist-upgrade required reboot {}".format(unit_name)) os_utils.reboot(unit_name) @@ -919,8 +920,8 @@ async def async_dist_upgrade(unit_name): """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") await model.async_run_on_unit(unit_name, dist_upgrade_cmd) - rdict = await model.async_run_on_unit(unit_name, - "cat /var/run/reboot-required") + rdict = await model.async_run_on_unit( + unit_name, "cat /var/run/reboot-required || true") if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): logging.info("dist-upgrade required reboot {}".format(unit_name)) await os_utils.async_reboot(unit_name) From 10756f7a663d611f6dae0f69d57fe07c97ff9243 Mon Sep 17 00:00:00 2001 From: Linda Guo Date: Fri, 11 Dec 2020 10:13:41 +1100 Subject: [PATCH 760/898] Add NovaComputeActionTest test class for virsh_audit action --- zaza/openstack/charm_tests/nova/tests.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 79e0e70..b8e7552 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -155,6 +155,29 @@ class NovaCompute(test_utils.OpenStackBaseTest): self.assertFalse(int(run['Code']) == 0) +class NovaComputeActionTest(test_utils.OpenStackBaseTest): + """Run nova-compute specific tests. + + Add this test class for new nova-compute action + to avoid breaking older version + """ + + def test_virsh_audit_action(self): + """Test virsh-audit action.""" + for unit in zaza.model.get_units('nova-compute', + model_name=self.model_name): + logging.info('Running `virsh-audit` action' + ' on unit {}'.format(unit.entity_id)) + action = zaza.model.run_action( + unit.entity_id, + 'virsh-audit', + model_name=self.model_name, + action_params={}) + if "failed" in action.data["status"]: + raise Exception( + "The action failed: {}".format(action.data["message"])) + + class NovaCloudController(test_utils.OpenStackBaseTest): """Run nova-cloud-controller specific tests.""" From cfdabe273a36cf82323619ab929e38b5773ae2ed Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Fri, 11 Dec 2020 09:49:33 +0100 Subject: [PATCH 761/898] Adjust tests after rework of the 'service' action into `start` and `stop` actions --- zaza/openstack/charm_tests/ceph/osd/tests.py | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 63f9c4e..c4e6e36 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -156,8 +156,8 @@ class ServiceTest(unittest.TestCase): This ensures that the environment is ready for the next tests. """ - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'start': 'all'}, + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'disks': 'all'}, raise_on_failure=True) @property @@ -186,16 +186,16 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop=all' action on {} " "unit".format(self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT], 'service', - action_params={'stop': 'all'}) + zaza_model.run_action_on_units([self.TESTED_UNIT], 'stop', + action_params={'disks': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='stopped') logging.info("Running 'service start=all' action on {} " "unit".format(self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'start': 'all'}) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'disks': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='running') @@ -208,16 +208,16 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'stop': action_params}) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop', + action_params={'disks': action_params}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='stopped') logging.info("Running 'service start={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'start': action_params}) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'disks': action_params}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='running') @@ -236,8 +236,8 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop={} on {} " "unit".format(to_stop.id, self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'stop': to_stop.id}) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop', + action_params={'disks': to_stop.id}) wait_for_service(unit_name=self.TESTED_UNIT, services=[to_stop.name, ], @@ -269,8 +269,8 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service start={} on {} " "unit".format(to_start.id, self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'start': to_start.id}) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'disks': to_start.id}) wait_for_service(unit_name=self.TESTED_UNIT, services=[to_start.name, ], From 96592c6a85d90eb2594b24798d0d564a0e17c009 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Fri, 11 Dec 2020 16:16:15 +0100 Subject: [PATCH 762/898] Rename parameter `disks` to `osds` --- zaza/openstack/charm_tests/ceph/osd/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index c4e6e36..63fb128 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -157,7 +157,7 @@ class ServiceTest(unittest.TestCase): This ensures that the environment is ready for the next tests. """ zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', - action_params={'disks': 'all'}, + action_params={'osds': 'all'}, raise_on_failure=True) @property @@ -187,7 +187,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop=all' action on {} " "unit".format(self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT], 'stop', - action_params={'disks': 'all'}) + action_params={'osds': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='stopped') @@ -195,7 +195,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service start=all' action on {} " "unit".format(self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', - action_params={'disks': 'all'}) + action_params={'osds': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='running') @@ -209,7 +209,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop', - action_params={'disks': action_params}) + action_params={'osds': action_params}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='stopped') @@ -217,7 +217,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service start={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', - action_params={'disks': action_params}) + action_params={'osds': action_params}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='running') @@ -237,7 +237,7 @@ class ServiceTest(unittest.TestCase): "unit".format(to_stop.id, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop', - action_params={'disks': to_stop.id}) + action_params={'osds': to_stop.id}) wait_for_service(unit_name=self.TESTED_UNIT, services=[to_stop.name, ], @@ -270,7 +270,7 @@ class ServiceTest(unittest.TestCase): "unit".format(to_start.id, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', - action_params={'disks': to_start.id}) + action_params={'osds': to_start.id}) wait_for_service(unit_name=self.TESTED_UNIT, services=[to_start.name, ], From a35ba0917edf4a22bec4d579926bfbd0157d1a08 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 6 Jan 2021 11:26:01 +0000 Subject: [PATCH 763/898] Allow an app to be set when calling get_os_release (#477) Ceph deployments may not contain a keystone service which causes calls to get_os_release to fail as it calls get_current_os_release_pair without sepecifying an application (keystone is the default). --- unit_tests/utilities/test_zaza_utilities_openstack.py | 8 ++++++++ zaza/openstack/utilities/openstack.py | 8 ++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 6d91d7d..48408e0 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -908,6 +908,14 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): release_comp = xenial_queens > xenial_mitaka self.assertTrue(release_comp) + # Check specifying an application + self._get_os_rel_pair.reset_mock() + self._get_os_rel_pair.return_value = 'xenial_mitaka' + expected = 4 + result = openstack_utils.get_os_release(application='myapp') + self.assertEqual(expected, result) + self._get_os_rel_pair.assert_called_once_with(application='myapp') + def test_get_keystone_api_version(self): self.patch_object(openstack_utils, "get_current_os_versions") self.patch_object(openstack_utils, "get_application_config_option") diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 06aa14c..a93e61d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1602,15 +1602,19 @@ def get_current_os_release_pair(application='keystone'): return '{}_{}'.format(series, os_version) -def get_os_release(release_pair=None): +def get_os_release(release_pair=None, application=None): """Return index of release in OPENSTACK_RELEASES_PAIRS. + :param release_pair: OpenStack release pair eg 'focal_ussuri' + :type release_pair: string + :param application: Name of application to derive release pair from. + :type application: string :returns: Index of the release :rtype: int :raises: exceptions.ReleasePairNotFound """ if release_pair is None: - release_pair = get_current_os_release_pair() + release_pair = get_current_os_release_pair(application=application) try: index = OPENSTACK_RELEASES_PAIRS.index(release_pair) except ValueError: From ed3b2737d1bc8790ad354e22e2b6136243256960 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 6 Jan 2021 11:38:41 +0000 Subject: [PATCH 764/898] Use ceph-mon to check ceph version not keystone The test class BlueStoreCompressionCharmOperation gates tests on whether the ceph release is mimic or newer but it uses the keystone application to calculate the currently deployed version. This PR switches the test class to ceck the version of ceph-mon instead which makes more sense and the keystone application may not always present in a ceph deployment. --- zaza/openstack/charm_tests/ceph/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 349d514..107eb9f 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -881,7 +881,8 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest): def setUpClass(cls): """Perform class one time initialization.""" super(BlueStoreCompressionCharmOperation, cls).setUpClass() - cls.current_release = zaza_openstack.get_os_release() + cls.current_release = zaza_openstack.get_os_release( + application='ceph-mon') cls.bionic_rocky = zaza_openstack.get_os_release('bionic_rocky') def setUp(self): From ddd9c7402809175ac60889a021bff0da5749b8bc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 6 Jan 2021 12:12:42 +0000 Subject: [PATCH 765/898] Use keystone in the first instance --- zaza/openstack/charm_tests/ceph/tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 107eb9f..c7d44bd 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -881,8 +881,13 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest): def setUpClass(cls): """Perform class one time initialization.""" super(BlueStoreCompressionCharmOperation, cls).setUpClass() + release_application = 'keystone' + try: + zaza_model.get_application(release_application) + except KeyError: + release_application = 'ceph-mon' cls.current_release = zaza_openstack.get_os_release( - application='ceph-mon') + application=release_application) cls.bionic_rocky = zaza_openstack.get_os_release('bionic_rocky') def setUp(self): From 282b8be577b9398c1af8415d0e521ee71a1ae719 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Wed, 6 Jan 2021 13:09:41 +0000 Subject: [PATCH 766/898] Update openstack upgrade tests for focal (#476) This patch modifies the existing openstack upgrade tests so that they work with focal (by explicitly supporting mysql-innodb-cluster), and are also interruptable and resumable (at a charm level). It also makes them work with the udpated 'get_upgrade_groups()' that ultimately gets a List of Tuples rather than a dictionary. --- .../test_zaza_utilities_openstack_upgrade.py | 118 +++++++++----- .../charm_tests/openstack_upgrade/__init__.py | 15 ++ .../charm_tests/openstack_upgrade/tests.py | 148 ++++++++++++++++++ zaza/openstack/charm_tests/vault/setup.py | 3 +- zaza/openstack/utilities/openstack.py | 33 ++-- zaza/openstack/utilities/openstack_upgrade.py | 136 +++++++++++----- zaza/openstack/utilities/upgrade_utils.py | 119 +++++++++++++- 7 files changed, 483 insertions(+), 89 deletions(-) create mode 100644 zaza/openstack/charm_tests/openstack_upgrade/__init__.py create mode 100644 zaza/openstack/charm_tests/openstack_upgrade/tests.py diff --git a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py index 4d619c5..3782926 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py @@ -44,25 +44,37 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.patch_object( openstack_upgrade.zaza.model, "get_application_config") + self.patch_object( + openstack_upgrade.zaza.model, + "block_until_all_units_idle") + self.patch_object( + openstack_upgrade, + "block_until_mysql_innodb_cluster_has_rw") def _get_application_config(app, model_name=None): app_config = { - 'ceph-mon': {'verbose': True, 'source': 'old-src'}, - 'neutron-openvswitch': {'verbose': True}, - 'ntp': {'verbose': True}, - 'percona-cluster': {'verbose': True, 'source': 'old-src'}, + 'ceph-mon': {'verbose': {'value': True}, + 'source': {'value': 'old-src'}}, + 'neutron-openvswitch': {'verbose': {'value': True}}, + 'ntp': {'verbose': {'value': True}}, + 'percona-cluster': {'verbose': {'value': True}, + 'source': {'value': 'old-src'}}, 'cinder': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, 'neutron-api': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, 'nova-compute': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, + 'mysql-innodb-cluster': { + 'verbose': {'value': True}, + 'source': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': True}}, } return app_config[app] self.get_application_config.side_effect = _get_application_config @@ -74,6 +86,10 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): 'subordinate-to': 'nova-compute'}, 'ntp': { # Filter as it has no source option 'charm': 'cs:ntp'}, + 'mysql-innodb-cluster': { + 'charm': 'cs:mysql-innodb-cluster', + 'units': { + 'mysql-innodb-cluster/0': {}}}, 'nova-compute': { 'charm': 'cs:nova-compute', 'units': { @@ -115,7 +131,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): model_name=None, raise_on_failure=True) - def test_action_upgrade_group(self): + def test_action_upgrade_apps(self): self.patch_object(openstack_upgrade, "pause_units") self.patch_object(openstack_upgrade, "action_unit_upgrade") self.patch_object(openstack_upgrade, "resume_units") @@ -127,7 +143,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): 'nova-compute': [mock_nova_compute_0], 'cinder': [mock_cinder_1]} self.get_units.side_effect = lambda app, model_name: units[app] - openstack_upgrade.action_upgrade_group(['nova-compute', 'cinder']) + openstack_upgrade.action_upgrade_apps(['nova-compute', 'cinder']) pause_calls = [ mock.call(['cinder-hacluster/0'], model_name=None), mock.call(['nova-compute/0', 'cinder/1'], model_name=None)] @@ -142,6 +158,30 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): mock.call(['cinder-hacluster/0'], model_name=None)] self.resume_units.assert_has_calls(resume_calls, any_order=False) + def test_action_upgrade_apps_mysql_innodb_cluster(self): + """Verify that mysql-innodb-cluster is settled before complete.""" + self.patch_object(openstack_upgrade, "pause_units") + self.patch_object(openstack_upgrade, "action_unit_upgrade") + self.patch_object(openstack_upgrade, "resume_units") + mock_mysql_innodb_cluster_0 = mock.MagicMock() + mock_mysql_innodb_cluster_0.entity_id = 'mysql-innodb-cluster/0' + units = {'mysql-innodb-cluster': [mock_mysql_innodb_cluster_0]} + self.get_units.side_effect = lambda app, model_name: units[app] + openstack_upgrade.action_upgrade_apps(['mysql-innodb-cluster']) + pause_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.pause_units.assert_has_calls(pause_calls, any_order=False) + action_unit_upgrade_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.action_unit_upgrade.assert_has_calls( + action_unit_upgrade_calls, + any_order=False) + resume_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.resume_units.assert_has_calls(resume_calls, any_order=False) + self.block_until_mysql_innodb_cluster_has_rw.assert_called_once_with( + None) + def test_set_upgrade_application_config(self): openstack_upgrade.set_upgrade_application_config( ['neutron-api', 'cinder'], @@ -177,17 +217,23 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.assertFalse( openstack_upgrade.is_action_upgradable('percona-cluster')) + def test_is_already_upgraded(self): + self.assertTrue( + openstack_upgrade.is_already_upgraded('cinder', 'old-src')) + self.assertFalse( + openstack_upgrade.is_already_upgraded('cinder', 'new-src')) + def test_run_action_upgrade(self): self.patch_object(openstack_upgrade, "set_upgrade_application_config") - self.patch_object(openstack_upgrade, "action_upgrade_group") - openstack_upgrade.run_action_upgrade( + self.patch_object(openstack_upgrade, "action_upgrade_apps") + openstack_upgrade.run_action_upgrades( ['cinder', 'neutron-api'], 'new-src') self.set_upgrade_application_config.assert_called_once_with( ['cinder', 'neutron-api'], 'new-src', model_name=None) - self.action_upgrade_group.assert_called_once_with( + self.action_upgrade_apps.assert_called_once_with( ['cinder', 'neutron-api'], model_name=None) @@ -196,7 +242,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.patch_object( openstack_upgrade.zaza.model, 'block_until_all_units_idle') - openstack_upgrade.run_all_in_one_upgrade( + openstack_upgrade.run_all_in_one_upgrades( ['percona-cluster'], 'new-src') self.set_upgrade_application_config.assert_called_once_with( @@ -207,34 +253,36 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.block_until_all_units_idle.assert_called_once_with() def test_run_upgrade(self): - self.patch_object(openstack_upgrade, "run_all_in_one_upgrade") - self.patch_object(openstack_upgrade, "run_action_upgrade") - openstack_upgrade.run_upgrade( + self.patch_object(openstack_upgrade, "run_all_in_one_upgrades") + self.patch_object(openstack_upgrade, "run_action_upgrades") + openstack_upgrade.run_upgrade_on_apps( ['cinder', 'neutron-api', 'ceph-mon'], 'new-src') - self.run_all_in_one_upgrade.assert_called_once_with( + self.run_all_in_one_upgrades.assert_called_once_with( ['ceph-mon'], 'new-src', model_name=None) - self.run_action_upgrade.assert_called_once_with( + self.run_action_upgrades.assert_called_once_with( ['cinder', 'neutron-api'], 'new-src', model_name=None) def test_run_upgrade_tests(self): - self.patch_object(openstack_upgrade, "run_upgrade") + self.patch_object(openstack_upgrade, "run_upgrade_on_apps") self.patch_object(openstack_upgrade, "get_upgrade_groups") - self.get_upgrade_groups.return_value = { - 'Compute': ['nova-compute'], - 'Control Plane': ['cinder', 'neutron-api'], - 'Core Identity': ['keystone'], - 'Storage': ['ceph-mon'], - 'sweep_up': ['designate']} + self.get_upgrade_groups.return_value = [ + ('Compute', ['nova-compute']), + ('Control Plane', ['cinder', 'neutron-api']), + ('Core Identity', ['keystone']), + ('Storage', ['ceph-mon']), + ('sweep_up', ['designate'])] openstack_upgrade.run_upgrade_tests('new-src', model_name=None) run_upgrade_calls = [ + mock.call(['nova-compute'], 'new-src', model_name=None), + mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None), mock.call(['keystone'], 'new-src', model_name=None), mock.call(['ceph-mon'], 'new-src', model_name=None), - mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None), - mock.call(['nova-compute'], 'new-src', model_name=None), - mock.call(['designate'], 'new-src', model_name=None)] - self.run_upgrade.assert_has_calls(run_upgrade_calls, any_order=False) + mock.call(['designate'], 'new-src', model_name=None), + ] + self.run_upgrade_on_apps.assert_has_calls( + run_upgrade_calls, any_order=False) diff --git a/zaza/openstack/charm_tests/openstack_upgrade/__init__.py b/zaza/openstack/charm_tests/openstack_upgrade/__init__.py new file mode 100644 index 0000000..9a1ca53 --- /dev/null +++ b/zaza/openstack/charm_tests/openstack_upgrade/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for testing openstack upgrades.""" diff --git a/zaza/openstack/charm_tests/openstack_upgrade/tests.py b/zaza/openstack/charm_tests/openstack_upgrade/tests.py new file mode 100644 index 0000000..82c7fbb --- /dev/null +++ b/zaza/openstack/charm_tests/openstack_upgrade/tests.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define class for OpenStack Upgrade.""" + +import logging +import unittest + +from zaza.openstack.utilities import ( + cli as cli_utils, + upgrade_utils as upgrade_utils, + openstack as openstack_utils, + openstack_upgrade as openstack_upgrade, +) +from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest + + +class OpenStackUpgradeVMLaunchBase(object): + """A base class to peform a simple validation on the cloud. + + This wraps an OpenStack upgrade with a VM launch before and after the + upgrade. + + This test requires a full OpenStack including at least: keystone, glance, + nova-cloud-controller, nova-compute, neutron-gateway, neutron-api and + neutron-openvswitch. + + This class should be used as a base class to the upgrade 'test'. + """ + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenStackUpgradeMixin setUpClass") + super().setUpClass() + cls.lts = LTSGuestCreateTest() + cls.lts.setUpClass() + + def test_100_validate_pre_openstack_upgrade_cloud(self): + """Validate pre openstack upgrade.""" + logging.info("Validate pre-openstack-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + def test_500_validate_openstack_upgraded_cloud(self): + """Validate post openstack upgrade.""" + logging.info("Validate post-openstack-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + +class WaitForMySQL(unittest.TestCase): + """Helper test to wait on mysql-innodb-cluster to be fully ready. + + In practice this means that there is at least on R/W unit available. + Sometimes, after restarting units in the mysql-innodb-cluster, all the + units are R/O until the cluster picks the R/W unit. + """ + + @classmethod + def setUpClass(cls): + """Set up class.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_100_wait_for_happy_mysql_innodb_cluster(self): + """Wait for mysql cluster to have at least one R/W node.""" + logging.info("Starting wait for an R/W unit.") + openstack_upgrade.block_until_mysql_innodb_cluster_has_rw() + logging.info("Done .. all seems well.") + + +class OpenStackUpgradeTestsFocalUssuri(OpenStackUpgradeVMLaunchBase): + """Upgrade OpenStack from distro -> cloud:focal-victoria.""" + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_200_run_openstack_upgrade(self): + """Run openstack upgrade, but work out what to do.""" + openstack_upgrade.run_upgrade_tests("cloud:focal-victoria") + + +class OpenStackUpgradeTests(OpenStackUpgradeVMLaunchBase): + """A Principal Class to encapsulate OpenStack Upgrade Tests. + + A generic Test class that can discover which Ubuntu version and OpenStack + version to upgrade from. + + TODO: Not used at present. Use the declarative tests directly that choose + the version to upgrade to. The functions that this class depends on need a + bit more work regarding how the determine which version to go to. + """ + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_200_run_openstack_upgrade(self): + """Run openstack upgrade, but work out what to do. + + TODO: This is really inefficient at the moment, and doesn't (yet) + determine which ubuntu version to work from. Don't use until we can + make it better. + """ + # TODO: work out the most recent Ubuntu version; we assume this is the + # version that OpenStack is running on. + ubuntu_version = "focal" + logging.info("Getting all principle applications ...") + principle_services = upgrade_utils.get_all_principal_applications() + logging.info( + "Getting OpenStack vesions from principal applications ...") + current_versions = openstack_utils.get_current_os_versions( + principle_services) + logging.info("current versions: %s" % current_versions) + # Find the lowest value openstack release across all services and make + # sure all servcies are upgraded to one release higher than the lowest + from_version = upgrade_utils.get_lowest_openstack_version( + current_versions) + logging.info("from version: %s" % from_version) + to_version = upgrade_utils.determine_next_openstack_release( + from_version)[1] + logging.info("to version: %s" % to_version) + # TODO: need to determine the ubuntu base verion that is being upgraded + target_source = upgrade_utils.determine_new_source( + ubuntu_version, from_version, to_version, single_increment=True) + logging.info("target source: %s" % target_source) + assert target_source is not None + openstack_upgrade.run_upgrade_tests(target_source) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index adfb92d..1e4e05e 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -174,7 +174,8 @@ def auto_initialize(cacert=None, validation_application='keystone', wait=True): zaza.model.wait_for_agent_status() test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( - states=test_config.get('target_deploy_status', {})) + states=test_config.get('target_deploy_status', {}), + timeout=7200) if validation_application: validate_ca(cacertificate, application=validation_application) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index a93e61d..3dd5361 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -16,6 +16,23 @@ This module contains a number of functions for interacting with OpenStack. """ +import datetime +import io +import itertools +import juju_wait +import logging +import os +import paramiko +import re +import six +import subprocess +import sys +import tempfile +import tenacity +import textwrap +import urllib + + from .os_versions import ( OPENSTACK_CODENAMES, SWIFT_CODENAMES, @@ -48,21 +65,6 @@ from neutronclient.common import exceptions as neutronexceptions from octaviaclient.api.v2 import octavia as octaviaclient from swiftclient import client as swiftclient -import datetime -import io -import itertools -import juju_wait -import logging -import os -import paramiko -import re -import six -import subprocess -import sys -import tempfile -import tenacity -import textwrap -import urllib import zaza @@ -1554,6 +1556,7 @@ def get_current_os_versions(deployed_applications, model_name=None): for application in UPGRADE_SERVICES: if application['name'] not in deployed_applications: continue + logging.info("looking at application: {}".format(application)) version = generic_utils.get_pkg_version(application['name'], application['type']['pkg'], diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py index 9ad4d50..c279e58 100755 --- a/zaza/openstack/utilities/openstack_upgrade.py +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -95,8 +95,8 @@ async def async_action_unit_upgrade(units, model_name=None): action_unit_upgrade = sync_wrapper(async_action_unit_upgrade) -def action_upgrade_group(applications, model_name=None): - """Upgrade units using action managed upgrades. +def action_upgrade_apps(applications, model_name=None): + """Upgrade units in the applications using action managed upgrades. Upgrade all units of the given applications using action managed upgrades. This involves the following process: @@ -142,6 +142,45 @@ def action_upgrade_group(applications, model_name=None): done.extend(target) + # Ensure that mysql-innodb-cluster has at least one R/W group (it can get + # into a state where all are R/O whilst it is sorting itself out after an + # openstack_upgrade + if "mysql-innodb-cluster" in applications: + block_until_mysql_innodb_cluster_has_rw(model_name) + + # Now we need to wait for the model to go back to idle. + zaza.model.block_until_all_units_idle(model_name) + + +async def async_block_until_mysql_innodb_cluster_has_rw(model=None, + timeout=None): + """Block until the mysql-innodb-cluster is in a healthy state. + + Curiously, after a series of pauses and restarts (e.g. during an upgrade) + the mysql-innodb-cluster charms may not yet have agreed which one is the + R/W node; i.e. they are all R/O. Anyway, eventually they sort it out and + one jumps to the front and says "it's me!". This is detected, externally, + by the status line including R/W in the output. + + This function blocks until that happens so that no charm attempts to have a + chat with the mysql server before it has settled, thus breaking the whole + test. + """ + async def async_check_workload_messages_for_rw(model=None): + """Return True if a least one work message contains R/W.""" + status = await zaza.model.async_get_status() + app_status = status.applications.get("mysql-innodb-cluster") + units_data = app_status.units.values() + workload_statuses = [d.workload_status.info for d in units_data] + return any("R/W" in s for s in workload_statuses) + + await zaza.model.async_block_until(async_check_workload_messages_for_rw, + timeout=timeout) + + +block_until_mysql_innodb_cluster_has_rw = sync_wrapper( + async_block_until_mysql_innodb_cluster_has_rw) + def set_upgrade_application_config(applications, new_source, action_managed=True, model_name=None): @@ -150,7 +189,7 @@ def set_upgrade_application_config(applications, new_source, Set the charm config for upgrade. :param applications: List of application names. - :type applications: [] + :type applications: List[str] :param new_source: New package origin. :type new_source: str :param action_managed: Whether to set action-managed-upgrade config option. @@ -180,8 +219,8 @@ def set_upgrade_application_config(applications, new_source, def is_action_upgradable(app, model_name=None): """Can application be upgraded using action managed upgrade method. - :param new_source: New package origin. - :type new_source: str + :param app: The application to check + :type app: str :param model_name: Name of model to query. :type model_name: str :returns: Whether app be upgraded using action managed upgrade method. @@ -196,66 +235,95 @@ def is_action_upgradable(app, model_name=None): return supported -def run_action_upgrade(group, new_source, model_name=None): +def is_already_upgraded(app, new_src, model_name=None): + """Return True if the app has already been upgraded. + + :param app: The application to check + :type app: str + :param new_src: the new source (distro, cloud:x-y, etc.) + :type new_src: str + :param model_name: Name of model to query. + :type model_name: str + :returns: Whether app be upgraded using action managed upgrade method. + :rtype: bool + """ + config = zaza.model.get_application_config(app, model_name=model_name) + try: + src = config['openstack-origin']['value'] + key_was = 'openstack-origin' + except KeyError: + src = config['source']['value'] + key_was = 'source' + logging.info("origin for {} is {}={}".format(app, key_was, src)) + return src == new_src + + +def run_action_upgrades(apps, new_source, model_name=None): """Upgrade payload of all applications in group using action upgrades. - :param group: List of applications to upgrade. - :type group + :param apps: List of applications to upgrade. + :type apps: List[str] :param new_source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ - set_upgrade_application_config(group, new_source, model_name=model_name) - action_upgrade_group(group, model_name=model_name) + set_upgrade_application_config(apps, new_source, model_name=model_name) + action_upgrade_apps(apps, model_name=model_name) -def run_all_in_one_upgrade(group, new_source, model_name=None): +def run_all_in_one_upgrades(apps, new_source, model_name=None): """Upgrade payload of all applications in group using all-in-one method. - :param group: List of applications to upgrade. - :type group: [] + :param apps: List of applications to upgrade. + :type apps: List[str] :source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ set_upgrade_application_config( - group, + apps, new_source, model_name=model_name, action_managed=False) zaza.model.block_until_all_units_idle() -def run_upgrade(group, new_source, model_name=None): +def run_upgrade_on_apps(apps, new_source, model_name=None): """Upgrade payload of all applications in group. Upgrade apps using action managed upgrades where possible and fallback to all_in_one method. - :param group: List of applications to upgrade. - :type group: [] + :param apps: List of applications to upgrade. + :type apps: [] :param new_source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ - action_upgrade = [] - all_in_one_upgrade = [] - for app in group: + action_upgrades = [] + all_in_one_upgrades = [] + for app in apps: + if is_already_upgraded(app, new_source, model_name=model_name): + logging.info("Application '%s' is already upgraded. Skipping.", + app) + continue if is_action_upgradable(app, model_name=model_name): - action_upgrade.append(app) + action_upgrades.append(app) else: - all_in_one_upgrade.append(app) - run_all_in_one_upgrade( - all_in_one_upgrade, - new_source, - model_name=model_name) - run_action_upgrade( - action_upgrade, - new_source, - model_name=model_name) + all_in_one_upgrades.append(app) + if all_in_one_upgrades: + run_all_in_one_upgrades( + all_in_one_upgrades, + new_source, + model_name=model_name) + if action_upgrades: + run_action_upgrades( + action_upgrades, + new_source, + model_name=model_name) def run_upgrade_tests(new_source, model_name=None): @@ -270,8 +338,6 @@ def run_upgrade_tests(new_source, model_name=None): :type model_name: str """ groups = get_upgrade_groups(model_name=model_name) - run_upgrade(groups['Core Identity'], new_source, model_name=model_name) - run_upgrade(groups['Storage'], new_source, model_name=model_name) - run_upgrade(groups['Control Plane'], new_source, model_name=model_name) - run_upgrade(groups['Compute'], new_source, model_name=model_name) - run_upgrade(groups['sweep_up'], new_source, model_name=model_name) + for name, apps in groups: + logging.info("Performing upgrade of %s", name) + run_upgrade_on_apps(apps, new_source, model_name=model_name) diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 995e0bb..8e3a063 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -19,6 +19,12 @@ import logging import re import zaza.model +import zaza.utilities.juju +from zaza.openstack.utilities.os_versions import ( + OPENSTACK_CODENAMES, + UBUNTU_OPENSTACK_RELEASE, + OPENSTACK_RELEASES_PAIRS, +) SERVICE_GROUPS = ( @@ -46,7 +52,7 @@ def get_upgrade_candidates(model_name=None, filters=None): :param filters: List of filter functions to apply :type filters: List[fn] :returns: List of application that can have their payload upgraded. - :rtype: [] + :rtype: Dict[str, Dict[str, ANY]] """ if filters is None: filters = [] @@ -163,8 +169,8 @@ def get_series_upgrade_groups(model_name=None, extra_filters=None): :param model_name: Name of model to query. :type model_name: str - :returns: Dict of group lists keyed on group name. - :rtype: collections.OrderedDict + :returns: List of tuples(group name, applications) + :rtype: List[Tuple[str, Dict[str, ANY]]] """ filters = [_filter_subordinates] filters = _apply_extra_filters(filters, extra_filters) @@ -226,3 +232,110 @@ def extract_charm_name_from_url(charm_url): """ charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) return charm_name.split(':')[-1] + + +def get_all_principal_applications(model_name=None): + """Return a list of all the prinical applications in the model. + + :param model_name: Optional model name + :type model_name: Optional[str] + :returns: List of principal application names + :rtype: List[str] + """ + status = zaza.utilities.juju.get_full_juju_status(model_name=model_name) + return [application for application in status.applications.keys() + if not status.applications.get(application)['subordinate-to']] + + +def get_lowest_openstack_version(current_versions): + """Get the lowest OpenStack version from the list of current versions. + + :param current_versions: The list of versions + :type current_versions: List[str] + :returns: the lowest version currently installed. + :rtype: str + """ + lowest_version = 'zebra' + for svc in current_versions.keys(): + if current_versions[svc] < lowest_version: + lowest_version = current_versions[svc] + return lowest_version + + +def determine_next_openstack_release(release): + """Determine the next release after the one passed as a str. + + The returned value is a tuple of the form: ('2020.1', 'ussuri') + + :param release: the release to use as the base + :type release: str + :returns: the release tuple immediately after the current one. + :rtype: Tuple[str, str] + :raises: KeyError if the current release doesn't actually exist + """ + old_index = list(OPENSTACK_CODENAMES.values()).index(release) + new_index = old_index + 1 + return list(OPENSTACK_CODENAMES.items())[new_index] + + +def determine_new_source(ubuntu_version, current_source, new_release, + single_increment=True): + """Determine the new source/openstack-origin value based on new release. + + This takes the ubuntu_version and the current_source (in the form of + 'distro' or 'cloud:xenial-mitaka') and either converts it to a new source, + or returns None if the new_release will match the current_source (i.e. it's + already at the right release), or it's simply not possible. + + If single_increment is set, then the returned source will only be returned + if the new_release is one more than the release in the current source. + + :param ubuntu_version: the ubuntu version that the app is installed on. + :type ubuntu_version: str + :param current_source: a source in the form of 'distro' or + 'cloud:xenial-mitaka' + :type current_source: str + :param new_release: a new OpenStack version codename. e.g. 'stein' + :type new_release: str + :param single_increment: If True, only allow single increment upgrade. + :type single_increment: boolean + :returns: The new source in the form of 'cloud:bionic-train' or None if not + possible + :rtype: Optional[str] + :raises: KeyError if any of the strings don't correspond to known values. + """ + logging.warn("determine_new_source: locals: %s", locals()) + if current_source == 'distro': + # convert to a ubuntu-openstack pair + current_source = "cloud:{}-{}".format( + ubuntu_version, UBUNTU_OPENSTACK_RELEASE[ubuntu_version]) + # strip out the current openstack version + if ':' not in current_source: + current_source = "cloud:{}-{}".format(ubuntu_version, current_source) + pair = current_source.split(':')[1] + u_version, os_version = pair.split('-', 2) + if u_version != ubuntu_version: + logging.warn("determine_new_source: ubuntu_versions don't match: " + "%s != %s" % (ubuntu_version, u_version)) + return None + # determine versions + openstack_codenames = list(OPENSTACK_CODENAMES.values()) + old_index = openstack_codenames.index(os_version) + try: + new_os_version = openstack_codenames[old_index + 1] + except IndexError: + logging.warn("determine_new_source: no OpenStack version after " + "'%s'" % os_version) + return None + if single_increment and new_release != new_os_version: + logging.warn("determine_new_source: requested version '%s' not a " + "single increment from '%s' which is '%s'" % ( + new_release, os_version, new_os_version)) + return None + # now check that there is a combination of u_version-new_os_version + new_pair = "{}_{}".format(u_version, new_os_version) + if new_pair not in OPENSTACK_RELEASES_PAIRS: + logging.warn("determine_new_source: now release pair candidate for " + " combination cloud:%s-%s" % (u_version, new_os_version)) + return None + return "cloud:{}-{}".format(u_version, new_os_version) From ebc51b490e92ed3f8d58e5af6a4c9606e52a5cf6 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 6 Jan 2021 17:19:56 +0100 Subject: [PATCH 767/898] Make the get_os_release default match get_current_os_release_pair --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 3dd5361..33f0e02 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1605,7 +1605,7 @@ def get_current_os_release_pair(application='keystone'): return '{}_{}'.format(series, os_version) -def get_os_release(release_pair=None, application=None): +def get_os_release(release_pair=None, application='keystone'): """Return index of release in OPENSTACK_RELEASES_PAIRS. :param release_pair: OpenStack release pair eg 'focal_ussuri' From cdba5b64ca2d62d812f328be85aca3ad15f5f9df Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 12 Jan 2021 13:11:17 +0000 Subject: [PATCH 768/898] Run update-status before manila api test The manila charm contains a 'band-aid' for Bug #1706699 which relies on update-status to bring up services if needed. When the tests run an update-status hook might not have run so services may still be stopped so force a hook execution. --- zaza/openstack/charm_tests/manila/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index 8a1fdee..c00b993 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -20,6 +20,7 @@ import tenacity from manilaclient import client as manilaclient +import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils @@ -35,6 +36,12 @@ class ManilaTests(test_utils.OpenStackBaseTest): def test_manila_api(self): """Test that the Manila API is working.""" + # The manila charm contains a 'band-aid' for Bug #1706699 which relies + # on update-status to bring up services if needed. When the tests run + # an update-status hook might not have run so services may still be + # stopped so force a hook execution. + for unit in zaza.model.get_units('manila'): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") self.assertEqual([], self._list_shares()) @tenacity.retry( From 9961c9468bac26dfe6675ada46fdd137316a5b4f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 13 Jan 2021 09:42:34 +0100 Subject: [PATCH 769/898] Sync pertinent tox.ini settings from release-tools The new PIP resolver wreaks havoc for this repository. Long term we should split unit test requirements into a separate file and whip unit tests into shape wrt. mocking out everything as opposed to relying on having random modules installed in the test environment. --- tox.ini | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tox.ini b/tox.ini index 7d67e9f..dc09948 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,22 @@ [tox] envlist = pep8, py3 skipsdist = True +# NOTE: Avoid build/test env pollution by not enabling sitepackages. +sitepackages = False +# NOTE: Avoid false positives by not skipping missing interpreters. +skip_missing_interpreters = False +# NOTES: +# * We avoid the new dependency resolver by pinning pip < 20.3, see +# https://github.com/pypa/pip/issues/9187 +# * Pinning dependencies requires tox >= 3.2.0, see +# https://tox.readthedocs.io/en/latest/config.html#conf-requires +# * It is also necessary to pin virtualenv as a newer virtualenv would still +# lead to fetching the latest pip in the func* tox targets, see +# https://stackoverflow.com/a/38133283 +requires = pip < 20.3 + virtualenv < 20.0 +# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci +minversion = 3.2.0 [testenv] setenv = VIRTUAL_ENV={envdir} From ee0dd80cec01f9bf8649851ba4c052f92cbee463 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 12 Jan 2021 17:11:35 +0100 Subject: [PATCH 770/898] Split configure_gateway_ext_port function The function does three separate things today, and two of its tasks are useful for other provider types such as MAAS. Also fix create_additional_port_for_machines idempotency. We previously added a run time assertion to fail early when attempting to configure networking for an invalid bundle. The check had the side effect of prohibiting subsequent runs on already configured models. --- .../test_zaza_utilities_openstack.py | 78 +++-- zaza/openstack/utilities/openstack.py | 327 ++++++++++++------ 2 files changed, 274 insertions(+), 131 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 48408e0..7e0e8f1 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1262,34 +1262,68 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.get_application.side_effect = KeyError self.assertFalse(openstack_utils.ngw_present()) - def test_configure_gateway_ext_port(self): - # FIXME: this is not a complete unit test for the function as one did - # not exist at all I'm adding this to test one bit and we'll add more - # as we go. + def test_get_charm_networking_data(self): self.patch_object(openstack_utils, 'deprecated_external_networking') self.patch_object(openstack_utils, 'dvr_enabled') self.patch_object(openstack_utils, 'ovn_present') self.patch_object(openstack_utils, 'ngw_present') + self.patch_object(openstack_utils, 'get_ovs_uuids') self.patch_object(openstack_utils, 'get_gateway_uuids') - self.patch_object(openstack_utils, 'get_admin_net') + self.patch_object(openstack_utils, 'get_ovn_uuids') + self.patch_object(openstack_utils.model, 'get_application') self.dvr_enabled.return_value = False self.ovn_present.return_value = False - self.ngw_present.return_value = True - self.get_admin_net.return_value = {'id': 'fakeid'} + self.ngw_present.return_value = False + self.get_ovs_uuids.return_value = [] + self.get_gateway_uuids.return_value = [] + self.get_ovn_uuids.return_value = [] + self.get_application.side_effect = KeyError - novaclient = mock.MagicMock() - neutronclient = mock.MagicMock() - - def _fake_empty_generator(empty=True): - if empty: - return - yield - - self.get_gateway_uuids.side_effect = _fake_empty_generator with self.assertRaises(RuntimeError): - openstack_utils.configure_gateway_ext_port( - novaclient, neutronclient) - # provide a uuid and check that we don't raise RuntimeError - self.get_gateway_uuids.side_effect = ['fake-uuid'] - openstack_utils.configure_gateway_ext_port( - novaclient, neutronclient) + openstack_utils.get_charm_networking_data() + self.ngw_present.return_value = True + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVS, + ['neutron-gateway'], + mock.ANY, + 'data-port', + {})) + self.dvr_enabled.return_value = True + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVS_DVR, + ['neutron-gateway', 'neutron-openvswitch'], + mock.ANY, + 'data-port', + {})) + self.ngw_present.return_value = False + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT, + ['neutron-openvswitch'], + mock.ANY, + 'data-port', + {})) + self.dvr_enabled.return_value = False + self.ovn_present.return_value = True + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVN, + ['ovn-chassis'], + mock.ANY, + 'bridge-interface-mappings', + {'ovn-bridge-mappings': 'physnet1:br-ex'})) + self.get_application.side_effect = None + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVN, + ['ovn-chassis', 'ovn-dedicated-chassis'], + mock.ANY, + 'bridge-interface-mappings', + {'ovn-bridge-mappings': 'physnet1:br-ex'})) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 33f0e02..33b8a52 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -16,7 +16,10 @@ This module contains a number of functions for interacting with OpenStack. """ +import collections +import copy import datetime +import enum import io import itertools import juju_wait @@ -721,6 +724,211 @@ def add_interface_to_netplan(server_name, mac_address): model.run_on_unit(unit_name, "sudo netplan apply") +class OpenStackNetworkingTopology(enum.Enum): + """OpenStack Charms Network Topologies.""" + + ML2_OVS = 'ML2+OVS' + ML2_OVS_DVR = 'ML2+OVS+DVR' + ML2_OVS_DVR_SNAT = 'ML2+OVS+DVR, no dedicated GWs' + ML2_OVN = 'ML2+OVN' + + +CharmedOpenStackNetworkingData = collections.namedtuple( + 'CharmedOpenStackNetworkingData', + [ + 'topology', + 'application_names', + 'unit_machine_ids', + 'port_config_key', + 'other_config', + ]) + + +def get_charm_networking_data(limit_gws=None): + """Inspect Juju model, determine networking topology and return data. + + :param limit_gws: Limit the number of gateways that get a port attached + :type limit_gws: Optional[int] + :rtype: CharmedOpenStackNetworkingData[ + OpenStackNetworkingTopology, + List[str], + Iterator[str], + str, + Dict[str,str]] + :returns: Named Tuple with networking data, example: + CharmedOpenStackNetworkingData( + OpenStackNetworkingTopology.ML2_OVN, + ['ovn-chassis', 'ovn-dedicated-chassis'], + ['machine-id-1', 'machine-id-2'], # generator object + 'bridge-interface-mappings', + {'ovn-bridge-mappings': 'physnet1:br-ex'}) + :raises: RuntimeError + """ + # Initialize defaults, these will be amended to fit the reality of the + # model in the checks below. + topology = OpenStackNetworkingTopology.ML2_OVS + other_config = {} + port_config_key = ( + 'data-port' if not deprecated_external_networking() else 'ext-port') + unit_machine_ids = [] + application_names = [] + + if dvr_enabled(): + if ngw_present(): + application_names = ['neutron-gateway', 'neutron-openvswitch'] + topology = OpenStackNetworkingTopology.ML2_OVS_DVR + else: + application_names = ['neutron-openvswitch'] + topology = OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT + unit_machine_ids = itertools.islice( + itertools.chain( + get_ovs_uuids(), + get_gateway_uuids()), + limit_gws) + elif ngw_present(): + unit_machine_ids = itertools.islice( + get_gateway_uuids(), limit_gws) + application_names = ['neutron-gateway'] + elif ovn_present(): + topology = OpenStackNetworkingTopology.ML2_OVN + unit_machine_ids = itertools.islice(get_ovn_uuids(), limit_gws) + application_names = ['ovn-chassis'] + try: + ovn_dc_name = 'ovn-dedicated-chassis' + model.get_application(ovn_dc_name) + application_names.append(ovn_dc_name) + except KeyError: + # ovn-dedicated-chassis not in deployment + pass + port_config_key = 'bridge-interface-mappings' + other_config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) + else: + raise RuntimeError('Unable to determine charm network topology.') + + return CharmedOpenStackNetworkingData( + topology, + application_names, + unit_machine_ids, + port_config_key, + other_config) + + +def create_additional_port_for_machines(novaclient, neutronclient, net_id, + unit_machine_ids, + add_dataport_to_netplan=False): + """Create additional port for machines for use with external networking. + + :param novaclient: Undercloud Authenticated novaclient. + :type novaclient: novaclient.Client object + :param neutronclient: Undercloud Authenticated neutronclient. + :type neutronclient: neutronclient.Client object + :param net_id: Network ID to create ports on. + :type net_id: string + :param unit_machine_ids: Juju provider specific machine IDs for which we + should add ports on. + :type unit_machine_ids: Iterator[str] + :param add_dataport_to_netplan: Whether the newly created port should be + added to instance system configuration so + that it is brought up on instance reboot. + :type add_dataport_to_netplan: Optional[bool] + :returns: List of MAC addresses for created ports. + :rtype: List[str] + :raises: RuntimeError + """ + eligible_machines = 0 + for uuid in unit_machine_ids: + eligible_machines += 1 + server = novaclient.servers.get(uuid) + ext_port_name = "{}_ext-port".format(server.name) + for port in neutronclient.list_ports(device_id=server.id)['ports']: + if port['name'] == ext_port_name: + logging.warning( + 'Instance {} already has additional port, skipping.' + .format(server.id)) + break + else: + logging.info('Attaching additional port to instance ("{}"), ' + 'connected to net id: {}' + .format(uuid, net_id)) + body_value = { + "port": { + "admin_state_up": True, + "name": ext_port_name, + "network_id": net_id, + "port_security_enabled": False, + } + } + port = neutronclient.create_port(body=body_value) + server.interface_attach(port_id=port['port']['id'], + net_id=None, fixed_ip=None) + if add_dataport_to_netplan: + mac_address = get_mac_from_port(port, neutronclient) + add_interface_to_netplan(server.name, + mac_address=mac_address) + if not eligible_machines: + # NOTE: unit_machine_ids may be an iterator so testing it for contents + # or length prior to iterating over it is futile. + raise RuntimeError('Unable to determine UUIDs for machines to attach ' + 'external networking to.') + + # Retrieve the just created ports from Neutron so that we can provide our + # caller with their MAC addresses. + return [ + port['mac_address'] + for port in neutronclient.list_ports(network_id=net_id)['ports'] + if 'ext-port' in port['name'] + ] + + +def configure_networking_charms(networking_data, macs, use_juju_wait=True): + """Configure external networking for networking charms. + + :param networking_data: Data on networking charm topology. + :type networking_data: CharmedOpenStackNetworkingData + :param macs: MAC addresses of ports for use with external networking. + :type macs: Iterator[str] + :param use_juju_wait: Whether to use juju wait to wait for the model to + settle once the gateway has been configured. Default is True + :type use_juju_wait: Optional[bool] + """ + br_mac_fmt = 'br-ex:{}' if not deprecated_external_networking() else '{}' + br_mac = [ + br_mac_fmt.format(mac) + for mac in macs + ] + + config = copy.deepcopy(networking_data.other_config) + config.update({networking_data.port_config_key: ' '.join(sorted(br_mac))}) + + for application_name in networking_data.application_names: + logging.info('Setting {} on {}'.format( + config, application_name)) + current_data_port = get_application_config_option( + application_name, + networking_data.port_config_key) + if current_data_port == config[networking_data.port_config_key]: + logging.info('Config already set to value') + return + + model.set_application_config( + application_name, + configuration=config) + # NOTE(fnordahl): We are stuck with juju_wait until we figure out how + # to deal with all the non ['active', 'idle', 'Unit is ready.'] + # workload/agent states and msgs that our mojo specs are exposed to. + if use_juju_wait: + juju_wait.wait(wait_for_workload=True) + else: + zaza.model.wait_for_agent_status() + # TODO: shouldn't access get_charm_config() here as it relies on + # ./tests/tests.yaml existing by default (regardless of the + # fatal=False) ... it's not great design. + test_config = zaza.charm_lifecycle.utils.get_charm_config( + fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get('target_deploy_status', {})) + + def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, add_dataport_to_netplan=False, limit_gws=None, @@ -739,123 +947,24 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, settle once the gateway has been configured. Default is True :type use_juju_wait: boolean """ - deprecated_extnet_mode = deprecated_external_networking() - - port_config_key = 'data-port' - if deprecated_extnet_mode: - port_config_key = 'ext-port' - - config = {} - if dvr_enabled(): - uuids = itertools.islice(itertools.chain(get_ovs_uuids(), - get_gateway_uuids()), - limit_gws) + networking_data = get_charm_networking_data(limit_gws=limit_gws) + if networking_data.topology in ( + OpenStackNetworkingTopology.ML2_OVS_DVR, + OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT): # If dvr, do not attempt to persist nic in netplan # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78 add_dataport_to_netplan = False - application_names = ['neutron-openvswitch'] - try: - ngw = 'neutron-gateway' - model.get_application(ngw) - application_names.append(ngw) - except KeyError: - # neutron-gateway not in deployment - pass - elif ngw_present(): - uuids = itertools.islice(get_gateway_uuids(), limit_gws) - application_names = ['neutron-gateway'] - elif ovn_present(): - uuids = itertools.islice(get_ovn_uuids(), limit_gws) - application_names = ['ovn-chassis'] - try: - ovn_dc_name = 'ovn-dedicated-chassis' - model.get_application(ovn_dc_name) - application_names.append(ovn_dc_name) - except KeyError: - # ovn-dedicated-chassis not in deployment - pass - port_config_key = 'bridge-interface-mappings' - config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) - add_dataport_to_netplan = True - else: - raise RuntimeError('Unable to determine charm network topology.') if not net_id: net_id = get_admin_net(neutronclient)['id'] - ports_created = 0 - for uuid in uuids: - server = novaclient.servers.get(uuid) - ext_port_name = "{}_ext-port".format(server.name) - for port in neutronclient.list_ports(device_id=server.id)['ports']: - if port['name'] == ext_port_name: - logging.warning( - 'Neutron Gateway already has additional port') - break - else: - logging.info('Attaching additional port to instance ("{}"), ' - 'connected to net id: {}' - .format(uuid, net_id)) - body_value = { - "port": { - "admin_state_up": True, - "name": ext_port_name, - "network_id": net_id, - "port_security_enabled": False, - } - } - port = neutronclient.create_port(body=body_value) - ports_created += 1 - server.interface_attach(port_id=port['port']['id'], - net_id=None, fixed_ip=None) - if add_dataport_to_netplan: - mac_address = get_mac_from_port(port, neutronclient) - add_interface_to_netplan(server.name, - mac_address=mac_address) - if not ports_created: - # NOTE: uuids is an iterator so testing it for contents or length prior - # to iterating over it is futile. - raise RuntimeError('Unable to determine UUIDs for machines to attach ' - 'external networking to.') + macs = create_additional_port_for_machines( + novaclient, neutronclient, net_id, networking_data.unit_machine_ids, + add_dataport_to_netplan) - ext_br_macs = [] - for port in neutronclient.list_ports(network_id=net_id)['ports']: - if 'ext-port' in port['name']: - if deprecated_extnet_mode: - ext_br_macs.append(port['mac_address']) - else: - ext_br_macs.append('br-ex:{}'.format(port['mac_address'])) - ext_br_macs.sort() - ext_br_macs_str = ' '.join(ext_br_macs) - - if ext_br_macs: - config.update({port_config_key: ext_br_macs_str}) - for application_name in application_names: - logging.info('Setting {} on {}'.format( - config, application_name)) - current_data_port = get_application_config_option(application_name, - port_config_key) - if current_data_port == ext_br_macs_str: - logging.info('Config already set to value') - return - - model.set_application_config( - application_name, - configuration=config) - # NOTE(fnordahl): We are stuck with juju_wait until we figure out how - # to deal with all the non ['active', 'idle', 'Unit is ready.'] - # workload/agent states and msgs that our mojo specs are exposed to. - if use_juju_wait: - juju_wait.wait(wait_for_workload=True) - else: - zaza.model.wait_for_agent_status() - # TODO: shouldn't access get_charm_config() here as it relies on - # ./tests/tests.yaml existing by default (regardless of the - # fatal=False) ... it's not great design. - test_config = zaza.charm_lifecycle.utils.get_charm_config( - fatal=False) - zaza.model.wait_for_application_states( - states=test_config.get('target_deploy_status', {})) + if macs: + configure_networking_charms( + networking_data, macs, use_juju_wait=use_juju_wait) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), From ec637329740247d8384917c9dbe4c57cf6857de2 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 13 Jan 2021 12:28:39 +0100 Subject: [PATCH 771/898] Support configuring networknig charms on MAAS When on MAAS support doing charm based configuration of OVS by retrieving MAC address of ports attached to external network from MAAS. Note that we should extend the MAAS support to also work with deployments where MAAS does the OVS configuration for us. --- unit_tests/__init__.py | 5 +++++ zaza/openstack/charm_tests/neutron/setup.py | 18 ++++++++++++++-- zaza/openstack/utilities/openstack.py | 23 +++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py index 8203d13..03c4879 100644 --- a/unit_tests/__init__.py +++ b/unit_tests/__init__.py @@ -11,3 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import sys +import unittest.mock as mock + +sys.modules['zaza.utilities.maas'] = mock.MagicMock() diff --git a/zaza/openstack/charm_tests/neutron/setup.py b/zaza/openstack/charm_tests/neutron/setup.py index f65b87a..a1d1dd4 100644 --- a/zaza/openstack/charm_tests/neutron/setup.py +++ b/zaza/openstack/charm_tests/neutron/setup.py @@ -15,6 +15,7 @@ """Setup for Neutron deployments.""" import functools +import logging from zaza.openstack.configure import ( network, @@ -89,12 +90,25 @@ def basic_overcloud_network(limit_gws=None): 'configure_gateway_ext_port_use_juju_wait', True) # Handle network for OpenStack-on-OpenStack scenarios - if juju_utils.get_provider_type() == "openstack": + provider_type = juju_utils.get_provider_type() + if provider_type == "openstack": undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session() network.setup_gateway_ext_port(network_config, keystone_session=undercloud_ks_sess, - limit_gws=None, + limit_gws=limit_gws, use_juju_wait=use_juju_wait) + elif provider_type == "maas": + # NOTE(fnordahl): After validation of the MAAS+Netplan Open vSwitch + # integration support, we would most likely want to add multiple modes + # of operation with MAAS. + # + # Perform charm based OVS configuration + openstack_utils.configure_charmed_openstack_on_maas( + network_config, limit_gws=limit_gws) + else: + logging.warning('Unknown Juju provider type, "{}", will not perform' + ' charm network configuration.' + .format(provider_type)) # Confugre the overcloud network network.setup_sdn(network_config, keystone_session=keystone_session) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 33b8a52..6e0fa20 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -62,6 +62,7 @@ from keystoneauth1.identity import ( import zaza.openstack.utilities.cert as cert import zaza.utilities.deployment_env as deployment_env import zaza.utilities.juju as juju_utils +import zaza.utilities.maas from novaclient import client as novaclient_client from neutronclient.v2_0 import client as neutronclient from neutronclient.common import exceptions as neutronexceptions @@ -967,6 +968,28 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, networking_data, macs, use_juju_wait=use_juju_wait) +def configure_charmed_openstack_on_maas(network_config, limit_gws=None): + """Configure networking charms for charm-based OVS config on MAAS provider. + + :param network_config: Network configuration as provided in environment. + :type network_config: Dict[str] + :param limit_gws: Limit the number of gateways that get a port attached + :type limit_gws: Optional[int] + """ + networking_data = get_charm_networking_data(limit_gws=limit_gws) + macs = [ + mim.mac + for mim in zaza.utilities.maas.get_macs_from_cidr( + zaza.utilities.maas.get_maas_client_from_juju_cloud_data( + zaza.model.get_cloud_data()), + network_config['external_net_cidr'], + link_mode=zaza.utilities.maas.LinkMode.LINK_UP) + ] + if macs: + configure_networking_charms( + networking_data, macs, use_juju_wait=False) + + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, retry=tenacity.retry_if_exception_type(KeyError)) def get_mac_from_port(port, neutronclient): From 2b715da7e43b94ad36975063764f33738cc7aa61 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 14 Jan 2021 10:15:34 +0100 Subject: [PATCH 772/898] Make HaclusterScalebackTest more robust (#482) Note that this test will be superseded by #369 in the future. --- zaza/openstack/charm_tests/hacluster/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index af47b76..1690c85 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -125,8 +125,8 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('Waiting for model to settle') zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active') - # NOTE(lourot): the principle application remains blocked after scaling - # back up until lp:1400481 is solved. - zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + # NOTE(lourot): the principle application sometimes remain blocked + # after scaling back up until lp:1400481 is solved. + # zaza.model.block_until_unit_wl_status(other_principle_unit, 'active') zaza.model.block_until_all_units_idle() logging.debug('OK') From 35840a66d6788dc4899847747ee7523895f46e1b Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sat, 16 Jan 2021 15:28:13 +0100 Subject: [PATCH 773/898] nova: Conditional security checklist based on presence of vault The current test expects Nova to never have TLS connections, let's expect them to be there whenever vault is present. Remove the 'is-volume-encryption-enabled' assertion as it is not a property of the Nova security checks. This was previously masked by the fact that action would always fail due to TLS tests not being enabled for any bundles. --- zaza/openstack/charm_tests/nova/tests.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index b8e7552..edad683 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -495,17 +495,24 @@ class SecurityTests(test_utils.OpenStackBaseTest): # Changes fixing the below expected failures will be made following # this initial work to get validation in. There will be bugs targeted # to each one and resolved independently where possible. - expected_failures = [ - 'is-volume-encryption-enabled', - 'validate-uses-tls-for-glance', - 'validate-uses-tls-for-keystone', ] expected_passes = [ 'validate-file-ownership', 'validate-file-permissions', 'validate-uses-keystone', ] + tls_checks = [ + 'validate-uses-tls-for-glance', + 'validate-uses-tls-for-keystone', + ] + if zaza.model.get_relation_id( + 'nova-cloud-controller', + 'vault', + remote_interface_name='certificates'): + expected_passes.extend(tls_checks) + else: + expected_failures.extend(tls_checks) for unit in zaza.model.get_units(self.application_name, model_name=self.model_name): @@ -519,4 +526,4 @@ class SecurityTests(test_utils.OpenStackBaseTest): action_params={}), expected_passes, expected_failures, - expected_to_pass=False) + expected_to_pass=not len(expected_failures)) From 44adc1fb944403d7d19a7fc2772c910fa8213399 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 19 Jan 2021 10:42:20 +0000 Subject: [PATCH 774/898] Fix policyd call to _login 8681b023 changed the signature of _login but did not update the policyd test. This was not immediately picked up because the policyd test is skipped before groovy due to Bug #1880959 Closes-Bug: #1911923 --- .../charm_tests/openstack_dashboard/tests.py | 102 ++++++++++-------- 1 file changed, 58 insertions(+), 44 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index d3fc625..9588c6f 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -67,7 +67,6 @@ def _login(dashboard_url, domain, username, password, cafile=None): # start session, get csrftoken client = requests.session() client.get(auth_url, verify=cafile) - if 'csrftoken' in client.cookies: csrftoken = client.cookies['csrftoken'] else: @@ -163,7 +162,60 @@ def _do_request(request, cafile=None): return urllib.request.urlopen(request, cafile=cafile) -class OpenStackDashboardTests(test_utils.OpenStackBaseTest): +class OpenStackDashboardBase(): + """Mixin for interacting with Horizon.""" + + def get_base_url(self): + """Return the base url for http(s) requests. + + :returns: URL + :rtype: str + """ + vip = (zaza_model.get_application_config(self.application_name) + .get("vip").get("value")) + if vip: + ip = vip + else: + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name(self.application_name)) + ip = unit.public_address + + logging.debug("Dashboard ip is:{}".format(ip)) + scheme = 'http' + if self.use_https: + scheme = 'https' + url = '{}://{}'.format(scheme, ip) + return url + + def get_horizon_url(self): + """Return the url for acccessing horizon. + + :returns: Horizon URL + :rtype: str + """ + url = '{}/horizon'.format(self.get_base_url()) + logging.info("Horizon URL is: {}".format(url)) + return url + + @property + def use_https(self): + """Whether dashboard is using https. + + :returns: Whether dashboard is using https + :rtype: boolean + """ + use_https = False + vault_relation = zaza_model.get_relation_id( + self.application, + 'vault', + remote_interface_name='certificates') + if vault_relation: + use_https = True + return use_https + + +class OpenStackDashboardTests(test_utils.OpenStackBaseTest, + OpenStackDashboardBase): """Encapsulate openstack dashboard charm tests.""" @classmethod @@ -171,13 +223,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): """Run class setup for running openstack dashboard charm tests.""" super(OpenStackDashboardTests, cls).setUpClass() cls.application = 'openstack-dashboard' - cls.use_https = False - vault_relation = zaza_model.get_relation_id( - cls.application, - 'vault', - remote_interface_name='certificates') - if vault_relation: - cls.use_https = True def test_050_local_settings_permissions_regression_check_lp1755027(self): """Assert regression check lp1755027. @@ -302,39 +347,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): mismatches.append(msg) return mismatches - def get_base_url(self): - """Return the base url for http(s) requests. - - :returns: URL - :rtype: str - """ - vip = (zaza_model.get_application_config(self.application_name) - .get("vip").get("value")) - if vip: - ip = vip - else: - unit = zaza_model.get_unit_from_name( - zaza_model.get_lead_unit_name(self.application_name)) - ip = unit.public_address - - logging.debug("Dashboard ip is:{}".format(ip)) - scheme = 'http' - if self.use_https: - scheme = 'https' - url = '{}://{}'.format(scheme, ip) - logging.debug("Base URL is: {}".format(url)) - return url - - def get_horizon_url(self): - """Return the url for acccessing horizon. - - :returns: Horizon URL - :rtype: str - """ - url = '{}/horizon'.format(self.get_base_url()) - logging.info("Horizon URL is: {}".format(url)) - return url - def test_400_connection(self): """Test that dashboard responds to http request. @@ -450,7 +462,8 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): logging.info("Testing pause resume") -class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): +class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization, + OpenStackDashboardBase): """Test the policyd override using the dashboard.""" good = { @@ -476,6 +489,7 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): super(OpenStackDashboardPolicydTests, cls).setUpClass( application_name="openstack-dashboard") cls.application_name = "openstack-dashboard" + cls.application = cls.application_name def get_client_and_attempt_operation(self, ip): """Attempt to list users on the openstack-dashboard service. @@ -500,7 +514,7 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): username = 'admin', password = overcloud_auth['OS_PASSWORD'], client, response = _login( - unit.public_address, domain, username, password) + self.get_horizon_url(), domain, username, password) # now attempt to get the domains page _url = self.url.format(unit.public_address) result = client.get(_url) From a828774c4868c6c74fc69e6b0f2a1e690a64d5d9 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sat, 23 Jan 2021 16:17:28 +0000 Subject: [PATCH 775/898] Handle change of CA cert. Closes issue #487 --- zaza/openstack/charm_tests/keystone/setup.py | 4 +- zaza/openstack/charm_tests/keystone/tests.py | 4 +- zaza/openstack/charm_tests/policyd/tests.py | 3 +- zaza/openstack/charm_tests/vault/setup.py | 4 +- zaza/openstack/charm_tests/vault/tests.py | 4 +- zaza/openstack/utilities/openstack.py | 82 +++++++++++++++++--- 6 files changed, 85 insertions(+), 16 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index 748439f..8d64993 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -41,9 +41,11 @@ def wait_for_cacert(model_name=None): :type model_name: str """ logging.info("Waiting for cacert") + cert_file = openstack_utils.get_cert_file_name( + 'keystone') zaza.model.block_until_file_has_contents( 'keystone', - openstack_utils.KEYSTONE_REMOTE_CACERT, + cert_file, 'CERTIFICATE', model_name=model_name) zaza.model.block_until_all_units_idle(model_name=model_name) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index b2830ef..477152d 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -229,7 +229,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): 'OS_DOMAIN_NAME': DEMO_DOMAIN, } if self.tls_rid: - openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_CACERT'] = openstack_utils.get_cacert() openrc['OS_AUTH_URL'] = ( openrc['OS_AUTH_URL'].replace('http', 'https')) logging.info('keystone IP {}'.format(ip)) @@ -259,7 +259,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): """ def _validate_token_data(openrc): if self.tls_rid: - openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_CACERT'] = openstack_utils.get_cacert() openrc['OS_AUTH_URL'] = ( openrc['OS_AUTH_URL'].replace('http', 'https')) logging.info('keystone IP {}'.format(ip)) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index eca316d..4722755 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -337,8 +337,7 @@ class BasePolicydSpecialization(PolicydTest, logging.info('Authentication for {} on keystone IP {}' .format(openrc['OS_USERNAME'], ip)) if self.tls_rid: - openrc['OS_CACERT'] = \ - openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_CACERT'] = openstack_utils.get_cacert() openrc['OS_AUTH_URL'] = ( openrc['OS_AUTH_URL'].replace('http', 'https')) logging.info('keystone IP {}'.format(ip)) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 1e4e05e..d605063 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -222,9 +222,11 @@ def validate_ca(cacertificate, application="keystone", port=5000): :returns: None :rtype: None """ + cert_file = zaza.openstack.utilities.openstack.get_cert_file_name( + application) zaza.model.block_until_file_has_contents( application, - zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT, + cert_file, cacertificate.decode().strip()) vip = (zaza.model.get_application_config(application) .get("vip").get("value")) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 40227fd..e96f36a 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -154,9 +154,11 @@ class VaultTest(BaseVaultTest): test_config = lifecycle_utils.get_charm_config() del test_config['target_deploy_status']['vault'] + cert_file = zaza.openstack.utilities.openstack.get_cert_file_name( + 'keystone') zaza.model.block_until_file_has_contents( 'keystone', - zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT, + cert_file, cacert.decode().strip()) zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 6e0fa20..b914c0f 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -69,6 +69,7 @@ from neutronclient.common import exceptions as neutronexceptions from octaviaclient.api.v2 import octavia as octaviaclient from swiftclient import client as swiftclient +from juju.errors import JujuError import zaza @@ -181,10 +182,68 @@ WORKLOAD_STATUS_EXCEPTIONS = { 'ceilometer and gnocchi')}} # For vault TLS certificates +LOCAL_CERT_DIR = "tests" KEYSTONE_CACERT = "keystone_juju_ca_cert.crt" KEYSTONE_REMOTE_CACERT = ( "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT)) -KEYSTONE_LOCAL_CACERT = ("tests/{}".format(KEYSTONE_CACERT)) +KEYSTONE_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, KEYSTONE_CACERT)) + +VAULT_CACERT = "vault_juju_ca_cert.crt" +VAULT_REMOTE_CACERT = ( + "/usr/local/share/ca-certificates/{}".format(VAULT_CACERT)) +VAULT_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, VAULT_CACERT)) + +REMOTE_CERTIFICATES = [VAULT_REMOTE_CACERT, KEYSTONE_REMOTE_CACERT] +LOCAL_CERTIFICATES = [VAULT_LOCAL_CACERT, KEYSTONE_LOCAL_CACERT] + + +async def async_get_cert_file_name(app, cert_files=None, block=True, + model_name=None, timeout=2700): + """Get the name of the CA cert file thats on all units of an application. + + :param app: Name of application + :type capp: str + :param cert_files: List of cert files to search for. + :type cert_files: List[str] + :param block: Whether to block until a consistent cert file is found. + :type block: bool + :param model_name: Name of model to run check in + :type model_name: str + :param timeout: Time to wait for consistent file + :type timeout: int + :returns: Credentials dictionary + :rtype: dict + """ + async def _check_for_file(model, cert_files): + units = model.applications[app].units + results = {u.entity_id: [] for u in units} + for unit in units: + try: + for cf in cert_files: + output = await unit.run('test -e "{}"; echo $?'.format(cf)) + contents = output.data.get('results')['Stdout'] + if "0" in contents: + results[unit.entity_id].append(cf) + except JujuError: + pass + for cert_file in cert_files: + # Check that the certificate file exists on all the units. + if all(cert_file in files for files in results.values()): + return cert_file + else: + return None + + if not cert_files: + cert_files = REMOTE_CERTIFICATES + cert_file = None + async with zaza.model.run_in_model(model_name) as model: + if block: + await zaza.model.async_block_until( + lambda: _check_for_file(model, cert_files), timeout=timeout) + cert_file = await _check_for_file(model, cert_files) + return cert_file + +get_cert_file_name = zaza.model.sync_wrapper(async_get_cert_file_name) def get_cacert(): @@ -193,8 +252,9 @@ def get_cacert(): :returns: Path to CA Certificate bundle or None. :rtype: Optional[str] """ - if os.path.exists(KEYSTONE_LOCAL_CACERT): - return KEYSTONE_LOCAL_CACERT + for _cert in LOCAL_CERTIFICATES: + if os.path.exists(_cert): + return _cert # OpenStack Client helpers @@ -1951,24 +2011,28 @@ def get_overcloud_auth(address=None, model_name=None): 'API_VERSION': 3, } if tls_rid: + cert_file = get_cert_file_name('keystone', model_name=model_name) unit = model.get_first_unit_name('keystone', model_name=model_name) # ensure that the path to put the local cacert in actually exists. The # assumption that 'tests/' exists for, say, mojo is false. # Needed due to: # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 - _dir = os.path.dirname(KEYSTONE_LOCAL_CACERT) + _dir = os.path.dirname(cert_file) if not os.path.exists(_dir): os.makedirs(_dir) + _local_cert_file = "{}/{}".format( + LOCAL_CERT_DIR, + os.path.basename(cert_file)) model.scp_from_unit( unit, - KEYSTONE_REMOTE_CACERT, - KEYSTONE_LOCAL_CACERT) + cert_file, + _local_cert_file) - if os.path.exists(KEYSTONE_LOCAL_CACERT): - os.chmod(KEYSTONE_LOCAL_CACERT, 0o644) - auth_settings['OS_CACERT'] = KEYSTONE_LOCAL_CACERT + if os.path.exists(_local_cert_file): + os.chmod(_local_cert_file, 0o644) + auth_settings['OS_CACERT'] = _local_cert_file return auth_settings From a20733cd1460ada0987ebb2999086eb6dd77cee8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 24 Jan 2021 14:31:29 +0000 Subject: [PATCH 776/898] Refactor ca functions --- zaza/openstack/charm_tests/keystone/setup.py | 4 +- zaza/openstack/charm_tests/vault/setup.py | 7 +- zaza/openstack/charm_tests/vault/tests.py | 4 +- zaza/openstack/utilities/openstack.py | 214 +++++++++++++------ 4 files changed, 153 insertions(+), 76 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index 8d64993..bec43fe 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -41,9 +41,7 @@ def wait_for_cacert(model_name=None): :type model_name: str """ logging.info("Waiting for cacert") - cert_file = openstack_utils.get_cert_file_name( - 'keystone') - zaza.model.block_until_file_has_contents( + zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', cert_file, 'CERTIFICATE', diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index d605063..ad1d535 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -222,9 +222,7 @@ def validate_ca(cacertificate, application="keystone", port=5000): :returns: None :rtype: None """ - cert_file = zaza.openstack.utilities.openstack.get_cert_file_name( - application) - zaza.model.block_until_file_has_contents( + zaza.openstack.utilities.openstack.block_until_ca_exists( application, cert_file, cacertificate.decode().strip()) @@ -238,3 +236,6 @@ def validate_ca(cacertificate, application="keystone", port=5000): fp.write(cacertificate.decode()) fp.flush() requests.get('https://{}:{}'.format(ip, str(port)), verify=fp.name) + +def get_cert(): + print(zaza.openstack.utilities.openstack.get_remote_ca_cert_file('masakari')) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index e96f36a..141a1e6 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -154,9 +154,7 @@ class VaultTest(BaseVaultTest): test_config = lifecycle_utils.get_charm_config() del test_config['target_deploy_status']['vault'] - cert_file = zaza.openstack.utilities.openstack.get_cert_file_name( - 'keystone') - zaza.model.block_until_file_has_contents( + zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', cert_file, cacert.decode().strip()) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index b914c0f..7d2fc9d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -27,6 +27,7 @@ import logging import os import paramiko import re +import shutil import six import subprocess import sys @@ -182,7 +183,10 @@ WORKLOAD_STATUS_EXCEPTIONS = { 'ceilometer and gnocchi')}} # For vault TLS certificates +CACERT_FILENAME_FORMAT = "{}_juju_ca_cert.crt" +CERT_PROVIDORS = ['vault'] LOCAL_CERT_DIR = "tests" +REMOTE_CERT_DIR = "/usr/local/share/ca-certificates" KEYSTONE_CACERT = "keystone_juju_ca_cert.crt" KEYSTONE_REMOTE_CACERT = ( "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT)) @@ -197,53 +201,91 @@ REMOTE_CERTIFICATES = [VAULT_REMOTE_CACERT, KEYSTONE_REMOTE_CACERT] LOCAL_CERTIFICATES = [VAULT_LOCAL_CACERT, KEYSTONE_LOCAL_CACERT] -async def async_get_cert_file_name(app, cert_files=None, block=True, - model_name=None, timeout=2700): - """Get the name of the CA cert file thats on all units of an application. - - :param app: Name of application - :type capp: str - :param cert_files: List of cert files to search for. - :type cert_files: List[str] - :param block: Whether to block until a consistent cert file is found. - :type block: bool - :param model_name: Name of model to run check in - :type model_name: str - :param timeout: Time to wait for consistent file - :type timeout: int - :returns: Credentials dictionary - :rtype: dict - """ - async def _check_for_file(model, cert_files): - units = model.applications[app].units - results = {u.entity_id: [] for u in units} - for unit in units: - try: - for cf in cert_files: - output = await unit.run('test -e "{}"; echo $?'.format(cf)) - contents = output.data.get('results')['Stdout'] - if "0" in contents: - results[unit.entity_id].append(cf) - except JujuError: - pass - for cert_file in cert_files: - # Check that the certificate file exists on all the units. - if all(cert_file in files for files in results.values()): - return cert_file +async def async_block_until_ca_exists(application_name, ca_cert, model_name=None, timeout=2700): + async def _check_ca_present(model, ca_files): + units = model.applications[application_name].units + print(ca_files) + for ca_file in ca_files: + for unit in units: + print(unit) + print(ca_file) + try: + output = await unit.run('cat {}'.format(ca_file)) + contents = output.data.get('results').get('Stdout', '') + if not ca_cert in contents: + print("It's not here!") + print(ca_cert) + print(contents) + break + if ca_cert in contents: + print("It's here!") + # libjuju throws a generic error for connection failure. So we + # cannot differentiate between a connectivity issue and a + # target file not existing error. For now just assume the + # latter. + except JujuError: + continue + else: + return True else: - return None - - if not cert_files: - cert_files = REMOTE_CERTIFICATES - cert_file = None + return False + ca_files = await _async_get_remote_ca_cert_file_candidates(application_name, model_name=model_name) + print(ca_files) async with zaza.model.run_in_model(model_name) as model: - if block: - await zaza.model.async_block_until( - lambda: _check_for_file(model, cert_files), timeout=timeout) - cert_file = await _check_for_file(model, cert_files) - return cert_file + await zaza.model.async_block_until( + lambda: _check_ca_present(model, ca_files), timeout=timeout) -get_cert_file_name = zaza.model.sync_wrapper(async_get_cert_file_name) +block_until_ca_exists = zaza.model.sync_wrapper(async_block_until_ca_exists) + + + +#async def async_get_cert_file_name(app, cert_files=None, block=True, +# model_name=None, timeout=2700): +# """Get the name of the CA cert file thats on all units of an application. +# +# :param app: Name of application +# :type capp: str +# :param cert_files: List of cert files to search for. +# :type cert_files: List[str] +# :param block: Whether to block until a consistent cert file is found. +# :type block: bool +# :param model_name: Name of model to run check in +# :type model_name: str +# :param timeout: Time to wait for consistent file +# :type timeout: int +# :returns: Credentials dictionary +# :rtype: dict +# """ +# async def _check_for_file(model, cert_files): +# units = model.applications[app].units +# results = {u.entity_id: [] for u in units} +# for unit in units: +# try: +# for cf in cert_files: +# output = await unit.run('test -e "{}"; echo $?'.format(cf)) +# contents = output.data.get('results')['Stdout'] +# if "0" in contents: +# results[unit.entity_id].append(cf) +# except JujuError: +# pass +# for cert_file in cert_files: +# # Check that the certificate file exists on all the units. +# if all(cert_file in files for files in results.values()): +# return cert_file +# else: +# return None +# +# if not cert_files: +# cert_files = REMOTE_CERTIFICATES +# cert_file = None +# async with zaza.model.run_in_model(model_name) as model: +# if block: +# await zaza.model.async_block_until( +# lambda: _check_for_file(model, cert_files), timeout=timeout) +# cert_file = await _check_for_file(model, cert_files) +# return cert_file +# +#get_cert_file_name = zaza.model.sync_wrapper(async_get_cert_file_name) def get_cacert(): @@ -2010,32 +2052,70 @@ def get_overcloud_auth(address=None, model_name=None): 'OS_PROJECT_DOMAIN_NAME': 'admin_domain', 'API_VERSION': 3, } - if tls_rid: - cert_file = get_cert_file_name('keystone', model_name=model_name) - unit = model.get_first_unit_name('keystone', model_name=model_name) - - # ensure that the path to put the local cacert in actually exists. The - # assumption that 'tests/' exists for, say, mojo is false. - # Needed due to: - # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 - _dir = os.path.dirname(cert_file) - if not os.path.exists(_dir): - os.makedirs(_dir) - - _local_cert_file = "{}/{}".format( - LOCAL_CERT_DIR, - os.path.basename(cert_file)) - model.scp_from_unit( - unit, - cert_file, - _local_cert_file) - - if os.path.exists(_local_cert_file): - os.chmod(_local_cert_file, 0o644) - auth_settings['OS_CACERT'] = _local_cert_file + local_ca_cert = get_remote_ca_cert_file('keystone', model_name=model_name) + if ca_cert: + auth_settings['OS_CACERT'] = local_ca_cert return auth_settings +async def _async_get_remote_ca_cert_file_candidates(application, model_name=None): + cert_files = [] + # unit = model.get_first_unit_name(application, model_name=model_name) + units = await model.async_get_units(application, model_name=model_name) + unit = units[0].name + for _providor in CERT_PROVIDORS: + tls_rid = await model.async_get_relation_id( + application, + _providor, + model_name=model_name, + remote_interface_name='certificates') + if tls_rid: + cert_files.append(REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format(_providor)) + cert_files.append(REMOTE_CERT_DIR + '/' + KEYSTONE_CACERT) + return cert_files + +_get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper(_async_get_remote_ca_cert_file_candidates) + +def get_remote_ca_cert_file(application, model_name=None): +# CACERT_FILENAME = "{}_juju_ca_cert.crt" +# cert_files = [] +# unit = model.get_first_unit_name(application, model_name=model_name) +# for _providor in CERT_PROVIDORS: +# tls_rid = model.get_relation_id( +# application, +# _providor, +# model_name=model_name, +# remote_interface_name='certificates') +# if tls_rid: +# cert_files.append(CACERT_FILENAME.format(_providor)) +# cert_files.append(KEYSTONE_CACERT) + unit = model.get_first_unit_name(application, model_name=model_name) + local_cert_file = None + cert_files = _get_remote_ca_cert_file_candidates(application, model_name=model_name) + for cert_file in cert_files: + _local_cert_file = "{}/{}".format( + LOCAL_CERT_DIR, + os.path.basename(cert_file)) + with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca_file: + try: + model.scp_from_unit( + unit, + cert_file, + _tmp_ca_file.name) + except JujuError: + continue + # ensure that the path to put the local cacert in actually exists. The + # assumption that 'tests/' exists for, say, mojo is false. + # Needed due to: + # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 + _dir = os.path.dirname(_local_cert_file) + if not os.path.exists(_dir): + os.makedirs(_dir) + shutil.move(_tmp_ca_file.name, _local_cert_file) + os.chmod(_local_cert_file, 0o644) + local_cert_file = _local_cert_file + break + return local_cert_file def get_urllib_opener(): """Create a urllib opener taking into account proxy settings. From 24fbc068c94f99a702f7b3f8a2a01b7de0bf855d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 24 Jan 2021 15:27:06 +0000 Subject: [PATCH 777/898] Correct args to block_until_ca_exists --- zaza/openstack/charm_tests/keystone/setup.py | 1 - zaza/openstack/charm_tests/vault/setup.py | 1 - zaza/openstack/charm_tests/vault/tests.py | 1 - 3 files changed, 3 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index bec43fe..73264cd 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -43,7 +43,6 @@ def wait_for_cacert(model_name=None): logging.info("Waiting for cacert") zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', - cert_file, 'CERTIFICATE', model_name=model_name) zaza.model.block_until_all_units_idle(model_name=model_name) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index ad1d535..25389d5 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -224,7 +224,6 @@ def validate_ca(cacertificate, application="keystone", port=5000): """ zaza.openstack.utilities.openstack.block_until_ca_exists( application, - cert_file, cacertificate.decode().strip()) vip = (zaza.model.get_application_config(application) .get("vip").get("value")) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 141a1e6..68130a0 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -156,7 +156,6 @@ class VaultTest(BaseVaultTest): del test_config['target_deploy_status']['vault'] zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', - cert_file, cacert.decode().strip()) zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) From d637646a9e139245029a4506afc2d0b71f2060bc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 24 Jan 2021 16:19:45 +0000 Subject: [PATCH 778/898] Fix typo --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 7d2fc9d..5fb4980 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2053,7 +2053,7 @@ def get_overcloud_auth(address=None, model_name=None): 'API_VERSION': 3, } local_ca_cert = get_remote_ca_cert_file('keystone', model_name=model_name) - if ca_cert: + if local_ca_cert: auth_settings['OS_CACERT'] = local_ca_cert return auth_settings From 401829f0a7b657f1eb9b3d129844476ce869839d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 24 Jan 2021 17:24:20 +0000 Subject: [PATCH 779/898] Code tidy and docstrings --- zaza/openstack/charm_tests/vault/setup.py | 3 - zaza/openstack/utilities/openstack.py | 152 ++++++++-------------- 2 files changed, 53 insertions(+), 102 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 25389d5..c792508 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -235,6 +235,3 @@ def validate_ca(cacertificate, application="keystone", port=5000): fp.write(cacertificate.decode()) fp.flush() requests.get('https://{}:{}'.format(ip, str(port)), verify=fp.name) - -def get_cert(): - print(zaza.openstack.utilities.openstack.get_remote_ca_cert_file('masakari')) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 5fb4980..a975206 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -184,7 +184,7 @@ WORKLOAD_STATUS_EXCEPTIONS = { # For vault TLS certificates CACERT_FILENAME_FORMAT = "{}_juju_ca_cert.crt" -CERT_PROVIDORS = ['vault'] +CERT_PROVIDERS = ['vault'] LOCAL_CERT_DIR = "tests" REMOTE_CERT_DIR = "/usr/local/share/ca-certificates" KEYSTONE_CACERT = "keystone_juju_ca_cert.crt" @@ -192,33 +192,29 @@ KEYSTONE_REMOTE_CACERT = ( "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT)) KEYSTONE_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, KEYSTONE_CACERT)) -VAULT_CACERT = "vault_juju_ca_cert.crt" -VAULT_REMOTE_CACERT = ( - "/usr/local/share/ca-certificates/{}".format(VAULT_CACERT)) -VAULT_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, VAULT_CACERT)) -REMOTE_CERTIFICATES = [VAULT_REMOTE_CACERT, KEYSTONE_REMOTE_CACERT] -LOCAL_CERTIFICATES = [VAULT_LOCAL_CACERT, KEYSTONE_LOCAL_CACERT] +async def async_block_until_ca_exists(application_name, ca_cert, + model_name=None, timeout=2700): + """Block until a CA cert is on all units of application_name. - -async def async_block_until_ca_exists(application_name, ca_cert, model_name=None, timeout=2700): + :param application_name: Name of application to check + :type application_name: str + :param ca_cert: The certificate to look for. + :type ca_cert: str + :param model_name: Name of model to query. + :type model_name: str + :param timeout: How long in seconds to wait + :type timeout: int + """ async def _check_ca_present(model, ca_files): units = model.applications[application_name].units - print(ca_files) for ca_file in ca_files: for unit in units: - print(unit) - print(ca_file) try: output = await unit.run('cat {}'.format(ca_file)) contents = output.data.get('results').get('Stdout', '') - if not ca_cert in contents: - print("It's not here!") - print(ca_cert) - print(contents) + if ca_cert not in contents: break - if ca_cert in contents: - print("It's here!") # libjuju throws a generic error for connection failure. So we # cannot differentiate between a connectivity issue and a # target file not existing error. For now just assume the @@ -229,8 +225,9 @@ async def async_block_until_ca_exists(application_name, ca_cert, model_name=None return True else: return False - ca_files = await _async_get_remote_ca_cert_file_candidates(application_name, model_name=model_name) - print(ca_files) + ca_files = await _async_get_remote_ca_cert_file_candidates( + application_name, + model_name=model_name) async with zaza.model.run_in_model(model_name) as model: await zaza.model.async_block_until( lambda: _check_ca_present(model, ca_files), timeout=timeout) @@ -238,65 +235,19 @@ async def async_block_until_ca_exists(application_name, ca_cert, model_name=None block_until_ca_exists = zaza.model.sync_wrapper(async_block_until_ca_exists) - -#async def async_get_cert_file_name(app, cert_files=None, block=True, -# model_name=None, timeout=2700): -# """Get the name of the CA cert file thats on all units of an application. -# -# :param app: Name of application -# :type capp: str -# :param cert_files: List of cert files to search for. -# :type cert_files: List[str] -# :param block: Whether to block until a consistent cert file is found. -# :type block: bool -# :param model_name: Name of model to run check in -# :type model_name: str -# :param timeout: Time to wait for consistent file -# :type timeout: int -# :returns: Credentials dictionary -# :rtype: dict -# """ -# async def _check_for_file(model, cert_files): -# units = model.applications[app].units -# results = {u.entity_id: [] for u in units} -# for unit in units: -# try: -# for cf in cert_files: -# output = await unit.run('test -e "{}"; echo $?'.format(cf)) -# contents = output.data.get('results')['Stdout'] -# if "0" in contents: -# results[unit.entity_id].append(cf) -# except JujuError: -# pass -# for cert_file in cert_files: -# # Check that the certificate file exists on all the units. -# if all(cert_file in files for files in results.values()): -# return cert_file -# else: -# return None -# -# if not cert_files: -# cert_files = REMOTE_CERTIFICATES -# cert_file = None -# async with zaza.model.run_in_model(model_name) as model: -# if block: -# await zaza.model.async_block_until( -# lambda: _check_for_file(model, cert_files), timeout=timeout) -# cert_file = await _check_for_file(model, cert_files) -# return cert_file -# -#get_cert_file_name = zaza.model.sync_wrapper(async_get_cert_file_name) - - def get_cacert(): """Return path to CA Certificate bundle for verification during test. :returns: Path to CA Certificate bundle or None. - :rtype: Optional[str] + :rtype: Union[str, None] """ - for _cert in LOCAL_CERTIFICATES: + for _provider in CERT_PROVIDERS: + _cert = LOCAL_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format( + _provider) if os.path.exists(_cert): return _cert + if os.path.exists(KEYSTONE_LOCAL_CACERT): + return KEYSTONE_LOCAL_CACERT # OpenStack Client helpers @@ -2058,65 +2009,68 @@ def get_overcloud_auth(address=None, model_name=None): return auth_settings -async def _async_get_remote_ca_cert_file_candidates(application, model_name=None): + +async def _async_get_remote_ca_cert_file_candidates(application, + model_name=None): cert_files = [] - # unit = model.get_first_unit_name(application, model_name=model_name) - units = await model.async_get_units(application, model_name=model_name) - unit = units[0].name - for _providor in CERT_PROVIDORS: + for _provider in CERT_PROVIDERS: tls_rid = await model.async_get_relation_id( application, - _providor, + _provider, model_name=model_name, remote_interface_name='certificates') if tls_rid: - cert_files.append(REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format(_providor)) - cert_files.append(REMOTE_CERT_DIR + '/' + KEYSTONE_CACERT) + cert_files.append( + REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format( + _provider)) + cert_files.append(KEYSTONE_LOCAL_CACERT) return cert_files -_get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper(_async_get_remote_ca_cert_file_candidates) +_get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper( + _async_get_remote_ca_cert_file_candidates) + def get_remote_ca_cert_file(application, model_name=None): -# CACERT_FILENAME = "{}_juju_ca_cert.crt" -# cert_files = [] -# unit = model.get_first_unit_name(application, model_name=model_name) -# for _providor in CERT_PROVIDORS: -# tls_rid = model.get_relation_id( -# application, -# _providor, -# model_name=model_name, -# remote_interface_name='certificates') -# if tls_rid: -# cert_files.append(CACERT_FILENAME.format(_providor)) -# cert_files.append(KEYSTONE_CACERT) + """Collect CA certificate from application. + + :param application: Name of application to collect file from. + :type application: str + :param model_name: Name of model to query. + :type model_name: str + :returns: Path to cafile + :rtype: str + """ unit = model.get_first_unit_name(application, model_name=model_name) local_cert_file = None - cert_files = _get_remote_ca_cert_file_candidates(application, model_name=model_name) + cert_files = _get_remote_ca_cert_file_candidates( + application, + model_name=model_name) for cert_file in cert_files: _local_cert_file = "{}/{}".format( LOCAL_CERT_DIR, os.path.basename(cert_file)) - with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca_file: + with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca: try: model.scp_from_unit( unit, cert_file, - _tmp_ca_file.name) + _tmp_ca.name) except JujuError: continue - # ensure that the path to put the local cacert in actually exists. The - # assumption that 'tests/' exists for, say, mojo is false. + # ensure that the path to put the local cacert in actually exists. + # The assumption that 'tests/' exists for, say, mojo is false. # Needed due to: # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 _dir = os.path.dirname(_local_cert_file) if not os.path.exists(_dir): os.makedirs(_dir) - shutil.move(_tmp_ca_file.name, _local_cert_file) + shutil.move(_tmp_ca.name, _local_cert_file) os.chmod(_local_cert_file, 0o644) local_cert_file = _local_cert_file break return local_cert_file + def get_urllib_opener(): """Create a urllib opener taking into account proxy settings. From e047150f5bae0b8a996cb57542491131348b92f1 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 25 Jan 2021 09:21:00 +0000 Subject: [PATCH 780/898] Add unit tests --- .../test_zaza_utilities_openstack.py | 157 +++++++++++++++++- ..._zaza_utilities_parallel_series_upgrade.py | 24 +-- unit_tests/utils.py | 22 +++ zaza/openstack/utilities/openstack.py | 12 +- 4 files changed, 190 insertions(+), 25 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 7e0e8f1..e4abfb1 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -17,6 +17,8 @@ import datetime import io import mock import subprocess +import sys +import unittest import tenacity import unit_tests.utils as ut_utils @@ -191,6 +193,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.patch_object(openstack_utils, 'get_application_config_option') self.patch_object(openstack_utils, 'get_keystone_ip') self.patch_object(openstack_utils, "get_current_os_versions") + self.patch_object(openstack_utils, "get_remote_ca_cert_file") self.patch_object(openstack_utils.juju_utils, 'leader_get') if tls_relation: self.patch_object(openstack_utils.model, "scp_from_unit") @@ -204,6 +207,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.get_relation_id.return_value = None self.get_application_config_option.return_value = None self.leader_get.return_value = 'openstack' + self.get_remote_ca_cert_file.return_value = None if tls_relation or ssl_cert: port = 35357 transport = 'https' @@ -245,7 +249,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'API_VERSION': 3, } if tls_relation: - expect['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT + self.get_remote_ca_cert_file.return_value = '/tmp/a.cert' + expect['OS_CACERT'] = '/tmp/a.cert' self.assertEqual(openstack_utils.get_overcloud_auth(), expect) @@ -1327,3 +1332,153 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): mock.ANY, 'bridge-interface-mappings', {'ovn-bridge-mappings': 'physnet1:br-ex'})) + + def test_get_cacert(self): + self.patch_object(openstack_utils.os.path, 'exists') + results = { + 'tests/vault_juju_ca_cert.crt': True} + self.exists.side_effect = lambda x: results[x] + self.assertEqual( + openstack_utils.get_cacert(), + 'tests/vault_juju_ca_cert.crt') + + results = { + 'tests/vault_juju_ca_cert.crt': False, + 'tests/keystone_juju_ca_cert.crt': True} + self.assertEqual( + openstack_utils.get_cacert(), + 'tests/keystone_juju_ca_cert.crt') + + results = { + 'tests/vault_juju_ca_cert.crt': False, + 'tests/keystone_juju_ca_cert.crt': False} + self.assertIsNone(openstack_utils.get_cacert()) + + def test_get_remote_ca_cert_file(self): + self.patch_object(openstack_utils.model, 'get_first_unit_name') + self.patch_object( + openstack_utils, + '_get_remote_ca_cert_file_candidates') + self.patch_object(openstack_utils.model, 'scp_from_unit') + self.patch_object(openstack_utils.os.path, 'exists') + self.patch_object(openstack_utils.shutil, 'move') + self.patch_object(openstack_utils.os, 'chmod') + self.patch_object(openstack_utils.tempfile, 'NamedTemporaryFile') + enter_mock = mock.MagicMock() + enter_mock.__enter__.return_value.name = 'tempfilename' + self.NamedTemporaryFile.return_value = enter_mock + self.get_first_unit_name.return_value = 'neutron-api/0' + self._get_remote_ca_cert_file_candidates.return_value = [ + '/tmp/ca1.cert'] + self.exists.return_value = True + + openstack_utils.get_remote_ca_cert_file('neutron-api') + self.scp_from_unit.assert_called_once_with( + 'neutron-api/0', + '/tmp/ca1.cert', + 'tempfilename') + self.chmod.assert_called_once_with('tests/ca1.cert', 0o644) + self.move.assert_called_once_with('tempfilename', 'tests/ca1.cert') + + +class TestAsyncOpenstackUtils(ut_utils.AioTestCase): + + def setUp(self): + super(TestAsyncOpenstackUtils, self).setUp() + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + model_mock = mock.MagicMock() + test_mock = mock.MagicMock() + + class AsyncContextManagerMock(test_mock): + async def __aenter__(self): + yield model_mock + + async def __aexit__(self, *args): + pass + + self.model_mock = model_mock + self.patch_object(openstack_utils.zaza.model, "async_block_until") + + async def _block_until(f, timeout): + # Store the result of the call to _check_ca_present to validate + # tests + self.result = await f() + self.async_block_until.side_effect = _block_until + self.patch('zaza.model.run_in_model', name='_run_in_model') + self._run_in_model.return_value = AsyncContextManagerMock + self._run_in_model().__aenter__.return_value = self.model_mock + + async def test_async_block_until_ca_exists(self): + def _get_action_output(stdout, code, stderr=None): + stderr = stderr or '' + action = mock.MagicMock() + action.data = { + 'results': { + 'Code': code, + 'Stderr': stderr, + 'Stdout': stdout}} + return action + results = { + '/tmp/missing.cert': _get_action_output( + '', + '1', + 'cat: /tmp/missing.cert: No such file or directory'), + '/tmp/good.cert': _get_action_output('CERTIFICATE', '0')} + + async def _run(command, timeout=None): + return results[command.split()[-1]] + self.unit1 = mock.MagicMock() + self.unit2 = mock.MagicMock() + self.unit2.run.side_effect = _run + self.unit1.run.side_effect = _run + self.units = [self.unit1, self.unit2] + _units = mock.MagicMock() + _units.units = self.units + self.model_mock.applications = { + 'keystone': _units + } + self.patch_object( + openstack_utils, + "_async_get_remote_ca_cert_file_candidates") + + # Test a missing cert then a good cert. + self._async_get_remote_ca_cert_file_candidates.return_value = [ + '/tmp/missing.cert', + '/tmp/good.cert'] + await openstack_utils.async_block_until_ca_exists( + 'keystone', + 'CERTIFICATE') + self.assertTrue(self.result) + + # Test a single missing + self._async_get_remote_ca_cert_file_candidates.return_value = [ + '/tmp/missing.cert'] + await openstack_utils.async_block_until_ca_exists( + 'keystone', + 'CERTIFICATE') + self.assertFalse(self.result) + + async def test__async_get_remote_ca_cert_file_candidates(self): + self.patch_object(openstack_utils.zaza.model, "async_get_relation_id") + rel_id_out = { + } + + def _get_relation_id(app, cert_app, model_name, remote_interface_name): + return rel_id_out[cert_app] + self.async_get_relation_id.side_effect = _get_relation_id + + rel_id_out['vault'] = 'certs:1' + r = await openstack_utils._async_get_remote_ca_cert_file_candidates( + 'neutron-api', 'mymodel') + self.assertEqual( + r, + ['/usr/local/share/ca-certificates/vault_juju_ca_cert.crt', + '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt']) + + rel_id_out['vault'] = None + r = await openstack_utils._async_get_remote_ca_cert_file_candidates( + 'neutron-api', 'mymodel') + self.assertEqual( + r, + ['/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt']) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 0231dbf..70ffd3b 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio import mock import sys import unittest @@ -139,28 +138,7 @@ class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase): self.assertEqual(expected, config) -class AioTestCase(ut_utils.BaseTestCase): - def __init__(self, methodName='runTest', loop=None): - self.loop = loop or asyncio.get_event_loop() - self._function_cache = {} - super(AioTestCase, self).__init__(methodName=methodName) - - def coroutine_function_decorator(self, func): - def wrapper(*args, **kw): - return self.loop.run_until_complete(func(*args, **kw)) - return wrapper - - def __getattribute__(self, item): - attr = object.__getattribute__(self, item) - if asyncio.iscoroutinefunction(attr) and item.startswith('test_'): - if item not in self._function_cache: - self._function_cache[item] = ( - self.coroutine_function_decorator(attr)) - return self._function_cache[item] - return attr - - -class TestParallelSeriesUpgrade(AioTestCase): +class TestParallelSeriesUpgrade(ut_utils.AioTestCase): def setUp(self): super(TestParallelSeriesUpgrade, self).setUp() if sys.version_info < (3, 6, 0): diff --git a/unit_tests/utils.py b/unit_tests/utils.py index 4694d0d..8e31f45 100644 --- a/unit_tests/utils.py +++ b/unit_tests/utils.py @@ -19,6 +19,7 @@ """Module to provide helper for writing unit tests.""" +import asyncio import contextlib import io import mock @@ -96,3 +97,24 @@ class BaseTestCase(unittest.TestCase): started.return_value = return_value self._patches_start[name] = started setattr(self, name, started) + + +class AioTestCase(BaseTestCase): + def __init__(self, methodName='runTest', loop=None): + self.loop = loop or asyncio.get_event_loop() + self._function_cache = {} + super(AioTestCase, self).__init__(methodName=methodName) + + def coroutine_function_decorator(self, func): + def wrapper(*args, **kw): + return self.loop.run_until_complete(func(*args, **kw)) + return wrapper + + def __getattribute__(self, item): + attr = object.__getattribute__(self, item) + if asyncio.iscoroutinefunction(attr) and item.startswith('test_'): + if item not in self._function_cache: + self._function_cache[item] = ( + self.coroutine_function_decorator(attr)) + return self._function_cache[item] + return attr diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index a975206..d883e0a 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -222,6 +222,7 @@ async def async_block_until_ca_exists(application_name, ca_cert, except JujuError: continue else: + # The CA was found in `ca_file` on all units. return True else: return False @@ -2012,6 +2013,15 @@ def get_overcloud_auth(address=None, model_name=None): async def _async_get_remote_ca_cert_file_candidates(application, model_name=None): + """Return a list of possible remote CA file names. + + :param application: Name of application to examine. + :type application: str + :param model_name: Name of model to query. + :type model_name: str + :returns: List of paths to possible ca files. + :rtype: List[str] + """ cert_files = [] for _provider in CERT_PROVIDERS: tls_rid = await model.async_get_relation_id( @@ -2023,7 +2033,7 @@ async def _async_get_remote_ca_cert_file_candidates(application, cert_files.append( REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format( _provider)) - cert_files.append(KEYSTONE_LOCAL_CACERT) + cert_files.append(KEYSTONE_REMOTE_CACERT) return cert_files _get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper( From 5be8fc377179e8fce9de8e8e7d9744fc169d65a4 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 25 Jan 2021 11:41:39 +0000 Subject: [PATCH 781/898] Fix docstring and bug --- zaza/openstack/utilities/openstack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index d883e0a..522df31 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -199,7 +199,7 @@ async def async_block_until_ca_exists(application_name, ca_cert, :param application_name: Name of application to check :type application_name: str - :param ca_cert: The certificate to look for. + :param ca_cert: The certificate content. :type ca_cert: str :param model_name: Name of model to query. :type model_name: str @@ -220,7 +220,7 @@ async def async_block_until_ca_exists(application_name, ca_cert, # target file not existing error. For now just assume the # latter. except JujuError: - continue + break else: # The CA was found in `ca_file` on all units. return True From 93a9aff92796ce626ae969fc0554f3411dc3adf2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 25 Jan 2021 11:47:42 +0000 Subject: [PATCH 782/898] Fix context manager mock --- unit_tests/utilities/test_zaza_utilities_openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index e4abfb1..67cd417 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1392,7 +1392,7 @@ class TestAsyncOpenstackUtils(ut_utils.AioTestCase): class AsyncContextManagerMock(test_mock): async def __aenter__(self): - yield model_mock + return self async def __aexit__(self, *args): pass From dc886f65d5a6fa5093a8cfdb9a0f293e6154a82f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 28 Jan 2021 08:49:11 +0100 Subject: [PATCH 783/898] Add workaround for OVS-OVN migration on Groovy The root of the issue is in Open vSwitch itself and it is not easilly workaroundable in the charms. We'll pursue a upstream/package level fix. --- zaza/openstack/charm_tests/ovn/tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index c5eae5e..9bea951 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -143,6 +143,13 @@ class ChassisCharmOperationTest(BaseCharmOperationTest): class OVSOVNMigrationTest(test_utils.BaseCharmTest): """OVS to OVN migration tests.""" + @classmethod + def setUpClass(cls): + """Run class setup for OVN migration tests.""" + super(OVSOVNMigrationTest, cls).setUpClass() + cls.current_release = openstack_utils.get_os_release( + openstack_utils.get_current_os_release_pair()) + def setUp(self): """Perform migration steps prior to validation.""" super(OVSOVNMigrationTest, self).setUp() @@ -410,3 +417,17 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): zaza.model.wait_for_agent_status() zaza.model.wait_for_application_states( states=self.target_deploy_status) + # Workaround for our old friend LP: #1852221 which hit us again on + # Groovy. We make the os_release check explicit so that we can + # re-evaluate the need for the workaround at the next release. + if self.current_release == openstack_utils.get_os_release( + 'groovy_victoria'): + try: + for application in ('ovn-chassis', 'ovn-dedicated-chassis'): + for unit in zaza.model.get_units(application): + zaza.model.run_on_unit( + unit.entity_id, + 'systemctl restart ovs-vswitchd') + except KeyError: + # One of the applications is not in the model, which is fine + pass From c4691ef1c74365bf2a33f4922f852247e205e8bb Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 29 Jan 2021 13:51:25 +0000 Subject: [PATCH 784/898] Skip OVN provider octavia test on Victoria The OVN provider octavia test on Victoria is currently broken due to Bug #1896603. Until it is fixed skip the test. --- zaza/openstack/charm_tests/octavia/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index e95a33f..26941a2 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -178,6 +178,14 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): for provider in octavia_client.provider_list().get('providers', []) if provider['name'] != 'octavia' # alias for `amphora`, skip } + if (openstack_utils.get_os_release() in [ + openstack_utils.get_os_release('focal_victoria'), + openstack_utils.get_os_release('groovy_victoria')]): + logging.info("Skipping tests of ovn backed lb (Bug #1896603)") + try: + del providers['ovn'] + except KeyError: + pass return providers def _create_lb_resources(self, octavia_client, provider, vip_subnet_id, From ea03e36273ea616a14e53be6640339ba750597e3 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Mon, 1 Feb 2021 15:50:31 -0800 Subject: [PATCH 785/898] Replace get_relation_from_unit for designate test --- zaza/openstack/charm_tests/designate/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 3c56ee1..78a5f31 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -23,7 +23,7 @@ import designateclient.v1.records as records import designateclient.v1.servers as servers import zaza.model -import zaza.openstack.utilities.juju as zaza_juju +import zaza.utilities.juju as juju_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.designate.utils as designate_utils @@ -170,7 +170,7 @@ class DesignateAPITests(BaseDesignateTest): reraise=True ) def _wait_to_resolve_test_record(self): - dns_ip = zaza_juju.get_relation_from_unit( + dns_ip = juju_utils.get_relation_from_unit( 'designate/0', 'designate-bind/0', 'dns-backend' From 33ab875eb25668ce4c65ce548ef97aaf55947219 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Tue, 2 Feb 2021 09:58:13 -0800 Subject: [PATCH 786/898] Replace get_relation_from_unit for ceph test --- zaza/openstack/charm_tests/ceph/tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index c7d44bd..27fefbd 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -33,7 +33,7 @@ import zaza.model as zaza_model import zaza.openstack.utilities.ceph as zaza_ceph import zaza.openstack.utilities.exceptions as zaza_exceptions import zaza.openstack.utilities.generic as zaza_utils -import zaza.openstack.utilities.juju as zaza_juju +import zaza.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as zaza_openstack @@ -124,7 +124,7 @@ class CephRelationTest(test_utils.OpenStackBaseTest): relation_name = 'osd' remote_unit = zaza_model.get_unit_from_name(remote_unit_name) remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( + relation = juju_utils.get_relation_from_unit( unit_name, remote_unit_name, relation_name @@ -153,7 +153,7 @@ class CephRelationTest(test_utils.OpenStackBaseTest): 'ceph-public-address': remote_ip, 'fsid': fsid, } - relation = zaza_juju.get_relation_from_unit( + relation = juju_utils.get_relation_from_unit( unit_name, remote_unit_name, relation_name @@ -980,7 +980,7 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest): ceph_pools_detail = zaza_ceph.get_ceph_pool_details( model_name=self.model_name) logging.debug('AFTER: {}'.format(ceph_pools_detail)) - logging.debug(zaza_juju.get_relation_from_unit( + logging.debug(juju_utils.get_relation_from_unit( 'ceph-mon', self.application_name, None, model_name=self.model_name)) logging.info('Checking Ceph pool compression_mode after restoring ' From 7878523e3457116497cd48bdc7fa8b3761d0c621 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Wed, 3 Feb 2021 18:01:28 +0000 Subject: [PATCH 787/898] Rename zaza_juju to be consistent with all of repo --- unit_tests/utilities/test_zaza_utilities_ceph.py | 2 +- zaza/openstack/utilities/ceph.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_ceph.py b/unit_tests/utilities/test_zaza_utilities_ceph.py index ea718b7..e6ac337 100644 --- a/unit_tests/utilities/test_zaza_utilities_ceph.py +++ b/unit_tests/utilities/test_zaza_utilities_ceph.py @@ -118,7 +118,7 @@ class TestCephUtils(ut_utils.BaseTestCase): model_name='amodel') def test_pools_from_broker_req(self): - self.patch_object(ceph_utils.zaza_juju, 'get_relation_from_unit') + self.patch_object(ceph_utils.juju_utils, 'get_relation_from_unit') self.get_relation_from_unit.return_value = { 'broker_req': ( '{"api-version": 1, "ops": [' diff --git a/zaza/openstack/utilities/ceph.py b/zaza/openstack/utilities/ceph.py index 6613154..a01f56e 100644 --- a/zaza/openstack/utilities/ceph.py +++ b/zaza/openstack/utilities/ceph.py @@ -3,7 +3,7 @@ import json import logging import zaza.model as zaza_model -import zaza.utilities.juju as zaza_juju +import zaza.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -225,7 +225,7 @@ def get_pools_from_broker_req(application_or_unit, model_name=None): """ # NOTE: we do not pass on a name for the remote_interface_name as that # varies between the Ceph consuming applications. - relation_data = zaza_juju.get_relation_from_unit( + relation_data = juju_utils.get_relation_from_unit( 'ceph-mon', application_or_unit, None, model_name=model_name) # NOTE: we probably should consume the Ceph broker code from c-h but c-h is From 2ab9cebbf67c8e88aabbe09250bd1c27396c5613 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 4 Feb 2021 08:48:17 +0100 Subject: [PATCH 788/898] Use per-model tmp-dir to store local copy of CA cert (#493) The current approach of storing the deployment CA certificate in the 'test/' relative path does not allow for executing tests for multiple targets from the same environment. We have previously moved (7a90110) the local copy of the SSH private key for similar reasons. Remove the global constants as we cannot build them without making function calls, and we'd rather avoid doing that at module import time. Code using the location of the local CA certificate has already been changed to use helper functions. --- .../test_zaza_utilities_openstack.py | 30 +++++++++++++------ zaza/openstack/utilities/openstack.py | 26 +++++++++++----- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 67cd417..c42fc9d 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1333,25 +1333,34 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'bridge-interface-mappings', {'ovn-bridge-mappings': 'physnet1:br-ex'})) + def test_get_cacert_absolute_path(self): + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir') + self.get_tmpdir.return_value = '/tmp/default' + self.assertEqual( + openstack_utils.get_cacert_absolute_path('filename'), + '/tmp/default/filename') + def test_get_cacert(self): + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir') + self.get_tmpdir.return_value = '/tmp/default' self.patch_object(openstack_utils.os.path, 'exists') results = { - 'tests/vault_juju_ca_cert.crt': True} + '/tmp/default/vault_juju_ca_cert.crt': True} self.exists.side_effect = lambda x: results[x] self.assertEqual( openstack_utils.get_cacert(), - 'tests/vault_juju_ca_cert.crt') + '/tmp/default/vault_juju_ca_cert.crt') results = { - 'tests/vault_juju_ca_cert.crt': False, - 'tests/keystone_juju_ca_cert.crt': True} + '/tmp/default/vault_juju_ca_cert.crt': False, + '/tmp/default/keystone_juju_ca_cert.crt': True} self.assertEqual( openstack_utils.get_cacert(), - 'tests/keystone_juju_ca_cert.crt') + '/tmp/default/keystone_juju_ca_cert.crt') results = { - 'tests/vault_juju_ca_cert.crt': False, - 'tests/keystone_juju_ca_cert.crt': False} + '/tmp/default/vault_juju_ca_cert.crt': False, + '/tmp/default/keystone_juju_ca_cert.crt': False} self.assertIsNone(openstack_utils.get_cacert()) def test_get_remote_ca_cert_file(self): @@ -1364,6 +1373,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.patch_object(openstack_utils.shutil, 'move') self.patch_object(openstack_utils.os, 'chmod') self.patch_object(openstack_utils.tempfile, 'NamedTemporaryFile') + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir') + self.get_tmpdir.return_value = '/tmp/default' enter_mock = mock.MagicMock() enter_mock.__enter__.return_value.name = 'tempfilename' self.NamedTemporaryFile.return_value = enter_mock @@ -1377,8 +1388,9 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'neutron-api/0', '/tmp/ca1.cert', 'tempfilename') - self.chmod.assert_called_once_with('tests/ca1.cert', 0o644) - self.move.assert_called_once_with('tempfilename', 'tests/ca1.cert') + self.chmod.assert_called_once_with('/tmp/default/ca1.cert', 0o644) + self.move.assert_called_once_with( + 'tempfilename', '/tmp/default/ca1.cert') class TestAsyncOpenstackUtils(ut_utils.AioTestCase): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 522df31..d277656 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -185,12 +185,10 @@ WORKLOAD_STATUS_EXCEPTIONS = { # For vault TLS certificates CACERT_FILENAME_FORMAT = "{}_juju_ca_cert.crt" CERT_PROVIDERS = ['vault'] -LOCAL_CERT_DIR = "tests" REMOTE_CERT_DIR = "/usr/local/share/ca-certificates" KEYSTONE_CACERT = "keystone_juju_ca_cert.crt" KEYSTONE_REMOTE_CACERT = ( "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT)) -KEYSTONE_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, KEYSTONE_CACERT)) async def async_block_until_ca_exists(application_name, ca_cert, @@ -236,6 +234,18 @@ async def async_block_until_ca_exists(application_name, ca_cert, block_until_ca_exists = zaza.model.sync_wrapper(async_block_until_ca_exists) +def get_cacert_absolute_path(filename): + """Build string containing location of the CA Certificate file. + + :param filename: Expected filename for CA Certificate file. + :type filename: str + :returns: Absolute path to file containing CA Certificate + :rtype: str + """ + return os.path.join( + deployment_env.get_tmpdir(), filename) + + def get_cacert(): """Return path to CA Certificate bundle for verification during test. @@ -243,12 +253,13 @@ def get_cacert(): :rtype: Union[str, None] """ for _provider in CERT_PROVIDERS: - _cert = LOCAL_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format( - _provider) + _cert = get_cacert_absolute_path( + CACERT_FILENAME_FORMAT.format(_provider)) if os.path.exists(_cert): return _cert - if os.path.exists(KEYSTONE_LOCAL_CACERT): - return KEYSTONE_LOCAL_CACERT + _keystone_local_cacert = get_cacert_absolute_path(KEYSTONE_CACERT) + if os.path.exists(_keystone_local_cacert): + return _keystone_local_cacert # OpenStack Client helpers @@ -2056,8 +2067,7 @@ def get_remote_ca_cert_file(application, model_name=None): application, model_name=model_name) for cert_file in cert_files: - _local_cert_file = "{}/{}".format( - LOCAL_CERT_DIR, + _local_cert_file = get_cacert_absolute_path( os.path.basename(cert_file)) with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca: try: From 4cbf70dd5e931285973ab230e65ba3678fa87994 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 4 Feb 2021 08:56:22 +0100 Subject: [PATCH 789/898] octavia: Configure SSH key to allow debugging of Amphorae (#495) --- zaza/openstack/charm_tests/octavia/setup.py | 50 +++++++++++++++++---- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/setup.py b/zaza/openstack/charm_tests/octavia/setup.py index 677c368..4b09aa0 100644 --- a/zaza/openstack/charm_tests/octavia/setup.py +++ b/zaza/openstack/charm_tests/octavia/setup.py @@ -25,6 +25,9 @@ import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.utilities.openstack as openstack import zaza.openstack.configure.guest +import zaza.openstack.charm_tests.nova.setup as nova_setup +import zaza.openstack.charm_tests.nova.utils as nova_utils + def ensure_lts_images(): """Ensure that bionic and focal images are available for the tests.""" @@ -51,13 +54,32 @@ def add_amphora_image(image_url=None): def configure_octavia(): - """Do mandatory post deployment configuration of Octavia.""" - # Tell Octavia charm it is safe to create cloud resources - logging.info('Running `configure-resources` action on Octavia leader unit') - zaza.model.run_action_on_leader( - 'octavia', - 'configure-resources', - action_params={}) + """Do post deployment configuration and initialization of Octavia. + + Certificates for the private Octavia worker <-> Amphorae communication must + be generated and set trough charm configuration. + + The optional SSH configuration options are set to enable debug and log + collection from Amphorae, we will use the same keypair as Zaza uses for + instance creation. + + The `configure-resources` action must be run to have the charm create + in-cloud resources such as management network and associated ports and + security groups. + """ + # Set up Nova client to create/retrieve keypair for Amphora debug purposes. + # + # We reuse the Nova setup code for this and in most cases the test + # declaration will already defined that the Nova manage_ssh_key setup + # helper to run before we get here. Re-run here to make sure this setup + # function can be used separately, manage_ssh_key is idempotent. + keystone_session = openstack.get_overcloud_keystone_session() + nova_client = openstack.get_nova_session_client( + keystone_session) + nova_setup.manage_ssh_key(nova_client) + ssh_public_key = openstack.get_public_key( + nova_client, nova_utils.KEYPAIR_NAME) + # Generate certificates for controller/load balancer instance communication (issuing_cakey, issuing_cacert) = cert.generate_cert( 'OSCI Zaza Issuer', @@ -71,7 +93,7 @@ def configure_octavia(): issuer_name='OSCI Zaza Octavia Controller', signing_key=controller_cakey) controller_bundle = controller_cert + controller_key - cert_config = { + charm_config = { 'lb-mgmt-issuing-cacert': base64.b64encode( issuing_cacert).decode('utf-8'), 'lb-mgmt-issuing-ca-private-key': base64.b64encode( @@ -81,6 +103,9 @@ def configure_octavia(): controller_cacert).decode('utf-8'), 'lb-mgmt-controller-cert': base64.b64encode( controller_bundle).decode('utf-8'), + 'amp-ssh-key-name': 'octavia', + 'amp-ssh-pub-key': base64.b64encode( + bytes(ssh_public_key, 'utf-8')).decode('utf-8'), } logging.info('Configuring certificates for mandatory Octavia ' 'client/server authentication ' @@ -93,10 +118,17 @@ def configure_octavia(): _singleton = zaza.openstack.charm_tests.test_utils.OpenStackBaseTest() _singleton.setUpClass(application_name='octavia') - with _singleton.config_change(cert_config, cert_config): + with _singleton.config_change(charm_config, charm_config): # wait for configuration to be applied then return pass + # Tell Octavia charm it is safe to create cloud resources + logging.info('Running `configure-resources` action on Octavia leader unit') + zaza.model.run_action_on_leader( + 'octavia', + 'configure-resources', + action_params={}) + def centralized_fip_network(): """Create network with centralized router for connecting lb and fips. From 83e5dc798c6a1b3114b8526d7a3cc1a5aa8eb618 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Tue, 9 Feb 2021 19:47:40 -0800 Subject: [PATCH 790/898] Refactor security checklist test to another class This will allow security checks to be logically separate from other functional tests, and adhere to a similar design pattern that is used in other charm tests (see nova [0]) for an example. It will also highight which security checks are currently passing, and which are not. [0] https://github.com/openstack-charmers/zaza-openstack-tests/blob/35840a66d6788dc4899847747ee7523895f46e1b/zaza/openstack/charm_tests/nova/tests.py#L485-L529 Partial-Bug: #1883196 --- .../charm_tests/openstack_dashboard/tests.py | 53 +++++++++++++++---- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 9588c6f..5944964 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -407,17 +407,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest, self.assertEqual(e.code, 404, msg) logging.info('OK') - def test_501_security_checklist_action(self): - """Verify expected result on a default install. - - Ported from amulet tests. - """ - logging.info("Testing security-checklist") - unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') - action = zaza_model.run_action(unit_name, 'security-checklist') - assert action.data.get(u"status") == "failed", \ - "Security check is expected to not pass by default" - def test_900_restart_on_config_change(self): """Verify that the specified services are restarted on config changed. @@ -520,3 +509,45 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization, result = client.get(_url) if result.status_code == 403: raise policyd.PolicydOperationFailedException("Not authenticated") + + +class SecurityTests(test_utils.OpenStackBaseTest, + OpenStackDashboardBase): + """Openstack-dashboard security tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running openstack-dashboard SecurityTests.""" + super(SecurityTests, cls).setUpClass() + + def test_security_checklist(self): + """Verify expected state with security checklist.""" + logging.info("Testing security checklist.") + + expected_failures = [ + 'csrf_cookie_set', + 'disable_password_reveal', + 'disallow-iframe-embed', + 'password-validator-is-not-default', + 'securie_proxy_ssl_header_is_set', + 'session_cookie-httponly', + 'session-cookie-store', + ] + expected_passes = [ + 'disable_password_autocomplete', + 'enforce-password-check', + 'validate-file-ownership', + 'validate-file-permissions' + ] + + logging.info('Running `security-checklist` action' + ' on {} leader'.format(self.application_name)) + test_utils.audit_assertions( + zaza_model.run_action_on_leader( + self.application_name, + 'security-checklist', + model_name=self.model_name, + action_params={}), + expected_passes, + expected_failures, + expected_to_pass=False) From c327e4c3a294eaeda27c07bafb54ab5a7ba625a4 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 10 Feb 2021 10:22:13 +0100 Subject: [PATCH 791/898] Pin cryptography<3.4 Newer versions require a Rust compiler to build. --- requirements.txt | 5 +++++ setup.py | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cec0286..78a41f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,3 +44,8 @@ paramiko sphinx sphinxcontrib-asyncio git+https://github.com/openstack-charmers/zaza#egg=zaza + +# Newer versions require a Rust compiler to build, see +# * https://github.com/openstack-charmers/zaza/issues/421 +# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html +cryptography<3.4 diff --git a/setup.py b/setup.py index b18cd9a..fb4377d 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,12 @@ install_require = [ 'futurist<2.0.0', 'async_generator', 'boto3', - 'cryptography', + + # Newer versions require a Rust compiler to build, see + # * https://github.com/openstack-charmers/zaza/issues/421 + # * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html + 'cryptography<3.4', + 'dnspython', 'hvac<0.7.0', 'jinja2', From 5b9db73fc7c63979741a52fabff449499a477132 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Wed, 10 Feb 2021 06:40:09 -0800 Subject: [PATCH 792/898] Fix indentation for pep8 --- zaza/openstack/charm_tests/openstack_dashboard/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 5944964..5cd341c 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -512,7 +512,7 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization, class SecurityTests(test_utils.OpenStackBaseTest, - OpenStackDashboardBase): + OpenStackDashboardBase): """Openstack-dashboard security tests.""" @classmethod @@ -541,9 +541,9 @@ class SecurityTests(test_utils.OpenStackBaseTest, ] logging.info('Running `security-checklist` action' - ' on {} leader'.format(self.application_name)) + ' on {} leader'.format(self.application_name)) test_utils.audit_assertions( - zaza_model.run_action_on_leader( + zaza_model.run_action_on_leader( self.application_name, 'security-checklist', model_name=self.model_name, From 1210e0c47e9dc18edca047a2c784ad9409c35589 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Wed, 10 Feb 2021 18:42:32 -0800 Subject: [PATCH 793/898] Skip security_checklist test for xenial_mitaka See bug 1915293 [0] for more details, but this test is already skipped in the repo's current state, this is just making it more explicit as part of the current refactor. [0] https://bugs.launchpad.net/charm-openstack-dashboard/+bug/1915293 --- zaza/openstack/charm_tests/openstack_dashboard/tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 5cd341c..36e7175 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -535,10 +535,15 @@ class SecurityTests(test_utils.OpenStackBaseTest, ] expected_passes = [ 'disable_password_autocomplete', - 'enforce-password-check', 'validate-file-ownership', 'validate-file-permissions' ] + if (openstack_utils.get_os_release() <= + openstack_utils.get_os_release('xenial_mitaka')): + logging.info("Test expected to fail until bug #1915293 is fixed") + expected_failures.append('enforce-password-check') + else: + expected_passes.append('enforce-password-check') logging.info('Running `security-checklist` action' ' on {} leader'.format(self.application_name)) From ce4fe120219a3d23e9a4d917d37df33bee7f6433 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Thu, 11 Feb 2021 12:43:04 -0800 Subject: [PATCH 794/898] Revert "Skip security_checklist test for xenial_mitaka" This reverts commit 1210e0c47e9dc18edca047a2c784ad9409c35589. Rather than using an if statement for gating xenial_mitaka and earlier, test cases can be customized in tests/tests.yaml for individual bundles. This will be handled in review 774305. [0] https://review.opendev.org/c/openstack/charm-openstack-dashboard/+/774305 --- zaza/openstack/charm_tests/openstack_dashboard/tests.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 36e7175..5cd341c 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -535,15 +535,10 @@ class SecurityTests(test_utils.OpenStackBaseTest, ] expected_passes = [ 'disable_password_autocomplete', + 'enforce-password-check', 'validate-file-ownership', 'validate-file-permissions' ] - if (openstack_utils.get_os_release() <= - openstack_utils.get_os_release('xenial_mitaka')): - logging.info("Test expected to fail until bug #1915293 is fixed") - expected_failures.append('enforce-password-check') - else: - expected_passes.append('enforce-password-check') logging.info('Running `security-checklist` action' ' on {} leader'.format(self.application_name)) From af2c6243e2c2ea5429f89cb4f36f3f2e216261de Mon Sep 17 00:00:00 2001 From: David Ames Date: Sat, 6 Feb 2021 00:31:28 +0000 Subject: [PATCH 795/898] Fix race condition in MySQL tests In the MySQL test we destroy a unit and then attempt to remove it from metadata. The metadata removal (remove_instance action) gets stuck when it is started while the destroyed unit has not been entirely removed and is still running MySQL. Wait for all the units to have cluster incomplete workload status message before attempting the remove_instance. --- zaza/openstack/charm_tests/mysql/tests.py | 32 +++++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index d4c89ec..982f0d2 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -753,13 +753,19 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Scale in test: remove leader") leader, nons = self.get_leaders_and_non_leaders() leader_unit = zaza.model.get_unit_from_name(leader) - zaza.model.destroy_unit(self.application_name, leader) - - logging.info("Wait until unit is in waiting state ...") - zaza.model.block_until_unit_wl_status(nons[0], "waiting") + # Wait until we are idle in the hopes clients are not running + # update-status hooks logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() + zaza.model.destroy_unit(self.application_name, leader) + + logging.info("Wait until all only 2 units ...") + zaza.model.block_until_unit_count(self.application, 2) + + logging.info("Wait until all units are cluster incomplete ...") + zaza.model.block_until_wl_status_info_starts_with( + self.application, "'cluster' incomplete") logging.info( "Removing old unit from cluster: {} " @@ -786,6 +792,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Adding unit after removed unit ...") zaza.model.add_unit(self.application_name) + logging.info("Wait until 3 units ...") + zaza.model.block_until_unit_count(self.application, 3) + logging.info("Wait for application states ...") zaza.model.wait_for_application_states(states=self.states) @@ -801,6 +810,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Adding unit after full cluster ...") zaza.model.add_unit(self.application_name) + logging.info("Wait until 4 units ...") + zaza.model.block_until_unit_count(self.application, 4) + logging.info("Wait for application states ...") zaza.model.wait_for_application_states(states=self.states) @@ -810,17 +822,21 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): We start with a four node full cluster, remove one, down to a three node full cluster. """ - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() - leader, nons = self.get_leaders_and_non_leaders() non_leader_unit = zaza.model.get_unit_from_name(nons[0]) - zaza.model.destroy_unit(self.application_name, nons[0]) + # Wait until we are idle in the hopes clients are not running + # update-status hooks logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() + zaza.model.destroy_unit(self.application_name, nons[0]) + logging.info("Scale in test: back down to three") + logging.info("Wait until 3 units ...") + zaza.model.block_until_unit_count(self.application, 3) + + logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.states) logging.info( From 1c4d2e05aff4e56859afec4ce80b4e079a41175e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 18 Feb 2021 09:51:00 +0100 Subject: [PATCH 796/898] Let test_930_scaleback remove 2 units instead of 1 --- zaza/openstack/charm_tests/hacluster/tests.py | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 28d0383..b1c0713 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -137,7 +137,10 @@ class HaclusterScalebackTest(HaclusterBaseTest): class HaclusterScaleBackAndForthTest(HaclusterBaseTest): - """hacluster tests scaling back and forth.""" + """hacluster tests scaling back and forth. + + Supersedes HaclusterScalebackTest. + """ @classmethod def setUpClass(cls): @@ -148,38 +151,46 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): cls._hacluster_charm_name = test_config['hacluster-charm-name'] def test_930_scaleback(self): - """Remove a unit, recalculate quorum and add a new one.""" + """Remove two units, recalculate quorum and re-add two units. + + NOTE(lourot): before lp:1400481 was fixed, quorum wasn't recalculated + when removing units. Within a cluster of 3 units, removing two units + and re-adding them led to a situation where the expected quorum became + 5 and was still considered unreached. + """ principle_units = sorted(zaza.model.get_status().applications[ self._principle_app_name]['units'].keys()) self.assertEqual(len(principle_units), 3) - doomed_principle_unit = principle_units[0] - other_principle_unit = principle_units[1] - doomed_hacluster_unit = juju_utils.get_subordinate_units( - [doomed_principle_unit], charm_name=self._hacluster_charm_name)[0] + doomed_principle_units = principle_units[:2] + other_principle_unit = principle_units[2] + doomed_hacluster_units = juju_utils.get_subordinate_units( + doomed_principle_units, charm_name=self._hacluster_charm_name) other_hacluster_unit = juju_utils.get_subordinate_units( [other_principle_unit], charm_name=self._hacluster_charm_name)[0] - logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) - zaza.model.run_action( - doomed_hacluster_unit, - 'pause', - raise_on_failure=True) - logging.info('OK') + for doomed_hacluster_unit in doomed_hacluster_units: + logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) + zaza.model.run_action( + doomed_hacluster_unit, + 'pause', + raise_on_failure=True) - logging.info('Removing {}'.format(doomed_principle_unit)) - zaza.model.destroy_unit( - self._principle_app_name, - doomed_principle_unit, - wait_disappear=True) - logging.info('OK') + for doomed_principle_unit in doomed_principle_units: + logging.info('Removing {}'.format(doomed_principle_unit)) + zaza.model.destroy_unit( + self._principle_app_name, + doomed_principle_unit, + wait_disappear=True) logging.info('Waiting for model to settle') zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') - zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + # NOTE(lourot): the principle unit (usually a keystone unit) isn't + # guaranteed to be blocked, so we don't validate that. zaza.model.block_until_all_units_idle() - logging.info('OK') logging.info('Updating corosync ring') + # NOTE(lourot): with Juju >= 2.8 this isn't actually necessary as it + # has already been done in hacluster's hanode departing hook: hacluster_app_name = zaza.model.get_unit_from_name( other_hacluster_unit).application zaza.model.run_action_on_leader( @@ -188,9 +199,9 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): action_params={'i-really-mean-it': True}, raise_on_failure=True) - logging.info('Adding an hacluster unit') - zaza.model.add_unit(self._principle_app_name, wait_appear=True) - logging.info('OK') + for article in ('an', 'another'): + logging.info('Adding {} hacluster unit'.format(article)) + zaza.model.add_unit(self._principle_app_name, wait_appear=True) logging.info('Waiting for model to settle') expected_states = {hacluster_app_name: { From f598eb93c29f0c7aec99ac22f1d5d5e5ec7cd7e1 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 18 Feb 2021 10:38:45 +0100 Subject: [PATCH 797/898] Improve comments --- zaza/openstack/charm_tests/hacluster/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index b1c0713..710d3e7 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -185,14 +185,14 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): logging.info('Waiting for model to settle') zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') # NOTE(lourot): the principle unit (usually a keystone unit) isn't - # guaranteed to be blocked, so we don't validate that. + # guaranteed to be blocked, so we don't validate that here. zaza.model.block_until_all_units_idle() logging.info('Updating corosync ring') - # NOTE(lourot): with Juju >= 2.8 this isn't actually necessary as it - # has already been done in hacluster's hanode departing hook: hacluster_app_name = zaza.model.get_unit_from_name( other_hacluster_unit).application + # NOTE(lourot): with Juju >= 2.8 this isn't actually necessary as it + # has already been done in hacluster's hanode departing hook: zaza.model.run_action_on_leader( hacluster_app_name, 'update-ring', From f97c717ba002adb5df348f97dee975884a02a386 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 18 Feb 2021 14:38:01 +0000 Subject: [PATCH 798/898] Add short-term workaround to Trilio bug --- zaza/openstack/charm_tests/trilio/setup.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zaza/openstack/charm_tests/trilio/setup.py b/zaza/openstack/charm_tests/trilio/setup.py index a7ba7b3..44f1d9d 100644 --- a/zaza/openstack/charm_tests/trilio/setup.py +++ b/zaza/openstack/charm_tests/trilio/setup.py @@ -81,3 +81,17 @@ def basic_setup(): else: logging.error("Unable to find Trilio License file") + + +def python2_workaround(): + """Workaround for Bug #1915914. + + Trilio code currently has a bug which assumes an executable called 'python' + will be on the path. To workaround this install a package which adds a + symlink to python + """ + for unit in zaza_model.get_units('trilio-wlm'): + zaza_model.run_on_unit( + unit.entity_id, + ("apt install --yes python-is-python3; " + "systemctl restart wlm\\*.service")) From c00aa18ffbd03dfad9b196b86a54b51e1ba3ef7e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 18 Feb 2021 16:22:43 +0100 Subject: [PATCH 799/898] Assert that corosync considers all nodes to be online --- zaza/openstack/charm_tests/hacluster/tests.py | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 710d3e7..f720d20 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -153,20 +153,22 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): def test_930_scaleback(self): """Remove two units, recalculate quorum and re-add two units. - NOTE(lourot): before lp:1400481 was fixed, quorum wasn't recalculated - when removing units. Within a cluster of 3 units, removing two units - and re-adding them led to a situation where the expected quorum became - 5 and was still considered unreached. + NOTE(lourot): before lp:1400481 was fixed, the corosync ring wasn't + recalculated when removing units. So within a cluster of 3 units, + removing two units and re-adding them led to a situation where corosync + considers having 3 nodes online out of 5, instead of just 3 out of 3. + This test covers this scenario. """ principle_units = sorted(zaza.model.get_status().applications[ self._principle_app_name]['units'].keys()) self.assertEqual(len(principle_units), 3) doomed_principle_units = principle_units[:2] - other_principle_unit = principle_units[2] + surviving_principle_unit = principle_units[2] doomed_hacluster_units = juju_utils.get_subordinate_units( doomed_principle_units, charm_name=self._hacluster_charm_name) - other_hacluster_unit = juju_utils.get_subordinate_units( - [other_principle_unit], charm_name=self._hacluster_charm_name)[0] + surviving_hacluster_unit = juju_utils.get_subordinate_units( + [surviving_principle_unit], + charm_name=self._hacluster_charm_name)[0] for doomed_hacluster_unit in doomed_hacluster_units: logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) @@ -183,14 +185,15 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): wait_disappear=True) logging.info('Waiting for model to settle') - zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') + zaza.model.block_until_unit_wl_status(surviving_hacluster_unit, + 'blocked') # NOTE(lourot): the principle unit (usually a keystone unit) isn't # guaranteed to be blocked, so we don't validate that here. zaza.model.block_until_all_units_idle() logging.info('Updating corosync ring') hacluster_app_name = zaza.model.get_unit_from_name( - other_hacluster_unit).application + surviving_hacluster_unit).application # NOTE(lourot): with Juju >= 2.8 this isn't actually necessary as it # has already been done in hacluster's hanode departing hook: zaza.model.run_action_on_leader( @@ -200,7 +203,7 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): raise_on_failure=True) for article in ('an', 'another'): - logging.info('Adding {} hacluster unit'.format(article)) + logging.info('Re-adding {} hacluster unit'.format(article)) zaza.model.add_unit(self._principle_app_name, wait_appear=True) logging.info('Waiting for model to settle') @@ -208,4 +211,16 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): "workload-status": "active", "workload-status-message": "Unit is ready and clustered"}} zaza.model.wait_for_application_states(states=expected_states) - logging.debug('OK') + + # At this point if the corosync ring has been properly updated, there + # shouldn't be any trace of the deleted units anymore: + logging.info('Checking that corosync considers all nodes to be online') + cmd = 'sudo crm status' + result = zaza.model.run_on_unit(surviving_hacluster_unit, cmd) + code = result.get('Code') + if code != '0': + raise zaza.model.CommandRunFailed(cmd, result) + output = result.get('Stdout').strip() + logging.debug('Output received: {}'.format(output)) + self.assertNotIn('OFFLINE', output, + "corosync shouldn't list any offline node") From 135da1e1a361e01cadfe17df9a25cfa92f4f1b63 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 18 Feb 2021 09:37:49 -0800 Subject: [PATCH 800/898] Fix reboot-cluster-from-complete-outage race --- zaza/openstack/charm_tests/mysql/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 982f0d2..6322684 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -589,6 +589,10 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): unit.entity_id, 'blocked') + # Wait until update-status hooks have completed + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + logging.info("Execute reboot-cluster-from-complete-outage " "action after cold boot ...") # We do not know which unit has the most up to date data From 8c819814fb01741575bee97ab74d202b00947123 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 18 Feb 2021 10:50:53 +0100 Subject: [PATCH 801/898] Don't wait for principle to get back to active --- zaza/openstack/charm_tests/hacluster/tests.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index f720d20..f340022 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -130,8 +130,7 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('Waiting for model to settle') zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active') # NOTE(lourot): the principle application sometimes remain blocked - # after scaling back up until lp:1400481 is solved. - # zaza.model.block_until_unit_wl_status(other_principle_unit, 'active') + # after scaling back up. zaza.model.block_until_all_units_idle() logging.debug('OK') @@ -207,10 +206,15 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): zaza.model.add_unit(self._principle_app_name, wait_appear=True) logging.info('Waiting for model to settle') - expected_states = {hacluster_app_name: { - "workload-status": "active", - "workload-status-message": "Unit is ready and clustered"}} - zaza.model.wait_for_application_states(states=expected_states) + # NOTE(lourot): the principle charm may remain blocked here. This seems + # to happen often when it is keystone and has a mysql-router as other + # subordinate charm. The keystone units seems to often remain blocked + # with 'Database not initialised'. This is not the hacluster charm's + # fault and this is why we don't validate here that the entire model + # goes back to active/idle. + zaza.model.block_until_unit_wl_status(surviving_hacluster_unit, + 'active') + zaza.model.block_until_all_units_idle() # At this point if the corosync ring has been properly updated, there # shouldn't be any trace of the deleted units anymore: From d728458afa1dca1ca4bbf59541ce248e78f26bbc Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Fri, 8 Jan 2021 15:59:28 +0000 Subject: [PATCH 802/898] Add multi-backend testing for Keystone SAML Mellon The new updated tests rely on new testing bundles deployed with two local IdPs via the Juju charm https://jaas.ai/u/ionutbalutoiu/test-saml-idp. --- .../charm_tests/saml_mellon/setup.py | 121 +++++++++-- .../charm_tests/saml_mellon/tests.py | 199 ++++++++++++++++++ zaza/openstack/utilities/generic.py | 26 +++ 3 files changed, 329 insertions(+), 17 deletions(-) diff --git a/zaza/openstack/charm_tests/saml_mellon/setup.py b/zaza/openstack/charm_tests/saml_mellon/setup.py index 7e5f96c..20b8cbd 100644 --- a/zaza/openstack/charm_tests/saml_mellon/setup.py +++ b/zaza/openstack/charm_tests/saml_mellon/setup.py @@ -25,6 +25,7 @@ from zaza.openstack.utilities import ( cert as cert_utils, cli as cli_utils, openstack as openstack_utils, + generic as generic_utils, ) @@ -34,8 +35,8 @@ FEDERATED_DOMAIN = "federated_domain" FEDERATED_GROUP = "federated_users" MEMBER = "Member" IDP = "samltest" +LOCAL_IDP_REMOTE_ID = "http://{}/simplesaml/saml2/idp/metadata.php" REMOTE_ID = "https://samltest.id/saml/idp" -MAP_NAME = "{}_mapping".format(IDP) PROTOCOL_NAME = "mapped" MAP_TEMPLATE = ''' [{{ @@ -45,7 +46,7 @@ MAP_TEMPLATE = ''' "name": "{{0}}" }}, "group": {{ - "name": "federated_users", + "name": "{group_id}", "domain": {{ "id": "{domain_id}" }} @@ -55,7 +56,7 @@ MAP_TEMPLATE = ''' "name": "{{0}}_project", "roles": [ {{ - "name": "Member" + "name": "{role_name}" }} ] }} @@ -81,7 +82,10 @@ SP_SIGNING_KEY_INFO_XML_TEMPLATE = ''' ''' -def keystone_federation_setup(): +def keystone_federation_setup(federated_domain=FEDERATED_DOMAIN, + federated_group=FEDERATED_GROUP, + idp_name=IDP, + idp_remote_id=REMOTE_ID): """Configure Keystone Federation.""" cli_utils.setup_logging() keystone_session = openstack_utils.get_overcloud_keystone_session() @@ -89,19 +93,19 @@ def keystone_federation_setup(): keystone_session) try: - domain = keystone_client.domains.find(name=FEDERATED_DOMAIN) + domain = keystone_client.domains.find(name=federated_domain) except keystoneauth1.exceptions.http.NotFound: domain = keystone_client.domains.create( - FEDERATED_DOMAIN, + federated_domain, description="Federated Domain", enabled=True) try: group = keystone_client.groups.find( - name=FEDERATED_GROUP, domain=domain) + name=federated_group, domain=domain) except keystoneauth1.exceptions.http.NotFound: group = keystone_client.groups.create( - FEDERATED_GROUP, + federated_group, domain=domain, enabled=True) @@ -109,30 +113,33 @@ def keystone_federation_setup(): keystone_client.roles.grant(role, group=group, domain=domain) try: - idp = keystone_client.federation.identity_providers.find( - name=IDP, domain_id=domain.id) + idp = keystone_client.federation.identity_providers.get(idp_name) except keystoneauth1.exceptions.http.NotFound: idp = keystone_client.federation.identity_providers.create( - IDP, - remote_ids=[REMOTE_ID], + idp_name, + remote_ids=[idp_remote_id], domain_id=domain.id, enabled=True) - JSON_RULES = json.loads(MAP_TEMPLATE.format(domain_id=domain.id)) + JSON_RULES = json.loads(MAP_TEMPLATE.format( + domain_id=domain.id, group_id=group.id, role_name=MEMBER)) + map_name = "{}_mapping".format(idp_name) try: - keystone_client.federation.mappings.find(name=MAP_NAME) + keystone_client.federation.mappings.get(map_name) except keystoneauth1.exceptions.http.NotFound: keystone_client.federation.mappings.create( - MAP_NAME, rules=JSON_RULES) + map_name, rules=JSON_RULES) try: - keystone_client.federation.protocols.get(IDP, PROTOCOL_NAME) + keystone_client.federation.protocols.get(idp_name, PROTOCOL_NAME) except keystoneauth1.exceptions.http.NotFound: keystone_client.federation.protocols.create( - PROTOCOL_NAME, mapping=MAP_NAME, identity_provider=idp) + PROTOCOL_NAME, mapping=map_name, identity_provider=idp) +# This setup method is deprecated. It will be removed once we fully drop the +# `samltest.id` dependency. def attach_saml_resources(application="keystone-saml-mellon"): """Attach resource to the Keystone SAML Mellon charm.""" test_idp_metadata_xml = "samltest.xml" @@ -161,3 +168,83 @@ def attach_saml_resources(application="keystone-saml-mellon"): fp.flush() zaza.model.attach_resource( application, sp_signing_keyinfo_name, fp.name) + + +def _attach_saml_resources_local_idp(keystone_saml_mellon_app_name=None, + test_saml_idp_app_name=None): + """Attach resources to the Keystone SAML Mellon and the local IdP.""" + action_result = zaza.model.run_action_on_leader( + test_saml_idp_app_name, 'get-idp-metadata') + idp_metadata = action_result.data['results']['output'] + + generic_utils.attach_file_resource( + keystone_saml_mellon_app_name, + 'idp-metadata', + idp_metadata, + '.xml') + + (key, cert) = cert_utils.generate_cert('SP Signing Key') + + cert = cert.decode().strip("-----BEGIN CERTIFICATE-----") + cert = cert.strip("-----END CERTIFICATE-----") + + generic_utils.attach_file_resource( + keystone_saml_mellon_app_name, + 'sp-private-key', + key.decode(), + '.pem') + generic_utils.attach_file_resource( + keystone_saml_mellon_app_name, + 'sp-signing-keyinfo', + SP_SIGNING_KEY_INFO_XML_TEMPLATE.format(cert), + '.xml') + + action_result = zaza.model.run_action_on_leader( + keystone_saml_mellon_app_name, 'get-sp-metadata') + sp_metadata = action_result.data['results']['output'] + + generic_utils.attach_file_resource( + test_saml_idp_app_name, + 'sp-metadata', + sp_metadata, + '.xml') + + +def attach_saml_resources_idp1(): + """Attach the SAML resources for the local IdP #1.""" + _attach_saml_resources_local_idp( + keystone_saml_mellon_app_name="keystone-saml-mellon1", + test_saml_idp_app_name="test-saml-idp1") + + +def attach_saml_resources_idp2(): + """Attach the SAML resources for the local IdP #2.""" + _attach_saml_resources_local_idp( + keystone_saml_mellon_app_name="keystone-saml-mellon2", + test_saml_idp_app_name="test-saml-idp2") + + +def keystone_federation_setup_idp1(): + """Configure Keystone Federation for the local IdP #1.""" + test_saml_idp_unit = zaza.model.get_units("test-saml-idp1")[0] + idp_remote_id = LOCAL_IDP_REMOTE_ID.format( + test_saml_idp_unit.public_address) + + keystone_federation_setup( + federated_domain="federated_domain_idp1", + federated_group="federated_users_idp1", + idp_name="test-saml-idp1", + idp_remote_id=idp_remote_id) + + +def keystone_federation_setup_idp2(): + """Configure Keystone Federation for the local IdP #2.""" + test_saml_idp_unit = zaza.model.get_units("test-saml-idp2")[0] + idp_remote_id = LOCAL_IDP_REMOTE_ID.format( + test_saml_idp_unit.public_address) + + keystone_federation_setup( + federated_domain="federated_domain_idp2", + federated_group="federated_users_idp2", + idp_name="test-saml-idp2", + idp_remote_id=idp_remote_id) diff --git a/zaza/openstack/charm_tests/saml_mellon/tests.py b/zaza/openstack/charm_tests/saml_mellon/tests.py index 9b5b211..962fa92 100644 --- a/zaza/openstack/charm_tests/saml_mellon/tests.py +++ b/zaza/openstack/charm_tests/saml_mellon/tests.py @@ -30,6 +30,8 @@ class FailedToReachIDP(Exception): pass +# This testing class is deprecated. It will be removed once we fully drop the +# `samltest.id` dependency. class CharmKeystoneSAMLMellonTest(BaseKeystoneTest): """Charm Keystone SAML Mellon tests.""" @@ -156,3 +158,200 @@ class CharmKeystoneSAMLMellonTest(BaseKeystoneTest): # We may need to try/except to allow horizon to build its pages _do_redirect_check(url, region, idp_expect, horizon_expect) logging.info("SUCCESS") + + +class BaseCharmKeystoneSAMLMellonTest(BaseKeystoneTest): + """Charm Keystone SAML Mellon tests.""" + + @classmethod + def setUpClass(cls, + application_name="keystone-saml-mellon", + test_saml_idp_app_name="test-saml-idp", + horizon_idp_option_name="myidp_mapped", + horizon_idp_display_name="myidp via mapped"): + """Run class setup for running Keystone SAML Mellon charm tests.""" + super(BaseCharmKeystoneSAMLMellonTest, cls).setUpClass() + # Note: The BaseKeystoneTest class sets the application_name to + # "keystone" which breaks keystone-saml-mellon actions. Explicitly set + # application name here. + cls.application_name = application_name + cls.test_saml_idp_app_name = test_saml_idp_app_name + cls.horizon_idp_option_name = horizon_idp_option_name + cls.horizon_idp_display_name = horizon_idp_display_name + cls.action = "get-sp-metadata" + cls.current_release = openstack_utils.get_os_release() + cls.FOCAL_USSURI = openstack_utils.get_os_release("focal_ussuri") + + @staticmethod + def check_horizon_redirect(horizon_url, horizon_expect, + horizon_idp_option_name, horizon_region, + idp_url, idp_expect): + """Validate the Horizon -> Keystone -> IDP redirects. + + This validation is done through `requests.session()`, and the proper + get / post http calls. + + :param horizon_url: The login page for the Horizon OpenStack dashboard. + :type horizon_url: string + :param horizon_expect: Information that needs to be displayed by + Horizon login page, when there is a proper + SAML IdP configuration. + :type horizon_expect: string + :param horizon_idp_option_name: The name of the IdP that is chosen + in the Horizon dropdown from the login + screen. This will go in the post body + as 'auth_type'. + :type horizon_idp_option_name: string + :param horizon_region: Information needed to complete the http post + data for the Horizon login. + :type horizon_region: string + :param idp_url: The url for the IdP where the user needs to be + redirected. + :type idp_url: string + :param idp_expect: Information that needs to be displayed by the IdP + after the user is redirected there. + :type idp_expect: string + :returns: None + """ + # start session, get csrftoken + client = requests.session() + # Verify=False see note below + login_page = client.get(horizon_url, verify=False) + + # Validate SAML method is available + assert horizon_expect in login_page.text + + # Get cookie + if "csrftoken" in client.cookies: + csrftoken = client.cookies["csrftoken"] + else: + raise Exception("Missing csrftoken") + + # Build and send post request + form_data = { + "auth_type": horizon_idp_option_name, + "csrfmiddlewaretoken": csrftoken, + "next": "/horizon/project/api_access", + "region": horizon_region, + } + + # Verify=False due to CA certificate bundles. + # If we don't set it validation fails for keystone/horizon + # We would have to install the keystone CA onto the system + # to validate end to end. + response = client.post( + horizon_url, + data=form_data, + headers={"Referer": horizon_url}, + allow_redirects=True, + verify=False) + + if idp_expect not in response.text: + msg = "FAILURE code={} text={}".format(response, response.text) + # Raise a custom exception. + raise FailedToReachIDP(msg) + + # Validate that we were redirected to the proper IdP + assert response.url.startswith(idp_url) + assert idp_url in response.text + + def test_run_get_sp_metadata_action(self): + """Validate the get-sp-metadata action.""" + unit = zaza.model.get_units(self.application_name)[0] + ip = self.vip if self.vip else unit.public_address + + action = zaza.model.run_action(unit.entity_id, self.action) + self.assertNotIn( + "failed", + action.data["status"], + msg="The action failed: {}".format(action.data["message"])) + + output = action.data["results"]["output"] + root = etree.fromstring(output) + for item in root.items(): + if "entityID" in item[0]: + self.assertIn(ip, item[1]) + + for appt in root.getchildren(): + for elem in appt.getchildren(): + for item in elem.items(): + if "Location" in item[0]: + self.assertIn(ip, item[1]) + + logging.info("Successul get-sp-metadata action") + + def test_saml_mellon_redirects(self): + """Validate the horizon -> keystone -> IDP redirects.""" + unit = zaza.model.get_units(self.application_name)[0] + keystone_ip = self.vip if self.vip else unit.public_address + + horizon = "openstack-dashboard" + horizon_config = zaza.model.get_application_config(horizon) + horizon_vip = horizon_config.get("vip").get("value") + unit = zaza.model.get_units("openstack-dashboard")[0] + + horizon_ip = horizon_vip if horizon_vip else unit.public_address + proto = "https" if self.tls_rid else "http" + + # Use Keystone URL for < Focal + if self.current_release < self.FOCAL_USSURI: + region = "{}://{}:5000/v3".format(proto, keystone_ip) + else: + region = "default" + + idp_address = zaza.model.get_units( + self.test_saml_idp_app_name)[0].public_address + + horizon_url = "{}://{}/horizon/auth/login/".format(proto, horizon_ip) + horizon_expect = ''.format( + self.horizon_idp_option_name, self.horizon_idp_display_name) + idp_url = ("http://{0}/simplesaml/" + "module.php/core/loginuserpass.php").format(idp_address) + # This is the message the local test-saml-idp displays after you are + # redirected. It shows we have been directed to: + # horizon -> keystone -> test-saml-idp + idp_expect = ( + "A service has requested you to authenticate yourself. Please " + "enter your username and password in the form below.") + + # Execute the check + BaseCharmKeystoneSAMLMellonTest.check_horizon_redirect( + horizon_url=horizon_url, + horizon_expect=horizon_expect, + horizon_idp_option_name=self.horizon_idp_option_name, + horizon_region=region, + idp_url=idp_url, + idp_expect=idp_expect) + logging.info("SUCCESS") + + +class CharmKeystoneSAMLMellonIDP1Test(BaseCharmKeystoneSAMLMellonTest): + """Charm Keystone SAML Mellon tests class for the local IDP #1.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Keystone SAML Mellon charm tests. + + It does the necessary setup for the local IDP #1. + """ + super(CharmKeystoneSAMLMellonIDP1Test, cls).setUpClass( + application_name="keystone-saml-mellon1", + test_saml_idp_app_name="test-saml-idp1", + horizon_idp_option_name="test-saml-idp1_mapped", + horizon_idp_display_name="Test SAML IDP #1") + + +class CharmKeystoneSAMLMellonIDP2Test(BaseCharmKeystoneSAMLMellonTest): + """Charm Keystone SAML Mellon tests class for the local IDP #2.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Keystone SAML Mellon charm tests. + + It does the necessary setup for the local IDP #2. + """ + super(CharmKeystoneSAMLMellonIDP2Test, cls).setUpClass( + application_name="keystone-saml-mellon2", + test_saml_idp_app_name="test-saml-idp2", + horizon_idp_option_name="test-saml-idp2_mapped", + horizon_idp_display_name="Test SAML IDP #2") diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index dd8b1e0..fcd7989 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -20,6 +20,7 @@ import os import socket import subprocess import telnetlib +import tempfile import yaml from zaza import model @@ -680,3 +681,28 @@ def get_mojo_cacert_path(): return cacert else: raise zaza_exceptions.CACERTNotFound("Could not find cacert.pem") + + +def attach_file_resource(application_name, resource_name, + file_content, file_suffix=".txt"): + """Attaches a file as a Juju resource given the file content and suffix. + + The file content will be written into a temporary file with the given + suffix, and it will be attached to the Juju application. + + :param application_name: Juju application name. + :type application_name: string + :param resource_name: Juju resource name. + :type resource_name: string + :param file_content: The content of the file that will be attached + :type file_content: string + :param file_suffix: File suffix. This should be used to set the file + extension for applications that are sensitive to this. + :type file_suffix: string + :returns: None + """ + with tempfile.NamedTemporaryFile(mode='w', suffix=file_suffix) as fp: + fp.write(file_content) + fp.flush() + model.attach_resource( + application_name, resource_name, fp.name) From d41f8b37240b794442c873327696ffe2ed9d4fbb Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Wed, 20 Jan 2021 20:10:34 +0200 Subject: [PATCH 803/898] Update Ceph RBD Mirror tests The updated tests add the possibility of testing deployments with `image` RBD mirroring mode implemented as part of the Cinder Ceph Replication charm spec. --- .../charm_tests/ceph/rbd_mirror/tests.py | 308 ++++++++++++++---- 1 file changed, 242 insertions(+), 66 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py index 6c2fa1b..75a3f42 100644 --- a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py @@ -16,6 +16,9 @@ import json import logging import re +import time + +import cinderclient.exceptions as cinder_exceptions import zaza.openstack.charm_tests.test_utils as test_utils @@ -39,16 +42,20 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): cls.site_a_model = cls.site_b_model = zaza.model.get_juju_model() cls.site_b_app_suffix = '-b' - def run_status_action(self, application_name=None, model_name=None): + def run_status_action(self, application_name=None, model_name=None, + pools=[]): """Run status action, decode and return response.""" + action_params = { + 'verbose': True, + 'format': 'json', + } + if len(pools) > 0: + action_params['pools'] = ','.join(pools) result = zaza.model.run_action_on_leader( application_name or self.application_name, 'status', model_name=model_name, - action_params={ - 'verbose': True, - 'format': 'json', - }) + action_params=action_params) return json.loads(result.results['output']) def get_pools(self): @@ -71,7 +78,8 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): def wait_for_mirror_state(self, state, application_name=None, model_name=None, check_entries_behind_master=False, - require_images_in=[]): + require_images_in=[], + pools=[]): """Wait until all images reach requested state. This function runs the ``status`` action and examines the data it @@ -90,6 +98,9 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): :type check_entries_behind_master: bool :param require_images_in: List of pools to require images in :type require_images_in: list of str + :param pools: List of pools to run status on. If this is empty, the + status action will run on all the pools. + :type pools: list of str :returns: True on success, never returns on failure """ rep = re.compile(r'.*entries_behind_master=(\d+)') @@ -97,7 +108,8 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): try: # encapsulate in try except to work around LP: #1820976 pool_status = self.run_status_action( - application_name=application_name, model_name=model_name) + application_name=application_name, model_name=model_name, + pools=pools) except KeyError: continue for pool, status in pool_status.items(): @@ -124,6 +136,119 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): # all images with state has expected state return True + def get_cinder_rbd_mirroring_mode(self, + cinder_ceph_app_name='cinder-ceph'): + """Get the RBD mirroring mode for the Cinder Ceph pool. + + :returns: A string representing the RBD mirroring mode. It can be + either 'pool' or 'image'. + """ + DEFAULT_RBD_MIRRORING_MODE = 'pool' + + rbd_mirroring_mode_config = zaza.model.get_application_config( + cinder_ceph_app_name).get('rbd-mirroring-mode') + if rbd_mirroring_mode_config: + rbd_mirroring_mode = rbd_mirroring_mode_config.get( + 'value', DEFAULT_RBD_MIRRORING_MODE).lower() + else: + rbd_mirroring_mode = DEFAULT_RBD_MIRRORING_MODE + + return rbd_mirroring_mode + + def create_cinder_volume(self, session, from_image=False): + """Create Cinder Volume from image. + + :rtype: :class:`Volume`. + """ + def get_glance_image(session): + glance = openstack.get_glance_session_client(session) + images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) + if images: + return images[0] + logging.info("Failed to find {} image, falling back to {}".format( + CIRROS_IMAGE_NAME, + LTS_IMAGE_NAME)) + return openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] + + def create_volume_type(cinder): + try: + vol_type = cinder.volume_types.find(name='repl') + except cinder_exceptions.NotFound: + vol_type = cinder.volume_types.create('repl') + vol_type.set_keys(metadata={ + 'volume_backend_name': 'cinder-ceph', + 'replication_enabled': ' True', + }) + return vol_type + + # NOTE(fnordahl): for some reason create volume from image often fails + # when run just after deployment is finished. We should figure out + # why, resolve the underlying issue and then remove this. + # + # We do not use tenacity here as it will interfere with tenacity used + # in ``resource_reaches_status`` + def create_volume(cinder, volume_params, retry=20): + if retry < 1: + return + volume = cinder.volumes.create(**volume_params) + try: + # Note(coreycb): stop_after_attempt is increased because using + # juju storage for ceph-osd backed by cinder on undercloud + # takes longer than the prior method of directory-backed OSD + # devices. + openstack.resource_reaches_status( + cinder.volumes, volume.id, msg='volume', + stop_after_attempt=20) + return volume + except AssertionError: + logging.info('retrying') + volume.delete() + return create_volume(cinder, volume_params, retry=retry - 1) + + volume_params = { + 'size': 8, + 'name': 'zaza', + } + if from_image: + volume_params['imageRef'] = get_glance_image(session).id + cinder = openstack.get_cinder_session_client(session) + if self.get_cinder_rbd_mirroring_mode() == 'image': + volume_params['volume_type'] = create_volume_type(cinder).id + + return create_volume(cinder, volume_params) + + def failover_cinder_volume_host(self, cinder_client, + backend_name='cinder-ceph', + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over', + timeout=300): + """Failover Cinder volume host.""" + host = 'cinder@{}'.format(backend_name) + logging.info( + 'Failover Cinder host %s to backend_id %s', + host, target_backend_id) + cinder_client.services.failover_host( + host=host, + backend_id=target_backend_id) + start = time.time() + while True: + elapsed = time.time() - start + if elapsed > timeout: + raise cinder_exceptions.TimeoutException( + obj=cinder_client.services, + action='failover_host') + service = cinder_client.services.list( + host=host, + binary='cinder-volume')[0] + if (service.status == target_status and + service.replication_status == target_replication_status): + break + time.sleep(5) + logging.info( + 'Successfully failed-over Cinder host %s to backend_id %s', + host, target_backend_id) + class CephRBDMirrorTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` tests.""" @@ -196,43 +321,7 @@ class CephRBDMirrorTest(CephRBDMirrorBase): test. """ session = openstack.get_overcloud_keystone_session() - glance = openstack.get_glance_session_client(session) - cinder = openstack.get_cinder_session_client(session) - - images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) - if images: - image = images[0] - else: - logging.info("Failed to find {} image, falling back to {}".format( - CIRROS_IMAGE_NAME, - LTS_IMAGE_NAME)) - image = openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] - - # NOTE(fnordahl): for some reason create volume from image often fails - # when run just after deployment is finished. We should figure out - # why, resolve the underlying issue and then remove this. - # - # We do not use tenacity here as it will interfere with tenacity used - # in ``resource_reaches_status`` - def create_volume_from_image(cinder, image, retry=20): - if retry < 1: - return - volume = cinder.volumes.create(8, name='zaza', imageRef=image.id) - try: - # Note(coreycb): stop_after_attempt is increased because using - # juju storage for ceph-osd backed by cinder on undercloud - # takes longer than the prior method of directory-backed OSD - # devices. - openstack.resource_reaches_status( - cinder.volumes, volume.id, msg='volume', - stop_after_attempt=20) - return volume - except AssertionError: - logging.info('retrying') - volume.delete() - return create_volume_from_image(cinder, image, retry=retry - 1) - volume = create_volume_from_image(cinder, image) - + volume = self.create_cinder_volume(session, from_image=True) site_a_hash = zaza.openstack.utilities.ceph.get_rbd_hash( zaza.model.get_lead_unit_name('ceph-mon', model_name=self.site_a_model), @@ -258,85 +347,170 @@ class CephRBDMirrorTest(CephRBDMirrorBase): class CephRBDMirrorControlledFailoverTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` controlled failover tests.""" + def cinder_fail_over_fall_back(self): + """Validate controlled fail over and fall back via the Cinder API.""" + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session) + volume = self.create_cinder_volume(session, from_image=True) + self.wait_for_mirror_state( + 'up+replaying', + check_entries_behind_master=True, + application_name=self.application_name + self.site_b_app_suffix, + model_name=self.site_b_model, + pools=['cinder-ceph']) + self.failover_cinder_volume_host( + cinder_client=cinder) + self.assertEqual(cinder.volumes.get(volume.id).status, 'available') + self.failover_cinder_volume_host( + cinder_client=cinder, + target_backend_id='default', + target_status='enabled', + target_replication_status='enabled') + self.assertEqual(cinder.volumes.get(volume.id).status, 'available') + def test_fail_over_fall_back(self): """Validate controlled fail over and fall back.""" site_a_pools, site_b_pools = self.get_pools() + site_a_action_params = {} + site_b_action_params = {} + if self.get_cinder_rbd_mirroring_mode() == 'image': + site_a_pools.remove('cinder-ceph') + site_a_action_params['pools'] = ','.join(site_a_pools) + site_b_pools.remove('cinder-ceph') + site_b_action_params['pools'] = ','.join(site_b_pools) result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror', 'demote', model_name=self.site_a_model, - action_params={}) + action_params=site_a_action_params) logging.info(result.results) n_pools_demoted = len(result.results['output'].split('\n')) self.assertEqual(len(site_a_pools), n_pools_demoted) - self.wait_for_mirror_state('up+unknown', model_name=self.site_a_model) + self.wait_for_mirror_state( + 'up+unknown', + model_name=self.site_a_model, + pools=site_a_pools) self.wait_for_mirror_state( 'up+unknown', application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model) + model_name=self.site_b_model, + pools=site_b_pools) result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror' + self.site_b_app_suffix, 'promote', model_name=self.site_b_model, - action_params={}) + action_params=site_b_action_params) logging.info(result.results) n_pools_promoted = len(result.results['output'].split('\n')) self.assertEqual(len(site_b_pools), n_pools_promoted) self.wait_for_mirror_state( 'up+replaying', - model_name=self.site_a_model) + model_name=self.site_a_model, + pools=site_a_pools) self.wait_for_mirror_state( 'up+stopped', application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model) + model_name=self.site_b_model, + pools=site_b_pools) result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror' + self.site_b_app_suffix, 'demote', model_name=self.site_b_model, - action_params={ - }) + action_params=site_b_action_params) logging.info(result.results) n_pools_demoted = len(result.results['output'].split('\n')) self.assertEqual(len(site_a_pools), n_pools_demoted) self.wait_for_mirror_state( 'up+unknown', - model_name=self.site_a_model) + model_name=self.site_a_model, + pools=site_a_pools) self.wait_for_mirror_state( 'up+unknown', application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model) + model_name=self.site_b_model, + pools=site_b_pools) result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror', 'promote', model_name=self.site_a_model, - action_params={ - }) + action_params=site_a_action_params) logging.info(result.results) n_pools_promoted = len(result.results['output'].split('\n')) self.assertEqual(len(site_b_pools), n_pools_promoted) self.wait_for_mirror_state( 'up+stopped', - model_name=self.site_a_model) + model_name=self.site_a_model, + pools=site_a_pools) + action_params = { + 'i-really-mean-it': True, + } + if self.get_cinder_rbd_mirroring_mode() == 'image': + action_params['pools'] = site_b_action_params['pools'] result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror' + self.site_b_app_suffix, 'resync-pools', model_name=self.site_b_model, - action_params={ - 'i-really-mean-it': True, - }) + action_params=action_params) logging.info(result.results) self.wait_for_mirror_state( 'up+replaying', application_name=self.application_name + self.site_b_app_suffix, model_name=self.site_b_model, - require_images_in=['cinder-ceph', 'glance']) + require_images_in=['cinder-ceph', 'glance'], + pools=site_a_pools) + if self.get_cinder_rbd_mirroring_mode() == 'image': + self.cinder_fail_over_fall_back() class CephRBDMirrorDisasterFailoverTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` destructive tests.""" + def forced_failover_cinder_volume_host(self, cinder_client): + """Validate forced Cinder volume host fail over.""" + def apply_cinder_workaround(): + """Set minimal timeouts / retries to the Cinder Ceph backend. + + This is needed because the failover via Cinder will try to do a + demotion of the site-a, and with the default timeouts / retries, + the operation takes an unreasonably amount of time. + """ + cinder_configs = { + 'rados_connect_timeout': '1', + 'rados_connection_retries': '1', + 'rados_connection_interval': '0', + 'replication_connect_timeout': '1', + } + update_cinder_conf_cmd = ( + "import configparser; " + "config = configparser.ConfigParser(); " + "config.read('/etc/cinder/cinder.conf'); " + "{}" + "f = open('/etc/cinder/cinder.conf', 'w'); " + "config.write(f); " + "f.close()") + cmd = '' + for config in cinder_configs: + cmd += "config.set('cinder-ceph', '{0}', '{1}'); ".format( + config, cinder_configs[config]) + cmd = update_cinder_conf_cmd.format(cmd) + zaza.model.run_on_leader( + 'cinder-ceph', + 'python3 -c "{}"; systemctl restart cinder-volume'.format(cmd)) + + apply_cinder_workaround() + self.failover_cinder_volume_host(cinder_client) + + for volume in cinder_client.volumes.list(): + self.assertEqual(volume.status, 'available') + def test_kill_site_a_fail_over(self): """Validate fail over after uncontrolled shutdown of primary.""" + action_params = {} + if self.get_cinder_rbd_mirroring_mode() == 'image': + _, site_b_pools = self.get_pools() + site_b_pools.remove('cinder-ceph') + action_params['pools'] = ','.join(site_b_pools) + for application in 'ceph-rbd-mirror', 'ceph-mon', 'ceph-osd': zaza.model.remove_application( application, @@ -346,14 +520,16 @@ class CephRBDMirrorDisasterFailoverTest(CephRBDMirrorBase): 'ceph-rbd-mirror' + self.site_b_app_suffix, 'promote', model_name=self.site_b_model, - action_params={ - }) + action_params=action_params) self.assertEqual(result.status, 'failed') + action_params['force'] = True result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror' + self.site_b_app_suffix, 'promote', model_name=self.site_b_model, - action_params={ - 'force': True, - }) + action_params=action_params) self.assertEqual(result.status, 'completed') + if self.get_cinder_rbd_mirroring_mode() == 'image': + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session) + self.forced_failover_cinder_volume_host(cinder) From 2054b7c4775ea8c79d99d60f23a684f6e1910db3 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 25 Feb 2021 09:16:34 +0000 Subject: [PATCH 804/898] Add test for deferred restarts --- zaza/openstack/charm_tests/neutron/tests.py | 14 ++ zaza/openstack/charm_tests/test_utils.py | 236 ++++++++++++++++++++ 2 files changed, 250 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index bebe3b6..412a352 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -952,3 +952,17 @@ class NeutronNetworkingVRRPTests(NeutronNetworkingBase): uc_neutron_client, gateway_hostname) self.check_connectivity(instance_1, instance_2) + + +class NeutronOVSDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super(NeutronOVSDeferredRestartTest, cls).setUpClass( + restart_config_file='/etc/neutron/neutron.conf', + test_service='neutron-openvswitch-agent', + restart_package='openvswitch-switch', + restart_package_service='openvswitch-switch', + application_name='neutron-openvswitch') diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 5a3e6c1..bd82a38 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -18,6 +18,7 @@ import subprocess import sys import tenacity import unittest +import yaml import novaclient @@ -701,3 +702,238 @@ class OpenStackBaseTest(BaseCharmTest): instance_2 = self.retrieve_guest( '{}-ins-1'.format(self.RESOURCE_PREFIX)) return instance_1, instance_2 + + +class BaseDeferredRestartTest(OpenStackBaseTest): + """Check deferred restarts. + + Example of adding a deferred restart test:: + + class NeutronOVSDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + + @classmethod + def setUpClass(cls): + super(NeutronOVSDeferredRestartTest, cls).setUpClass( + restart_config_file = '/etc/neutron/neutron.conf', + test_service = 'neutron-openvswitch-agent', + restart_package = 'openvswitch-switch', + restart_package_service = 'openvswitch-switch', + application_name = 'neutron-openvswitch') + + NOTE: The test has been broken into various class methods which may require + specialisation if the charm being tested is not a standard OpenStack + charm e.g. `trigger_deferred_restart_via_charm` if the charm is not + an oslo config or does not have a debug option. + """ + + @classmethod + def setUpClass(cls, restart_config_file, test_service, restart_package, + restart_package_service, application_name): + """Run test setup. + + :param restart_config_file: Config file that will be changed to trigger + a service restart. + :type restart_config_file: str + :param test_service: Service that will require a restart after + restart_config_file has changed. + :type test_service: str + :param restart_package: Package that will be changed to trigger a + service restart. + :type restart_package: str + :param restart_package_service: Service that will require a restart + after restart_package has changed. + :type restart_package_service: str + :param application_name: Name of application to run tests against. + :type application_name: str + """ + cls.restart_config_file = restart_config_file + cls.test_service = test_service + cls.restart_package = restart_package + cls.restart_package_service = restart_package_service + cls.application_name = application_name + super(BaseDeferredRestartTest, cls).setUpClass( + application_name=cls.application_name) + + def check_clear_restarts(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in model.get_units(self.application_name): + model.run_action( + unit.entity_id, + 'restart-services', + action_params={'deferred': True}) + + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + assert unit.workload_status_message == 'Unit is ready' + + def check_show_deferred_restarts_action(self, test_service, + restart_reason): + """Check the output from the action to list deferred restarts. + + Run the action to list any deferred restarts and check it has entry for + the given service and reason. + + :param test_service: Service that should need a restart + :type test_service: str + :param restart_reason: The reason the action should list for the + service needing to be restarted. This can be a + substring. + :type restart_reason: str + """ + # Ensure that the deferred restart and cause are listed via action + for unit in model.get_units(self.application_name): + action = model.run_action( + unit.entity_id, + 'show-deferred-restarts') + logging.info( + ("Checking {} is marked as needing restart in " + "show-deferred-restarts action on {}").format( + test_service, + unit.entity_id)) + for event in yaml.safe_load(action.data['results']['output']): + if test_service in event and restart_reason in event: + break + else: + msg = 'No entry for restart of {} for reason {} found'.format( + test_service, + restart_reason) + raise Exception(msg) + + def check_show_deferred_restarts_wlm(self, test_service): + """Check the workload status message lists deferred restart. + + :param test_service: Service that should need a restart + :type test_service: str + """ + # Ensure that the deferred restarts are visible in Juju status + for unit in model.get_units(self.application_name): + # Just checking one example service should we be checking all? + logging.info( + ("Checking {} is marked as needing restart in workload " + "message of {}".format(test_service, unit.entity_id))) + assert test_service in unit.workload_status_message + + def trigger_deferred_restart_via_charm(self): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be renderred in + applications config file. + + NOTE: The implementation assumes the charm has a `debug` option and + self.restart_config_file in an oslo config file where that + debug option is renderred. If that is not true the specaliasation + class should override this method. + """ + app_config = model.get_application_config(self.application_name) + logging.info("Triggering deferred restart via config change") + new_debug_value = str(not app_config['debug']['value']) + logging.info("Setting debug: {}".format(new_debug_value)) + model.set_application_config( + self.application_name, + {'debug': new_debug_value}) + expected_contents = { + 'DEFAULT': { + 'debug': [new_debug_value]}} + logging.info("Waiting for debug to be {} in {}".format( + new_debug_value, + self.restart_config_file)) + model.block_until_oslo_config_entries_match( + self.application_name, + self.restart_config_file, + expected_contents) + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + + def trigger_deferred_restart_via_package(self): + """Update a package which requires a service restart.""" + logging.info("Triggering deferred restart via package change") + # Test restart requested by package + for unit in model.get_units(self.application_name): + model.run_on_unit( + unit.entity_id, + 'dpkg-reconfigure {}; ./hooks/update-status'.format( + self.restart_package)) + + def run_charm_change_test(self): + """Trigger a deferred restart by updating a config file via the charm. + + Trigger a config-changed hook in the charm which will update a config + file and add a deferred restart. + + NOTE: If this test is not relevant for the target charm then override + this method and raise unittest.SkipTest + """ + self.trigger_deferred_restart_via_charm() + + self.check_show_deferred_restarts_wlm(self.test_service) + self.check_show_deferred_restarts_action( + self.test_service, + self.restart_config_file) + logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def run_package_change_test(self): + """Trigger a deferred restart by updating a package. + + Update a package which requires will add a deferred restart. + + NOTE: If this test is not relevant for the target charm then override + this method and raise unittest.SkipTest + """ + self.trigger_deferred_restart_via_package() + + self.check_show_deferred_restarts_wlm(self.restart_package_service) + self.check_show_deferred_restarts_action( + self.restart_package_service, + 'Pkg Update') + logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def test_deferred_restarts(self): + """Run deferred restart tests.""" + app_config = model.get_application_config(self.application_name) + auto_restart_config_key = 'enable-auto-restarts' + if auto_restart_config_key not in app_config: + raise unittest.SkipTest("Deferred restarts not implemented") + + # Ensure auto restarts are off. + policy_file = '/etc/policy-rc.d/charm-{}.policy'.format( + self.application_name) + if app_config[auto_restart_config_key]['value']: + logging.info("Turning off auto restarts") + model.set_application_config( + self.application_name, {auto_restart_config_key: 'False'}) + logging.info("Waiting for {} to appear on units of {}".format( + policy_file, + self.application_name)) + model.block_until_file_has_contents( + self.application_name, + policy_file, + 'policy_requestor_name') + # The block_until_file_has_contents ensures the change we waiting + # for has happened, now just wait for any hooks to finish. + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + else: + logging.info("Auto restarts already disabled") + + # Trigger a config change which requires a restart + self.run_charm_change_test() + + # Trigger a package change which requires a restart + self.run_package_change_test() + + # Finished so turn auto-restarts back on. + logging.info("Turning on auto restarts") + model.set_application_config( + self.application_name, {auto_restart_config_key: 'True'}) + model.block_until_file_missing( + self.application_name, + policy_file) + model.block_until_all_units_idle() From d4b8fbaed037df0fb9050c9837c5f9efdc33455d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 25 Feb 2021 11:01:22 +0000 Subject: [PATCH 805/898] Rerun self.trigger_deferred_restart_via_charm --- zaza/openstack/charm_tests/test_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index bd82a38..3aff816 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -875,6 +875,8 @@ class BaseDeferredRestartTest(OpenStackBaseTest): self.check_show_deferred_restarts_action( self.test_service, self.restart_config_file) + # Rerunning to flip config option back to previous value. + self.trigger_deferred_restart_via_charm() logging.info("Running restart action to clear deferred restarts") self.check_clear_restarts() From 20bd5dfb5908f79b6e2c1c19b5e6e893baad86ee Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 25 Feb 2021 13:05:30 +0100 Subject: [PATCH 806/898] Remove obsolete comment The automatic corosync cleanup on Juju >= 2.8 turned out not to work and has been removed from charm-hacluster. The 'update-ring' action is now always needed. --- zaza/openstack/charm_tests/hacluster/tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index f340022..ea613ec 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -193,8 +193,6 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): logging.info('Updating corosync ring') hacluster_app_name = zaza.model.get_unit_from_name( surviving_hacluster_unit).application - # NOTE(lourot): with Juju >= 2.8 this isn't actually necessary as it - # has already been done in hacluster's hanode departing hook: zaza.model.run_action_on_leader( hacluster_app_name, 'update-ring', From adda37269958edad8c6087d264e58b8267f5261e Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 2 Mar 2021 22:44:40 +0000 Subject: [PATCH 807/898] Show cluster status before removal --- zaza/openstack/charm_tests/mysql/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 6322684..15256e7 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -771,6 +771,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): zaza.model.block_until_wl_status_info_starts_with( self.application, "'cluster' incomplete") + # Show status + logging.info(self.get_cluster_status()) + logging.info( "Removing old unit from cluster: {} " .format(leader_unit.public_address)) @@ -843,6 +846,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.states) + # Show status + logging.info(self.get_cluster_status()) + logging.info( "Removing old unit from cluster: {} " .format(non_leader_unit.public_address)) From c80f5fdef8fe80564406b084961d5a6b4f7e03a0 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 3 Mar 2021 10:50:53 +0100 Subject: [PATCH 808/898] Make HaclusterScaleBackAndForthTest less stressful Removing and re-adding two units out of a cluster of three so fast seems to be too stressful for corosync/pacemaker, which makes the test fail in rare cases. Removing and re-adding one unit only is enough in order to validate the functionality. This also makes the test execution time much shorter. --- zaza/openstack/charm_tests/hacluster/tests.py | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index ea613ec..b365960 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -154,40 +154,39 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): NOTE(lourot): before lp:1400481 was fixed, the corosync ring wasn't recalculated when removing units. So within a cluster of 3 units, - removing two units and re-adding them led to a situation where corosync - considers having 3 nodes online out of 5, instead of just 3 out of 3. + removing a unit and re-adding one led to a situation where corosync + considers having 3 nodes online out of 4, instead of just 3 out of 3. This test covers this scenario. """ principle_units = sorted(zaza.model.get_status().applications[ self._principle_app_name]['units'].keys()) self.assertEqual(len(principle_units), 3) - doomed_principle_units = principle_units[:2] - surviving_principle_unit = principle_units[2] - doomed_hacluster_units = juju_utils.get_subordinate_units( - doomed_principle_units, charm_name=self._hacluster_charm_name) + surviving_principle_unit = principle_units[0] + doomed_principle_unit = principle_units[1] surviving_hacluster_unit = juju_utils.get_subordinate_units( [surviving_principle_unit], charm_name=self._hacluster_charm_name)[0] + doomed_hacluster_unit = juju_utils.get_subordinate_units( + [doomed_principle_unit], + charm_name=self._hacluster_charm_name)[0] - for doomed_hacluster_unit in doomed_hacluster_units: - logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) - zaza.model.run_action( - doomed_hacluster_unit, - 'pause', - raise_on_failure=True) + logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) + zaza.model.run_action( + doomed_hacluster_unit, + 'pause', + raise_on_failure=True) - for doomed_principle_unit in doomed_principle_units: - logging.info('Removing {}'.format(doomed_principle_unit)) - zaza.model.destroy_unit( - self._principle_app_name, - doomed_principle_unit, - wait_disappear=True) + logging.info('Removing {}'.format(doomed_principle_unit)) + zaza.model.destroy_unit( + self._principle_app_name, + doomed_principle_unit, + wait_disappear=True) logging.info('Waiting for model to settle') zaza.model.block_until_unit_wl_status(surviving_hacluster_unit, 'blocked') - # NOTE(lourot): the principle unit (usually a keystone unit) isn't - # guaranteed to be blocked, so we don't validate that here. + # NOTE(lourot): the surviving principle units (usually keystone units) + # aren't guaranteed to be blocked, so we don't validate that here. zaza.model.block_until_all_units_idle() logging.info('Updating corosync ring') @@ -199,9 +198,8 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): action_params={'i-really-mean-it': True}, raise_on_failure=True) - for article in ('an', 'another'): - logging.info('Re-adding {} hacluster unit'.format(article)) - zaza.model.add_unit(self._principle_app_name, wait_appear=True) + logging.info('Re-adding an hacluster unit') + zaza.model.add_unit(self._principle_app_name, wait_appear=True) logging.info('Waiting for model to settle') # NOTE(lourot): the principle charm may remain blocked here. This seems @@ -215,7 +213,7 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): zaza.model.block_until_all_units_idle() # At this point if the corosync ring has been properly updated, there - # shouldn't be any trace of the deleted units anymore: + # shouldn't be any trace of the deleted unit anymore: logging.info('Checking that corosync considers all nodes to be online') cmd = 'sudo crm status' result = zaza.model.run_on_unit(surviving_hacluster_unit, cmd) From 0931ebb456574fb3b4b25344e3ed26861eca7429 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 3 Mar 2021 10:56:15 +0100 Subject: [PATCH 809/898] Fix docstring --- zaza/openstack/charm_tests/hacluster/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index b365960..66d4f45 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -150,7 +150,7 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): cls._hacluster_charm_name = test_config['hacluster-charm-name'] def test_930_scaleback(self): - """Remove two units, recalculate quorum and re-add two units. + """Remove one unit, recalculate quorum and re-add one unit. NOTE(lourot): before lp:1400481 was fixed, the corosync ring wasn't recalculated when removing units. So within a cluster of 3 units, From 2fefca5a406647a1efcd91e5f67441766f6976b7 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Tue, 2 Mar 2021 18:03:06 +0200 Subject: [PATCH 810/898] Code cleanup * Move general function out of the testing class. This will make them easy to be reused. * Properly organize the tests, and add tons of docstrings and comments to have them as clear as possible. * Add `failover_cinder_volume_host` to the Zaza `utilities/openstack.py`, since this is a general purpose function. --- .../charm_tests/ceph/rbd_mirror/tests.py | 776 ++++++++++++------ zaza/openstack/utilities/openstack.py | 34 + 2 files changed, 558 insertions(+), 252 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py index 75a3f42..d8d7967 100644 --- a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py @@ -16,7 +16,6 @@ import json import logging import re -import time import cinderclient.exceptions as cinder_exceptions @@ -31,6 +30,129 @@ from zaza.openstack.charm_tests.glance.setup import ( CIRROS_IMAGE_NAME) +DEFAULT_CINDER_RBD_MIRRORING_MODE = 'pool' + + +def get_cinder_rbd_mirroring_mode(cinder_ceph_app_name='cinder-ceph'): + """Get the RBD mirroring mode for the Cinder Ceph pool. + + :param cinder_ceph_app_name: Cinder Ceph Juju application name. + :type cinder_ceph_app_name: str + :returns: A string representing the RBD mirroring mode. It can be + either 'pool' or 'image'. + :rtype: str + """ + rbd_mirroring_mode_config = zaza.model.get_application_config( + cinder_ceph_app_name).get('rbd-mirroring-mode') + if rbd_mirroring_mode_config: + rbd_mirroring_mode = rbd_mirroring_mode_config.get( + 'value', DEFAULT_CINDER_RBD_MIRRORING_MODE).lower() + else: + rbd_mirroring_mode = DEFAULT_CINDER_RBD_MIRRORING_MODE + + return rbd_mirroring_mode + + +def get_glance_image(glance): + """Get the Glance image object to be used by the Ceph tests. + + It looks for the Cirros Glance image, and it's returned if it's found. + If the Cirros image is not found, it will try and find the Ubuntu + LTS image. + + :param glance: Authenticated glanceclient + :type glance: glanceclient.Client + :returns: Glance image object + :rtype: glanceclient.image + """ + images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) + if images: + return images[0] + logging.info("Failed to find {} image, falling back to {}".format( + CIRROS_IMAGE_NAME, + LTS_IMAGE_NAME)) + return openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] + + +def setup_cinder_repl_volume_type(cinder, type_name='repl', + backend_name='cinder-ceph'): + """Set up the Cinder volume replication type. + + :param cinder: Authenticated cinderclient + :type cinder: cinder.Client + :param type_name: Cinder volume type name + :type type_name: str + :param backend_name: Cinder volume backend name with replication enabled. + :type backend_name: str + :returns: Cinder volume type object + :rtype: cinderclient.VolumeType + """ + try: + vol_type = cinder.volume_types.find(name=type_name) + except cinder_exceptions.NotFound: + vol_type = cinder.volume_types.create(type_name) + + vol_type.set_keys(metadata={ + 'volume_backend_name': backend_name, + 'replication_enabled': ' True', + }) + return vol_type + + +# TODO: This function should be incorporated into +# 'zaza.openstack.utilities.openstack.create_volume' helper, once the below +# flakiness comments are addressed. +def create_cinder_volume(cinder, name='zaza', image_id=None, type_id=None): + """Create a new Cinder volume. + + :param cinder: Authenticated cinderclient. + :type cinder: cinder.Client + :param name: Volume name. + :type name: str + :param image_id: Glance image id, if the volume is created from image. + :type image_id: str + :param type_id: Cinder Volume type id, if the volume needs to use an + explicit volume type. + :type type_id: boolean + :returns: Cinder volume + :rtype: :class:`Volume`. + """ + # NOTE(fnordahl): for some reason create volume from image often fails + # when run just after deployment is finished. We should figure out + # why, resolve the underlying issue and then remove this. + # + # We do not use tenacity here as it will interfere with tenacity used + # in ``resource_reaches_status`` + def create_volume(cinder, volume_params, retry=20): + if retry < 1: + return + volume = cinder.volumes.create(**volume_params) + try: + # Note(coreycb): stop_after_attempt is increased because using + # juju storage for ceph-osd backed by cinder on undercloud + # takes longer than the prior method of directory-backed OSD + # devices. + openstack.resource_reaches_status( + cinder.volumes, volume.id, msg='volume', + stop_after_attempt=20) + return volume + except AssertionError: + logging.info('retrying') + volume.delete() + return create_volume(cinder, volume_params, retry=retry - 1) + + volume_params = { + 'size': 8, + 'name': name, + } + if image_id: + volume_params['imageRef'] = image_id + if type_id: + volume_params['volume_type'] = type_id + + return create_volume(cinder, volume_params) + + class CephRBDMirrorBase(test_utils.OpenStackBaseTest): """Base class for ``ceph-rbd-mirror`` tests.""" @@ -38,6 +160,8 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run setup for ``ceph-rbd-mirror`` tests.""" super().setUpClass() + cls.cinder_ceph_app_name = 'cinder-ceph' + cls.test_cinder_volume_name = 'test-cinder-ceph-volume' # get ready for multi-model Zaza cls.site_a_model = cls.site_b_model = zaza.model.get_juju_model() cls.site_b_app_suffix = '-b' @@ -75,6 +199,21 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): model_name=self.site_b_model) return sorted(site_a_pools.keys()), sorted(site_b_pools.keys()) + def get_failover_pools(self): + """Get the failover Ceph pools' names, from both sites. + + If the Cinder RBD mirroring mode is 'image', the 'cinder-ceph' pool + needs to be excluded, since Cinder orchestrates the failover then. + + :returns: Tuple with site-a pools and site-b pools. + :rtype: Tuple[List[str], List[str]] + """ + site_a_pools, site_b_pools = self.get_pools() + if get_cinder_rbd_mirroring_mode(self.cinder_ceph_app_name) == 'image': + site_a_pools.remove(self.cinder_ceph_app_name) + site_b_pools.remove(self.cinder_ceph_app_name) + return site_a_pools, site_b_pools + def wait_for_mirror_state(self, state, application_name=None, model_name=None, check_entries_behind_master=False, @@ -136,118 +275,40 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): # all images with state has expected state return True - def get_cinder_rbd_mirroring_mode(self, - cinder_ceph_app_name='cinder-ceph'): - """Get the RBD mirroring mode for the Cinder Ceph pool. + def setup_test_cinder_volume(self): + """Set up the test Cinder volume into the Ceph RBD mirror environment. - :returns: A string representing the RBD mirroring mode. It can be - either 'pool' or 'image'. - """ - DEFAULT_RBD_MIRRORING_MODE = 'pool' + If the volume already exists, then it's returned. - rbd_mirroring_mode_config = zaza.model.get_application_config( - cinder_ceph_app_name).get('rbd-mirroring-mode') - if rbd_mirroring_mode_config: - rbd_mirroring_mode = rbd_mirroring_mode_config.get( - 'value', DEFAULT_RBD_MIRRORING_MODE).lower() - else: - rbd_mirroring_mode = DEFAULT_RBD_MIRRORING_MODE - - return rbd_mirroring_mode - - def create_cinder_volume(self, session, from_image=False): - """Create Cinder Volume from image. + Also, if the Cinder RBD mirroring mode is 'image', the volume will + use an explicit volume type with the appropriate replication flags. + Otherwise, it is just a simple Cinder volume using the default backend. + :returns: Cinder volume :rtype: :class:`Volume`. """ - def get_glance_image(session): - glance = openstack.get_glance_session_client(session) - images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) - if images: - return images[0] - logging.info("Failed to find {} image, falling back to {}".format( - CIRROS_IMAGE_NAME, - LTS_IMAGE_NAME)) - return openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session, version=3) - def create_volume_type(cinder): - try: - vol_type = cinder.volume_types.find(name='repl') - except cinder_exceptions.NotFound: - vol_type = cinder.volume_types.create('repl') - vol_type.set_keys(metadata={ - 'volume_backend_name': 'cinder-ceph', - 'replication_enabled': ' True', - }) - return vol_type + try: + return cinder.volumes.find(name=self.test_cinder_volume_name) + except cinder_exceptions.NotFound: + logging.info("Test Cinder volume doesn't exist. Creating it") - # NOTE(fnordahl): for some reason create volume from image often fails - # when run just after deployment is finished. We should figure out - # why, resolve the underlying issue and then remove this. - # - # We do not use tenacity here as it will interfere with tenacity used - # in ``resource_reaches_status`` - def create_volume(cinder, volume_params, retry=20): - if retry < 1: - return - volume = cinder.volumes.create(**volume_params) - try: - # Note(coreycb): stop_after_attempt is increased because using - # juju storage for ceph-osd backed by cinder on undercloud - # takes longer than the prior method of directory-backed OSD - # devices. - openstack.resource_reaches_status( - cinder.volumes, volume.id, msg='volume', - stop_after_attempt=20) - return volume - except AssertionError: - logging.info('retrying') - volume.delete() - return create_volume(cinder, volume_params, retry=retry - 1) - - volume_params = { - 'size': 8, - 'name': 'zaza', + glance = openstack.get_glance_session_client(session) + image = get_glance_image(glance) + kwargs = { + 'cinder': cinder, + 'name': self.test_cinder_volume_name, + 'image_id': image.id, } - if from_image: - volume_params['imageRef'] = get_glance_image(session).id - cinder = openstack.get_cinder_session_client(session) - if self.get_cinder_rbd_mirroring_mode() == 'image': - volume_params['volume_type'] = create_volume_type(cinder).id + if get_cinder_rbd_mirroring_mode(self.cinder_ceph_app_name) == 'image': + volume_type = setup_cinder_repl_volume_type( + cinder, + backend_name=self.cinder_ceph_app_name) + kwargs['type_id'] = volume_type.id - return create_volume(cinder, volume_params) - - def failover_cinder_volume_host(self, cinder_client, - backend_name='cinder-ceph', - target_backend_id='ceph', - target_status='disabled', - target_replication_status='failed-over', - timeout=300): - """Failover Cinder volume host.""" - host = 'cinder@{}'.format(backend_name) - logging.info( - 'Failover Cinder host %s to backend_id %s', - host, target_backend_id) - cinder_client.services.failover_host( - host=host, - backend_id=target_backend_id) - start = time.time() - while True: - elapsed = time.time() - start - if elapsed > timeout: - raise cinder_exceptions.TimeoutException( - obj=cinder_client.services, - action='failover_host') - service = cinder_client.services.list( - host=host, - binary='cinder-volume')[0] - if (service.status == target_status and - service.replication_status == target_replication_status): - break - time.sleep(5) - logging.info( - 'Successfully failed-over Cinder host %s to backend_id %s', - host, target_backend_id) + return create_cinder_volume(**kwargs) class CephRBDMirrorTest(CephRBDMirrorBase): @@ -320,8 +381,7 @@ class CephRBDMirrorTest(CephRBDMirrorBase): site B and subsequently comparing the contents we get a full end to end test. """ - session = openstack.get_overcloud_keystone_session() - volume = self.create_cinder_volume(session, from_image=True) + volume = self.setup_test_cinder_volume() site_a_hash = zaza.openstack.utilities.ceph.get_rbd_hash( zaza.model.get_lead_unit_name('ceph-mon', model_name=self.site_a_model), @@ -333,6 +393,8 @@ class CephRBDMirrorTest(CephRBDMirrorBase): check_entries_behind_master=True, application_name=self.application_name + self.site_b_app_suffix, model_name=self.site_b_model) + logging.info('Checking the Ceph RBD hashes of the primary and ' + 'the secondary Ceph images') site_b_hash = zaza.openstack.utilities.ceph.get_rbd_hash( zaza.model.get_lead_unit_name('ceph-mon' + self.site_b_app_suffix, model_name=self.site_b_model), @@ -347,189 +409,399 @@ class CephRBDMirrorTest(CephRBDMirrorBase): class CephRBDMirrorControlledFailoverTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` controlled failover tests.""" - def cinder_fail_over_fall_back(self): - """Validate controlled fail over and fall back via the Cinder API.""" + def execute_failover_juju_actions(self, + primary_site_app_name, + primary_site_model, + primary_site_pools, + secondary_site_app_name, + secondary_site_model, + secondary_site_pools): + """Execute the failover Juju actions. + + The failover / failback via Juju actions shares the same workflow. The + failback is just a failover with sites in reversed order. + + This function encapsulates the tasks to failover a primary site to + a secondary site: + 1. Demote primary site + 2. Validation of the primary site demotion + 3. Promote secondary site + 4. Validation of the secondary site promotion + + :param primary_site_app_name: Primary site Ceph RBD mirror app name. + :type primary_site_app_name: str + :param primary_site_model: Primary site Juju model name. + :type primary_site_model: str + :param primary_site_pools: Primary site pools. + :type primary_site_pools: List[str] + :param secondary_site_app_name: Secondary site Ceph RBD mirror + app name. + :type secondary_site_app_name: str + :param secondary_site_model: Secondary site Juju model name. + :type secondary_site_model: str + :param secondary_site_pools: Secondary site pools. + :type secondary_site_pools: List[str] + """ + # Check if primary and secondary pools sizes are the same. + self.assertEqual(len(primary_site_pools), len(secondary_site_pools)) + + # Run the 'demote' Juju action against the primary site pools. + logging.info('Demoting {} from model {}.'.format( + primary_site_app_name, primary_site_model)) + result = zaza.model.run_action_on_leader( + primary_site_app_name, + 'demote', + model_name=primary_site_model, + action_params={ + 'pools': ','.join(primary_site_pools) + }) + logging.info(result.results) + self.assertEqual(int(result.results['Code']), 0) + + # Validate that the demoted pools count matches the total primary site + # pools count. + n_pools_demoted = len(result.results['output'].split('\n')) + self.assertEqual(len(primary_site_pools), n_pools_demoted) + + # At this point, both primary and secondary sites are demoted. Validate + # that the Ceph images, from both sites, report 'up+unknown', since + # there isn't a primary site at the moment. + logging.info('Waiting until {} is demoted.'.format( + primary_site_app_name)) + self.wait_for_mirror_state( + 'up+unknown', + application_name=primary_site_app_name, + model_name=primary_site_model, + pools=primary_site_pools) + self.wait_for_mirror_state( + 'up+unknown', + application_name=secondary_site_app_name, + model_name=secondary_site_model, + pools=secondary_site_pools) + + # Run the 'promote' Juju against the secondary site. + logging.info('Promoting {} from model {}.'.format( + secondary_site_app_name, secondary_site_model)) + result = zaza.model.run_action_on_leader( + secondary_site_app_name, + 'promote', + model_name=secondary_site_model, + action_params={ + 'pools': ','.join(secondary_site_pools) + }) + logging.info(result.results) + self.assertEqual(int(result.results['Code']), 0) + + # Validate that the promoted pools count matches the total secondary + # site pools count. + n_pools_promoted = len(result.results['output'].split('\n')) + self.assertEqual(len(secondary_site_pools), n_pools_promoted) + + # Validate that the Ceph images from the newly promoted site + # report 'up+stopped' state (which is reported by primary Ceph images). + logging.info('Waiting until {} is promoted.'.format( + secondary_site_app_name)) + self.wait_for_mirror_state( + 'up+stopped', + application_name=secondary_site_app_name, + model_name=secondary_site_model, + pools=secondary_site_pools) + + # Validate that the Ceph images from site-a report 'up+replaying' + # (which is reported by secondary Ceph images). + self.wait_for_mirror_state( + 'up+replaying', + check_entries_behind_master=True, + application_name=primary_site_app_name, + model_name=primary_site_model, + pools=primary_site_pools) + + def test_100_cinder_failover(self): + """Validate controlled failover via the Cinder API. + + This test only makes sense if Cinder RBD mirroring mode is 'image'. + It will return early, if this is not the case. + """ + cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode( + self.cinder_ceph_app_name) + if cinder_rbd_mirroring_mode != 'image': + logging.warning( + "Skipping 'test_100_cinder_failover' since Cinder RBD " + "mirroring mode is {}.".format(cinder_rbd_mirroring_mode)) + return + session = openstack.get_overcloud_keystone_session() - cinder = openstack.get_cinder_session_client(session) - volume = self.create_cinder_volume(session, from_image=True) + cinder = openstack.get_cinder_session_client(session, version=3) + + # Check if the Cinder volume host is available with replication + # enabled. + host = 'cinder@{}'.format(self.cinder_ceph_app_name) + svc = cinder.services.list(host=host, binary='cinder-volume')[0] + self.assertEqual(svc.replication_status, 'enabled') + self.assertEqual(svc.status, 'enabled') + + # Setup the test Cinder volume + volume = self.setup_test_cinder_volume() + + # Check if the volume is properly mirrored self.wait_for_mirror_state( 'up+replaying', check_entries_behind_master=True, application_name=self.application_name + self.site_b_app_suffix, model_name=self.site_b_model, - pools=['cinder-ceph']) - self.failover_cinder_volume_host( - cinder_client=cinder) + pools=[self.cinder_ceph_app_name]) + + # Execute the Cinder volume failover + openstack.failover_cinder_volume_host( + cinder=cinder, + backend_name=self.cinder_ceph_app_name, + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over') + + # Check if the test volume is still available after failover self.assertEqual(cinder.volumes.get(volume.id).status, 'available') - self.failover_cinder_volume_host( - cinder_client=cinder, + + def test_101_cinder_failback(self): + """Validate controlled failback via the Cinder API. + + This test only makes sense if Cinder RBD mirroring mode is 'image'. + It will return early, if this is not the case. + + The test needs to be executed when the Cinder volume host is already + failed-over with the test volume on it. + """ + cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode( + self.cinder_ceph_app_name) + if cinder_rbd_mirroring_mode != 'image': + logging.warning( + "Skipping 'test_101_cinder_failback' since Cinder RBD " + "mirroring mode is {}.".format(cinder_rbd_mirroring_mode)) + return + + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session, version=3) + + # Check if the Cinder volume host is already failed-over + host = 'cinder@{}'.format(self.cinder_ceph_app_name) + svc = cinder.services.list(host=host, binary='cinder-volume')[0] + self.assertEqual(svc.replication_status, 'failed-over') + self.assertEqual(svc.status, 'disabled') + + # Check if the test Cinder volume is already present. The method + # 'cinder.volumes.find' raises 404 if the volume is not found. + volume = cinder.volumes.find(name=self.test_cinder_volume_name) + + # Execute the Cinder volume failback + openstack.failover_cinder_volume_host( + cinder=cinder, + backend_name=self.cinder_ceph_app_name, target_backend_id='default', target_status='enabled', target_replication_status='enabled') + + # Check if the test volume is still available after failback self.assertEqual(cinder.volumes.get(volume.id).status, 'available') - def test_fail_over_fall_back(self): - """Validate controlled fail over and fall back.""" - site_a_pools, site_b_pools = self.get_pools() - site_a_action_params = {} - site_b_action_params = {} - if self.get_cinder_rbd_mirroring_mode() == 'image': - site_a_pools.remove('cinder-ceph') - site_a_action_params['pools'] = ','.join(site_a_pools) - site_b_pools.remove('cinder-ceph') - site_b_action_params['pools'] = ','.join(site_b_pools) + def test_200_juju_failover(self): + """Validate controlled failover via Juju actions.""" + # Get the Ceph pools needed to failover + site_a_pools, site_b_pools = self.get_failover_pools() + + # Execute the failover Juju actions with the appropriate parameters. + site_b_app_name = self.application_name + self.site_b_app_suffix + self.execute_failover_juju_actions( + primary_site_app_name=self.application_name, + primary_site_model=self.site_a_model, + primary_site_pools=site_a_pools, + secondary_site_app_name=site_b_app_name, + secondary_site_model=self.site_b_model, + secondary_site_pools=site_b_pools) + + def test_201_juju_failback(self): + """Validate controlled failback via Juju actions.""" + # Get the Ceph pools needed to failback + site_a_pools, site_b_pools = self.get_failover_pools() + + # Execute the failover Juju actions with the appropriate parameters. + # The failback operation is just a failover with sites in reverse + # order. + site_b_app_name = self.application_name + self.site_b_app_suffix + self.execute_failover_juju_actions( + primary_site_app_name=site_b_app_name, + primary_site_model=self.site_b_model, + primary_site_pools=site_b_pools, + secondary_site_app_name=self.application_name, + secondary_site_model=self.site_a_model, + secondary_site_pools=site_a_pools) + + def test_203_juju_resync(self): + """Validate the 'resync-pools' Juju action. + + The 'resync-pools' Juju action is meant to flag Ceph images from the + secondary site to re-sync against the Ceph images from the primary + site. + + This use case is useful when the Ceph secondary images are out of sync. + """ + # Get the Ceph pools needed to failback + _, site_b_pools = self.get_failover_pools() + + # Run the 'resync-pools' Juju action against the pools from site-b. + # This will make sure that the Ceph images from site-b are properly + # synced with the primary images from site-a. + site_b_app_name = self.application_name + self.site_b_app_suffix + logging.info('Re-syncing {} from model {}'.format( + site_b_app_name, self.site_b_model)) result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror', - 'demote', - model_name=self.site_a_model, - action_params=site_a_action_params) - logging.info(result.results) - n_pools_demoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_a_pools), n_pools_demoted) - self.wait_for_mirror_state( - 'up+unknown', - model_name=self.site_a_model, - pools=site_a_pools) - self.wait_for_mirror_state( - 'up+unknown', - application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model, - pools=site_b_pools) - result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, - 'promote', - model_name=self.site_b_model, - action_params=site_b_action_params) - logging.info(result.results) - n_pools_promoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_b_pools), n_pools_promoted) - self.wait_for_mirror_state( - 'up+replaying', - model_name=self.site_a_model, - pools=site_a_pools) - self.wait_for_mirror_state( - 'up+stopped', - application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model, - pools=site_b_pools) - result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, - 'demote', - model_name=self.site_b_model, - action_params=site_b_action_params) - logging.info(result.results) - n_pools_demoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_a_pools), n_pools_demoted) - self.wait_for_mirror_state( - 'up+unknown', - model_name=self.site_a_model, - pools=site_a_pools) - self.wait_for_mirror_state( - 'up+unknown', - application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model, - pools=site_b_pools) - result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror', - 'promote', - model_name=self.site_a_model, - action_params=site_a_action_params) - logging.info(result.results) - n_pools_promoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_b_pools), n_pools_promoted) - self.wait_for_mirror_state( - 'up+stopped', - model_name=self.site_a_model, - pools=site_a_pools) - action_params = { - 'i-really-mean-it': True, - } - if self.get_cinder_rbd_mirroring_mode() == 'image': - action_params['pools'] = site_b_action_params['pools'] - result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, + site_b_app_name, 'resync-pools', model_name=self.site_b_model, - action_params=action_params) + action_params={ + 'pools': ','.join(site_b_pools), + 'i-really-mean-it': True, + }) logging.info(result.results) + self.assertEqual(int(result.results['Code']), 0) + + # Validate that the Ceph images from site-b report 'up+replaying' + # (which is reported by secondary Ceph images). And check that images + # exist in Cinder and Glance pools. self.wait_for_mirror_state( 'up+replaying', - application_name=self.application_name + self.site_b_app_suffix, + check_entries_behind_master=True, + application_name=site_b_app_name, model_name=self.site_b_model, - require_images_in=['cinder-ceph', 'glance'], - pools=site_a_pools) - if self.get_cinder_rbd_mirroring_mode() == 'image': - self.cinder_fail_over_fall_back() + require_images_in=[self.cinder_ceph_app_name, 'glance'], + pools=site_b_pools) class CephRBDMirrorDisasterFailoverTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` destructive tests.""" - def forced_failover_cinder_volume_host(self, cinder_client): - """Validate forced Cinder volume host fail over.""" - def apply_cinder_workaround(): - """Set minimal timeouts / retries to the Cinder Ceph backend. + def apply_cinder_ceph_workaround(self): + """Set minimal timeouts / retries to the Cinder Ceph backend. - This is needed because the failover via Cinder will try to do a - demotion of the site-a, and with the default timeouts / retries, - the operation takes an unreasonably amount of time. - """ - cinder_configs = { - 'rados_connect_timeout': '1', - 'rados_connection_retries': '1', - 'rados_connection_interval': '0', - 'replication_connect_timeout': '1', - } - update_cinder_conf_cmd = ( - "import configparser; " - "config = configparser.ConfigParser(); " - "config.read('/etc/cinder/cinder.conf'); " - "{}" - "f = open('/etc/cinder/cinder.conf', 'w'); " - "config.write(f); " - "f.close()") - cmd = '' - for config in cinder_configs: - cmd += "config.set('cinder-ceph', '{0}', '{1}'); ".format( - config, cinder_configs[config]) - cmd = update_cinder_conf_cmd.format(cmd) - zaza.model.run_on_leader( - 'cinder-ceph', - 'python3 -c "{}"; systemctl restart cinder-volume'.format(cmd)) + This is needed because the failover via Cinder API will try to do a + demotion of the site-a. However, when site-a is down, and with the + default timeouts / retries, the operation takes an unreasonably amount + of time (or sometimes it never finishes). + """ + # These new config options need to be set under the Cinder Ceph backend + # section in the main Cinder config file. + # At the moment, we don't the possibility of using Juju config to set + # these options. And also, it's not even a good practice to have them + # in production. + # These should be set only to do the Ceph failover via Cinder API, and + # they need to be removed after. + configs = { + 'rados_connect_timeout': '1', + 'rados_connection_retries': '1', + 'rados_connection_interval': '0', + 'replication_connect_timeout': '1', + } - apply_cinder_workaround() - self.failover_cinder_volume_host(cinder_client) + # Small Python script that will be executed via Juju run to update + # the Cinder config file. + update_cinder_conf_script = ( + "import configparser; " + "config = configparser.ConfigParser(); " + "config.read('/etc/cinder/cinder.conf'); " + "{}" + "f = open('/etc/cinder/cinder.conf', 'w'); " + "config.write(f); " + "f.close()") + set_cmd = '' + for cfg_name in configs: + set_cmd += "config.set('{0}', '{1}', '{2}'); ".format( + self.cinder_ceph_app_name, cfg_name, configs[cfg_name]) + script = update_cinder_conf_script.format(set_cmd) - for volume in cinder_client.volumes.list(): - self.assertEqual(volume.status, 'available') + # Run the workaround script via Juju run + zaza.model.run_on_leader( + self.cinder_ceph_app_name, + 'python3 -c "{}"; systemctl restart cinder-volume'.format(script)) - def test_kill_site_a_fail_over(self): - """Validate fail over after uncontrolled shutdown of primary.""" - action_params = {} - if self.get_cinder_rbd_mirroring_mode() == 'image': - _, site_b_pools = self.get_pools() - site_b_pools.remove('cinder-ceph') - action_params['pools'] = ','.join(site_b_pools) - - for application in 'ceph-rbd-mirror', 'ceph-mon', 'ceph-osd': + def kill_primary_site(self): + """Simulate an unexpected primary site shutdown.""" + logging.info('Killing the Ceph primary site') + for application in ['ceph-rbd-mirror', 'ceph-mon', 'ceph-osd']: zaza.model.remove_application( application, model_name=self.site_a_model, forcefully_remove_machines=True) + + def test_100_forced_juju_failover(self): + """Validate Ceph failover via Juju when the primary site is down. + + * Kill the primary site + * Execute the forced failover via Juju actions + """ + # Get the site-b Ceph pools that need to be promoted + _, site_b_pools = self.get_failover_pools() + site_b_app_name = self.application_name + self.site_b_app_suffix + + # Simulate primary site unexpected shutdown + self.kill_primary_site() + + # Try and promote the site-b to primary. result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, + site_b_app_name, 'promote', model_name=self.site_b_model, - action_params=action_params) + action_params={ + 'pools': ','.join(site_b_pools), + }) + self.assertEqual(int(result.results['Code']), 0) + + # The site-b 'promote' Juju action is expected to fail, because the + # primary site is down. self.assertEqual(result.status, 'failed') - action_params['force'] = True + + # Retry to promote site-b using the 'force' Juju action parameter. result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, + site_b_app_name, 'promote', model_name=self.site_b_model, - action_params=action_params) + action_params={ + 'force': True, + 'pools': ','.join(site_b_pools), + }) + self.assertEqual(int(result.results['Code']), 0) + + # Validate successful Juju action execution self.assertEqual(result.status, 'completed') - if self.get_cinder_rbd_mirroring_mode() == 'image': - session = openstack.get_overcloud_keystone_session() - cinder = openstack.get_cinder_session_client(session) - self.forced_failover_cinder_volume_host(cinder) + + def test_200_forced_cinder_failover(self): + """Validate Ceph failover via Cinder when the primary site is down. + + This test only makes sense if Cinder RBD mirroring mode is 'image'. + It will return early, if this is not the case. + + This assumes that the primary site is already killed. + """ + cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode( + self.cinder_ceph_app_name) + if cinder_rbd_mirroring_mode != 'image': + logging.warning( + "Skipping 'test_200_cinder_failover_without_primary_site' " + "since Cinder RBD mirroring mode is {}.".format( + cinder_rbd_mirroring_mode)) + return + + # Make sure that the Cinder Ceph backend workaround is applied. + self.apply_cinder_ceph_workaround() + + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session, version=3) + openstack.failover_cinder_volume_host( + cinder=cinder, + backend_name=self.cinder_ceph_app_name, + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over') + + # Check that the Cinder volumes are still available after forced + # failover. + for volume in cinder.volumes.list(): + self.assertEqual(volume.status, 'available') diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index d277656..4ed75b9 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2489,6 +2489,40 @@ def attach_volume(nova, volume_id, instance_id): device='/dev/vdx') +def failover_cinder_volume_host(cinder, backend_name='cinder-ceph', + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over'): + """Failover Cinder volume host with replication enabled. + + :param cinder: Authenticated cinderclient + :type cinder: cinder.Client + :param backend_name: Cinder volume backend name with + replication enabled. + :type backend_name: str + :param target_backend_id: Failover target Cinder backend id. + :type target_backend_id: str + :param target_status: Target Cinder volume status after failover. + :type target_status: str + :param target_replication_status: Target Cinder volume replication + status after failover. + :type target_replication_status: str + :raises: AssertionError + """ + host = 'cinder@{}'.format(backend_name) + logging.info('Failover Cinder volume host %s to backend_id %s', + host, target_backend_id) + cinder.services.failover_host(host=host, backend_id=target_backend_id) + for attempt in tenacity.Retrying( + retry=tenacity.retry_if_exception_type(AssertionError), + stop=tenacity.stop_after_attempt(10), + wait=tenacity.wait_exponential(multiplier=1, min=2, max=10)): + with attempt: + svc = cinder.services.list(host=host, binary='cinder-volume')[0] + assert svc.status == target_status + assert svc.replication_status == target_replication_status + + def create_volume_backup(cinder, volume_id, name=None): """Create cinder volume backup. From 73c78a0db3c2a1b89314405bf83540d7b0444509 Mon Sep 17 00:00:00 2001 From: Bartosz Woronicz Date: Tue, 2 Mar 2021 00:33:55 +0100 Subject: [PATCH 811/898] add reload and restart action tests for vault --- zaza/openstack/charm_tests/vault/tests.py | 54 ++++++++++++++++++++++- zaza/openstack/charm_tests/vault/utils.py | 14 ++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 68130a0..219291e 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -244,8 +244,6 @@ class VaultTest(BaseVaultTest): Pause service and check services are stopped, then resume and check they are started. """ - # Restarting vault process will set it as sealed so it's - # important to have the test executed at the end. vault_actions = zaza.model.get_actions( 'vault') if 'pause' not in vault_actions or 'resume' not in vault_actions: @@ -257,6 +255,58 @@ class VaultTest(BaseVaultTest): lead_client = vault_utils.extract_lead_unit_client(self.clients) self.assertTrue(lead_client.hvac_client.seal_status['sealed']) + def test_vault_reload(self): + """Run reload tests. + + Reload service and check services were restarted + by doing simple change in the running config by API. + Then confirm that service is not sealed + """ + vault_actions = zaza.model.get_actions( + 'vault') + if 'reload' not in vault_actions: + raise unittest.SkipTest("The version of charm-vault tested does " + "not have reload action") + + lead_client = vault_utils.extract_lead_unit_client(self.clients) + running_config = vault_utils.get_running_config(lead_client) + value_to_set = not running_config['data']['disable_mlock'] + + zaza.model.set_application_config( + 'vault', + {'disable-mlock': str(value_to_set)}) + + logging.info("Testing reload") + zaza.model.run_action_on_leader( + 'vault', + 'reload', + action_params={}) + + self.assertEqual( + value_to_set, + vault_utils.get_running_config(lead_client)[ + 'data']['disable_mlock']) + self.assertFalse(lead_client.hvac_client.seal_status['sealed']) + + def test_vault_restart(self): + """Run pause and resume tests. + + Restart service and check services are started. + """ + vault_actions = zaza.model.get_actions( + 'vault') + if 'restart' not in vault_actions: + raise unittest.SkipTest("The version of charm-vault tested does " + "not have restart action") + logging.info("Testing restart") + zaza.model.run_action_on_leader( + 'vault', + 'restart', + action_params={}) + + lead_client = vault_utils.extract_lead_unit_client(self.clients) + self.assertTrue(lead_client.hvac_client.seal_status['sealed']) + if __name__ == '__main__': unittest.main() diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 9814243..9098b12 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -137,6 +137,20 @@ def get_vip_client(cacert=None): return client +def get_running_config(client): + """Get Vault running config. + + :param client: Client to use for initiliasation + :type client: CharmVaultClient + The hvac library does not support getting info + from endpoint /v1/sys/config/state/sanitized + Therefore we implement it here + """ + return requests.get( + client.hvac_client.adapter.base_uri + '/v1/sys/config/state/sanitized', + headers={'X-Vault-Token': client.hvac_client.token}).json() + + def init_vault(client, shares=1, threshold=1): """Initialise vault. From 9901248f7982481c5434caaa93175e9b1f549fff Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Mon, 1 Mar 2021 12:20:06 +0200 Subject: [PATCH 812/898] Refactor Manila Ganesha test The current Manila Ganesha `test_manila_share` had a lot of functionality that can be re-used to any configured Manila backend. It's a good idea to have this functionality generalized into `zaza.manila.tests.ManilaBaseTest`, which can be reused for any tested Manila backend. --- zaza/openstack/charm_tests/manila/tests.py | 264 ++++++++++++++++++ .../charm_tests/manila_ganesha/setup.py | 5 +- .../charm_tests/manila_ganesha/tests.py | 118 +------- 3 files changed, 276 insertions(+), 111 deletions(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index c00b993..e7cba8f 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -16,12 +16,47 @@ """Encapsulate Manila testing.""" +import logging import tenacity from manilaclient import client as manilaclient import zaza.model +import zaza.openstack.configure.guest as guest +import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.charm_tests.nova.utils as nova_utils +import zaza.openstack.charm_tests.neutron.tests as neutron_tests + + +def verify_status(stdin, stdout, stderr): + """Callable to verify the command output. + + It checks if the command successfully executed. + + This is meant to be given as parameter 'verify' to the helper function + 'openstack_utils.ssh_command'. + """ + status = stdout.channel.recv_exit_status() + if status: + logging.info("{}".format(stderr.readlines()[0].strip())) + assert status == 0 + + +def verify_manila_testing_file(stdin, stdout, stderr): + """Callable to verify the command output. + + It checks if the command successfully executed, and it validates the + testing file written on the Manila share. + + This is meant to be given as parameter 'verify' to the helper function + 'openstack_utils.ssh_command'. + """ + verify_status(stdin, stdout, stderr) + out = "" + for line in iter(stdout.readline, ""): + out += line + assert out == "test\n" class ManilaTests(test_utils.OpenStackBaseTest): @@ -49,3 +84,232 @@ class ManilaTests(test_utils.OpenStackBaseTest): wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) def _list_shares(self): return self.manila_client.shares.list() + + +class ManilaBaseTest(test_utils.OpenStackBaseTest): + """Encapsulate a Manila basic functionality test.""" + + RESOURCE_PREFIX = 'zaza-manilatests' + INSTANCE_KEY = 'bionic' + INSTANCE_USERDATA = """#cloud-config +packages: +- nfs-common +""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaBaseTest, cls).setUpClass() + cls.nova_client = openstack_utils.get_nova_session_client( + session=cls.keystone_session) + cls.manila_client = manilaclient.Client( + session=cls.keystone_session, client_version='2') + cls.share_name = 'test-manila-share' + cls.share_type_name = 'default_share_type' + cls.share_protocol = 'nfs' + cls.mount_dir = '/mnt/manila_share' + cls.share_network = None + + @classmethod + def tearDownClass(cls): + """Run class teardown after tests finished.""" + # Cleanup Nova servers + logging.info('Cleaning up test Nova servers') + fips_reservations = [] + for vm in cls.nova_client.servers.list(): + fips_reservations += neutron_tests.floating_ips_from_instance(vm) + vm.delete() + openstack_utils.resource_removed( + cls.nova_client.servers, + vm.id, + msg="Waiting for the Nova VM {} to be deleted".format(vm.name)) + + # Delete FiPs reservations + logging.info('Cleaning up test FiPs reservations') + neutron = openstack_utils.get_neutron_session_client( + session=cls.keystone_session) + for fip in neutron.list_floatingips()['floatingips']: + if fip['floating_ip_address'] in fips_reservations: + neutron.delete_floatingip(fip['id']) + + # Cleanup Manila shares + logging.info('Cleaning up test shares') + for share in cls.manila_client.shares.list(): + share.delete() + openstack_utils.resource_removed( + cls.manila_client.shares, + share.id, + msg="Waiting for the Manila share {} to be deleted".format( + share.name)) + + # Cleanup test Manila share servers (spawned by the driver when DHSS + # is enabled). + logging.info('Cleaning up test shares servers (if found)') + for server in cls.manila_client.share_servers.list(): + server.delete() + openstack_utils.resource_removed( + cls.manila_client.share_servers, + server.id, + msg="Waiting for the share server {} to be deleted".format( + server.id)) + + def _get_mount_options(self): + """Get the appropriate mount options used to mount the Manila share. + + :returns: The proper mount options flags for the share protocol. + :rtype: string + """ + if self.share_protocol == 'nfs': + return 'nfsvers=4.1,proto=tcp' + else: + raise NotImplementedError( + 'Share protocol not supported yet: {}'.format( + self.share_protocol)) + + def _mount_share_on_instance(self, instance_ip, ssh_user_name, + ssh_private_key, share_path): + """Mount a share into a Nova instance. + + The mount command is executed via SSH. + + :param instance_ip: IP of the Nova instance. + :type instance_ip: string + :param ssh_user_name: SSH user name. + :type ssh_user_name: string + :param ssh_private_key: SSH private key. + :type ssh_private_key: string + :param share_path: Share network path. + :type share_path: string + """ + ssh_cmd = ( + 'sudo mkdir -p {0} && ' + 'sudo mount -t {1} -o {2} {3} {0}'.format( + self.mount_dir, + self.share_protocol, + self._get_mount_options(), + share_path)) + + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)): + with attempt: + openstack_utils.ssh_command( + vm_name="instance-{}".format(instance_ip), + ip=instance_ip, + username=ssh_user_name, + privkey=ssh_private_key, + command=ssh_cmd, + verify=verify_status) + + @tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) + def _write_testing_file_on_instance(self, instance_ip, ssh_user_name, + ssh_private_key): + """Write a file on a Manila share mounted into a Nova instance. + + Mount the Manila share in the given Nova instance, and write a testing + file on it (which is meant to be validated from another instance). + These commands are executed via SSH. + + :param instance_ip: IP of the Nova instance. + :type instance_ip: string + :param ssh_user_name: SSH user name. + :type ssh_user_name: string + :param ssh_private_key: SSH private key. + :type ssh_private_key: string + """ + openstack_utils.ssh_command( + vm_name="instance-{}".format(instance_ip), + ip=instance_ip, + username=ssh_user_name, + privkey=ssh_private_key, + command='echo "test" | sudo tee {}/test'.format( + self.mount_dir), + verify=verify_status) + + @tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) + def _validate_testing_file_from_instance(self, instance_ip, ssh_user_name, + ssh_private_key): + """Validate a file from the Manila share mounted into a Nova instance. + + This is meant to run after the testing file was already written into + another Nova instance. It validates the written file. The commands are + executed via SSH. + + :param instance_ip: IP of the Nova instance. + :type instance_ip: string + :param ssh_user_name: SSH user name. + :type ssh_user_name: string + :param ssh_private_key: SSH private key. + :type ssh_private_key: string + """ + openstack_utils.ssh_command( + vm_name="instance-{}".format(instance_ip), + ip=instance_ip, + username=ssh_user_name, + privkey=ssh_private_key, + command='sudo cat {}/test'.format(self.mount_dir), + verify=verify_manila_testing_file) + + def test_manila_share(self): + """Test that a Manila share can be accessed on two instances. + + 1. Create a share + 2. Spawn two servers + 3. Mount it on both + 4. Write a file on one + 5. Read it on the other + 6. Profit + """ + # Create a share + share = self.manila_client.shares.create( + share_type=self.share_type_name, + name=self.share_name, + share_proto=self.share_protocol, + share_network=self.share_network, + size=1) + + # Spawn Servers + instance_1 = self.launch_guest( + guest_name='ins-1', + userdata=self.INSTANCE_USERDATA, + instance_key=self.INSTANCE_KEY) + instance_2 = self.launch_guest( + guest_name='ins-2', + userdata=self.INSTANCE_USERDATA, + instance_key=self.INSTANCE_KEY) + + fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] + fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] + + # Wait for the created share to become available before it gets used. + openstack_utils.resource_reaches_status( + self.manila_client.shares, + share.id, + wait_iteration_max_time=120, + stop_after_attempt=2, + expected_status="available", + msg="Waiting for a share to become available") + + # Grant access to the Manila share for both Nova instances. + share.allow(access_type='ip', access=fip_1, access_level='rw') + share.allow(access_type='ip', access=fip_2, access_level='rw') + + ssh_user_name = guest.boot_tests[self.INSTANCE_KEY]['username'] + privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) + share_path = share.export_locations[0] + + # Write a testing file on instance #1 + self._mount_share_on_instance( + fip_1, ssh_user_name, privkey, share_path) + self._write_testing_file_on_instance( + fip_1, ssh_user_name, privkey) + + # Validate the testing file from instance #2 + self._mount_share_on_instance( + fip_2, ssh_user_name, privkey, share_path) + self._validate_testing_file_from_instance( + fip_2, ssh_user_name, privkey) diff --git a/zaza/openstack/charm_tests/manila_ganesha/setup.py b/zaza/openstack/charm_tests/manila_ganesha/setup.py index d2b694e..a804958 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/setup.py +++ b/zaza/openstack/charm_tests/manila_ganesha/setup.py @@ -23,6 +23,9 @@ import zaza.openstack.utilities.openstack as openstack_utils from manilaclient import client as manilaclient +MANILA_GANESHA_TYPE_NAME = "cephfsnfstype" + + def setup_ganesha_share_type(manila_client=None): """Create a share type for manila with Ganesha. @@ -35,7 +38,7 @@ def setup_ganesha_share_type(manila_client=None): session=keystone_session, client_version='2') manila_client.share_types.create( - name="cephfsnfstype", spec_driver_handles_share_servers=False, + name=MANILA_GANESHA_TYPE_NAME, spec_driver_handles_share_servers=False, extra_specs={ 'vendor_name': 'Ceph', 'storage_protocol': 'NFS', diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 02ea825..f5d7b63 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -16,122 +16,20 @@ """Encapsulate Manila Ganesha testing.""" -import logging +from zaza.openstack.charm_tests.manila_ganesha.setup import ( + MANILA_GANESHA_TYPE_NAME, +) -from tenacity import Retrying, stop_after_attempt, wait_exponential - -from manilaclient import client as manilaclient - -import zaza.openstack.charm_tests.neutron.tests as neutron_tests -import zaza.openstack.charm_tests.nova.utils as nova_utils -import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.configure.guest as guest -import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.manila.tests as manila_tests -class ManilaGaneshaTests(test_utils.OpenStackBaseTest): +class ManilaGaneshaTests(manila_tests.ManilaBaseTest): """Encapsulate Manila Ganesha tests.""" - RESOURCE_PREFIX = 'zaza-manilatests' - INSTANCE_USERDATA = """#cloud-config -packages: -- nfs-common -""" - @classmethod def setUpClass(cls): """Run class setup for running tests.""" super(ManilaGaneshaTests, cls).setUpClass() - cls.nova_client = ( - openstack_utils.get_nova_session_client(cls.keystone_session)) - cls.manila_client = manilaclient.Client( - session=cls.keystone_session, client_version='2') - - def test_manila_share(self): - """Test that Manila + Ganesha shares can be accessed on two instances. - - 1. create a share - 2. Spawn two servers - 3. mount it on both - 4. write a file on one - 5. read it on the other - 6. profit - """ - # Create a share - share = self.manila_client.shares.create( - share_type='cephfsnfstype', name='cephnfsshare1', - share_proto="nfs", size=1) - - # Spawn Servers - instance_1, instance_2 = self.launch_guests( - userdata=self.INSTANCE_USERDATA) - - fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] - fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] - - # Wait for the created share to become available before it gets used. - openstack_utils.resource_reaches_status( - self.manila_client.shares, - share.id, - wait_iteration_max_time=120, - stop_after_attempt=2, - expected_status="available", - msg="Waiting for a share to become available") - - share.allow(access_type='ip', access=fip_1, access_level='rw') - share.allow(access_type='ip', access=fip_2, access_level='rw') - - # Mount Share - username = guest.boot_tests['bionic']['username'] - password = guest.boot_tests['bionic'].get('password') - privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) - - # Write a file on instance_1 - def verify_setup(stdin, stdout, stderr): - status = stdout.channel.recv_exit_status() - if status: - logging.info("{}".format(stderr.readlines()[0].strip())) - self.assertEqual(status, 0) - - mount_path = share.export_locations[0] - - for attempt in Retrying( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10)): - with attempt: - openstack_utils.ssh_command( - username, fip_1, 'instance-1', - 'sudo mkdir -p /mnt/ceph && ' - 'sudo mount -t nfs -o nfsvers=4.1,proto=tcp ' - '{} /mnt/ceph && ' - 'echo "test" | sudo tee /mnt/ceph/test'.format( - mount_path), - password=password, privkey=privkey, verify=verify_setup) - - for attempt in Retrying( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10)): - with attempt: - # Setup that file on instance_2 - openstack_utils.ssh_command( - username, fip_2, 'instance-2', - 'sudo mkdir -p /mnt/ceph && ' - 'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp ' - '{} /mnt/ceph' - .format(mount_path), - password=password, privkey=privkey, verify=verify_setup) - - def verify(stdin, stdout, stderr): - status = stdout.channel.recv_exit_status() - if status: - logging.info("{}".format(stderr.readlines()[0].strip())) - self.assertEqual(status, 0) - out = "" - for line in iter(stdout.readline, ""): - out += line - self.assertEqual(out, "test\n") - - openstack_utils.ssh_command( - username, fip_2, 'instance-2', - 'sudo cat /mnt/ceph/test', - password=password, privkey=privkey, verify=verify) + cls.share_name = 'cephnfsshare1' + cls.share_type_name = MANILA_GANESHA_TYPE_NAME + cls.share_protocol = 'nfs' From 42d955209a422f6cccb887c369e10163d6484110 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 4 Mar 2021 13:55:09 +0000 Subject: [PATCH 813/898] Add setup code for s3 back Trilio Add code to prepare trilio applications to use an s3 backend. --- zaza/openstack/charm_tests/trilio/setup.py | 81 +++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/setup.py b/zaza/openstack/charm_tests/trilio/setup.py index 44f1d9d..1c8e97c 100644 --- a/zaza/openstack/charm_tests/trilio/setup.py +++ b/zaza/openstack/charm_tests/trilio/setup.py @@ -19,12 +19,16 @@ import logging import os +import boto3 + +import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.model as zaza_model import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.openstack as openstack_utils -def basic_setup(): +def nfs_setup(): """Run setup for testing Trilio. Setup for testing Trilio is currently part of functional @@ -54,6 +58,9 @@ def basic_setup(): target_status="active", ) + +def trust_setup(): + """Run setup Trilio trust setup.""" logging.info("Executing create-cloud-admin-trust") password = juju_utils.leader_get("keystone", "admin_passwd") @@ -66,6 +73,9 @@ def basic_setup(): ) ) + +def license_setup(): + """Run setup Trilio license setup.""" logging.info("Executing create-license") test_license = os.environ.get("TEST_TRILIO_LICENSE") if test_license and os.path.exists(test_license): @@ -83,6 +93,75 @@ def basic_setup(): logging.error("Unable to find Trilio License file") +def s3_setup(): + """Run setup of s3 options for Trilio.""" + session = openstack_utils.get_overcloud_keystone_session() + ks_client = openstack_utils.get_keystone_session_client( + session) + + # Get token data so we can glean our user_id and project_id + token_data = ks_client.tokens.get_token_data(session.get_token()) + project_id = token_data['token']['project']['id'] + user_id = token_data['token']['user']['id'] + + # Store URL to service providing S3 compatible API + for entry in token_data['token']['catalog']: + if entry['type'] == 's3': + for endpoint in entry['endpoints']: + if endpoint['interface'] == 'public': + s3_region = endpoint['region'] + s3_endpoint = endpoint['url'] + + # Create AWS compatible application credentials in Keystone + ec2_creds = ks_client.ec2.create(user_id, project_id) + cacert = openstack_utils.get_cacert() + kwargs = { + 'region_name': s3_region, + 'aws_access_key_id': ec2_creds.access, + 'aws_secret_access_key': ec2_creds.secret, + 'endpoint_url': s3_endpoint, + 'verify': cacert, + } + s3 = boto3.resource('s3', **kwargs) + + # Create bucket + bucket_name = 'zaza-trilio' + logging.info("Creating bucket: {}".format(bucket_name)) + bucket = s3.Bucket(bucket_name) + bucket.create() + + s3_config = { + 'tv-s3-secret-key': ec2_creds.secret, + 'tv-s3-access-key': ec2_creds.access, + 'tv-s3-region-name': s3_region, + 'tv-s3-bucket': bucket_name, + 'tv-s3-endpoint-url': s3_endpoint} + for app in ['trilio-wlm', 'trilio-data-mover']: + logging.info("Setting s3 config for {}".format(app)) + zaza_model.set_application_config(app, s3_config) + test_config = lifecycle_utils.get_charm_config(fatal=False) + states = test_config.get('target_deploy_status', {}) + states['trilio-wlm'] = { + 'workload-status': 'blocked', + 'workload-status-message': 'application not trusted'} + zaza_model.wait_for_application_states( + states=test_config.get('target_deploy_status', {}), + timeout=7200) + zaza_model.block_until_all_units_idle() + + +def basic_setup(): + """Run basic setup for Trilio apps.""" + backup_target_type = zaza_model.get_application_config( + 'trilio-wlm')['backup-target-type']['value'] + if backup_target_type == "nfs": + nfs_setup() + if backup_target_type in ["s3", "experimental-s3"]: + s3_setup() + trust_setup() + license_setup() + + def python2_workaround(): """Workaround for Bug #1915914. From 93a48133b35e20d79251922c321d97b1ddf03e4e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 5 Mar 2021 12:12:00 +0100 Subject: [PATCH 814/898] Add assertions to HaclusterScaleBackAndForthTest --- zaza/openstack/charm_tests/hacluster/tests.py | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 66d4f45..53c7392 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -189,6 +189,10 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): # aren't guaranteed to be blocked, so we don't validate that here. zaza.model.block_until_all_units_idle() + # At this point the corosync ring hasn't been updated yet, so it should + # still remember the deleted unit: + self.__assert_some_corosync_nodes_are_offline(surviving_hacluster_unit) + logging.info('Updating corosync ring') hacluster_app_name = zaza.model.get_unit_from_name( surviving_hacluster_unit).application @@ -198,6 +202,10 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): action_params={'i-really-mean-it': True}, raise_on_failure=True) + # At this point if the corosync ring has been properly updated, there + # shouldn't be any trace of the deleted unit anymore: + self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit) + logging.info('Re-adding an hacluster unit') zaza.model.add_unit(self._principle_app_name, wait_appear=True) @@ -212,15 +220,30 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): 'active') zaza.model.block_until_all_units_idle() - # At this point if the corosync ring has been properly updated, there - # shouldn't be any trace of the deleted unit anymore: + # At this point the corosync ring should still not contain any offline + # node: + self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit) + + def __assert_some_corosync_nodes_are_offline(self, hacluster_unit): + logging.info('Checking that corosync considers at least one node to ' + 'be offline') + output = self._get_crm_status(hacluster_unit) + self.assertIn('OFFLINE', output, + "corosync should list at least one offline node") + + def __assert_all_corosync_nodes_are_online(self, hacluster_unit): logging.info('Checking that corosync considers all nodes to be online') + output = self._get_crm_status(hacluster_unit) + self.assertNotIn('OFFLINE', output, + "corosync shouldn't list any offline node") + + @staticmethod + def _get_crm_status(hacluster_unit): cmd = 'sudo crm status' - result = zaza.model.run_on_unit(surviving_hacluster_unit, cmd) + result = zaza.model.run_on_unit(hacluster_unit, cmd) code = result.get('Code') if code != '0': raise zaza.model.CommandRunFailed(cmd, result) output = result.get('Stdout').strip() - logging.debug('Output received: {}'.format(output)) - self.assertNotIn('OFFLINE', output, - "corosync shouldn't list any offline node") + logging.debug('crm output received: {}'.format(output)) + return output From aafdc4070f0984d87dea05008de9a86e0beed920 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 5 Mar 2021 18:08:54 +0100 Subject: [PATCH 815/898] Workaround for lp:1874719 --- zaza/openstack/charm_tests/hacluster/tests.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 53c7392..5f46565 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -220,8 +220,17 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): 'active') zaza.model.block_until_all_units_idle() - # At this point the corosync ring should still not contain any offline - # node: + # Because of lp:1874719 the corosync ring may show a mysterious offline + # 'node1' node. We clean up the ring by re-running the 'update-ring' + # action: + logging.info('Updating corosync ring - workaround for lp:1874719') + zaza.model.run_action_on_leader( + hacluster_app_name, + 'update-ring', + action_params={'i-really-mean-it': True}, + raise_on_failure=True) + + # At this point the corosync ring should not contain any offline node: self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit) def __assert_some_corosync_nodes_are_offline(self, hacluster_unit): From 79a78c2ca04424f39aa14ca6aa45cb7c240f74f5 Mon Sep 17 00:00:00 2001 From: "Bartosz \"mastier\" Woronicz" Date: Mon, 8 Mar 2021 08:59:23 +0100 Subject: [PATCH 816/898] fix reload test for vault (#517) The test was failing for xenial-ha-mysql bundle This fixes the issue for cluster vault by picking the right vault cluster leader to run the commands against. Co-authored-by: Bartosz Woronicz --- zaza/openstack/charm_tests/vault/tests.py | 17 +++++++++++------ zaza/openstack/charm_tests/vault/utils.py | 18 +++++++++++++++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 219291e..8c3daca 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -31,6 +31,8 @@ import zaza.openstack.charm_tests.vault.utils as vault_utils import zaza.openstack.utilities.cert import zaza.openstack.utilities.openstack import zaza.model +import zaza.utilities.juju as juju_utils + class BaseVaultTest(test_utils.OpenStackBaseTest): @@ -268,7 +270,7 @@ class VaultTest(BaseVaultTest): raise unittest.SkipTest("The version of charm-vault tested does " "not have reload action") - lead_client = vault_utils.extract_lead_unit_client(self.clients) + lead_client = vault_utils.get_cluster_leader(self.clients) running_config = vault_utils.get_running_config(lead_client) value_to_set = not running_config['data']['disable_mlock'] @@ -277,15 +279,18 @@ class VaultTest(BaseVaultTest): {'disable-mlock': str(value_to_set)}) logging.info("Testing reload") - zaza.model.run_action_on_leader( - 'vault', + zaza.model.run_action( + juju_utils.get_unit_name_from_ip_address( + lead_client.addr, 'vault'), 'reload', - action_params={}) + model_name=self.model_name) + new_value = vault_utils.get_running_config(lead_client)[ + 'data']['disable_mlock'] + logging.info(new_value) self.assertEqual( value_to_set, - vault_utils.get_running_config(lead_client)[ - 'data']['disable_mlock']) + new_value) self.assertFalse(lead_client.hvac_client.seal_status['sealed']) def test_vault_restart(self): diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 9098b12..97fd060 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -137,10 +137,26 @@ def get_vip_client(cacert=None): return client +def get_cluster_leader(clients): + """Get Vault cluster leader + :param clients: Clients list to get leader + :type clients: List of CharmVaultClient + We have to make sure we run api calls + against the actual leader + """ + if len(clients) == 1: + return clients[0] + + for client in clients: + if client.hvac_client.ha_status['is_self']: + return client + return None + + def get_running_config(client): """Get Vault running config. - :param client: Client to use for initiliasation + :param client: Client used to get config :type client: CharmVaultClient The hvac library does not support getting info from endpoint /v1/sys/config/state/sanitized From 975f5058f44d3a2c1bb4b1e28e635f30482e02d1 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 8 Mar 2021 11:46:15 +0100 Subject: [PATCH 817/898] Add github actions support with linting and unit tests --- .github/workflows/tox.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/tox.yaml diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml new file mode 100644 index 0000000..880048a --- /dev/null +++ b/.github/workflows/tox.yaml @@ -0,0 +1,27 @@ +name: Python package + +on: + - push + - pull_request + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.4, 3.5, 3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + - name: Lint with tox + run: tox -e pep8 + - name: Test with tox + run: tox -e py${{ matrix.python-version }} \ No newline at end of file From 2d753a32bababc6b31aba116b098188816f4497a Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 8 Mar 2021 15:45:46 +0100 Subject: [PATCH 818/898] remove python 3.4 support --- .github/workflows/tox.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index 880048a..754ce29 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.4, 3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v1 From d7ec4fd0b8e70f6f96323c7dddcde4b30169a124 Mon Sep 17 00:00:00 2001 From: Bartosz Woronicz Date: Mon, 8 Mar 2021 16:18:52 +0100 Subject: [PATCH 819/898] fix formatting issues and docstring That fixes the issues introduced in PR #512 #517 Now flake8 should not complain --- zaza/openstack/charm_tests/vault/tests.py | 1 - zaza/openstack/charm_tests/vault/utils.py | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 8c3daca..c5211e0 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -34,7 +34,6 @@ import zaza.model import zaza.utilities.juju as juju_utils - class BaseVaultTest(test_utils.OpenStackBaseTest): """Base class for vault tests.""" diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 97fd060..60ac429 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -138,9 +138,12 @@ def get_vip_client(cacert=None): def get_cluster_leader(clients): - """Get Vault cluster leader + """Get Vault cluster leader. + :param clients: Clients list to get leader :type clients: List of CharmVaultClient + :returns: CharmVaultClient + :rtype: CharmVaultClient or None We have to make sure we run api calls against the actual leader """ @@ -158,6 +161,8 @@ def get_running_config(client): :param client: Client used to get config :type client: CharmVaultClient + :returns: dict from Vault api response + :rtype: dict The hvac library does not support getting info from endpoint /v1/sys/config/state/sanitized Therefore we implement it here From 4551f665059542bac1320f3dba067209d30dd71f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Tue, 9 Mar 2021 17:59:33 +0000 Subject: [PATCH 820/898] Fix the kerberos set-up by loosening ubuntu check (#521) * Fix the kerberos set-up by loosening ubuntu check The cs:ubuntu charm's workload-status-message has changed recently (as the charm was re-written). This PR loosens the application states check to ignore the workload-status-message and just wait for the charm to be active workload-status. This is almost certainly enough, as it's a very simple charm. * Fix tox.ini to pass tests (maybe) --- tox.ini | 20 ++++++++++++++++++++ zaza/openstack/charm_tests/kerberos/setup.py | 10 +++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index dc09948..aea173f 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,26 @@ commands = nosetests --with-coverage --cover-package=zaza.openstack {posargs} {t basepython = python3 deps = -r{toxinidir}/requirements.txt +[testenv:py3.5] +basepython = python3.5 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.6] +basepython = python3.6 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.7] +basepython = python3.7 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.8] +basepython = python3.8 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.9] +basepython = python3.9 +deps = -r{toxinidir}/requirements.txt + [testenv:pep8] basepython = python3 deps = -r{toxinidir}/requirements.txt diff --git a/zaza/openstack/charm_tests/kerberos/setup.py b/zaza/openstack/charm_tests/kerberos/setup.py index d441f8c..11f4afa 100644 --- a/zaza/openstack/charm_tests/kerberos/setup.py +++ b/zaza/openstack/charm_tests/kerberos/setup.py @@ -113,7 +113,15 @@ def retrieve_and_attach_keytab(): 'keystone_keytab', tmp_file) - zaza.model.wait_for_application_states() + # cs:ubuntu charm has changed behaviour and we can't rely on the workload + # staus message. Thus, ignore it. + states = { + "ubuntu-test-host": { + "workload-status": "active", + "workload-status-message": "", + } + } + zaza.model.wait_for_application_states(states=states) zaza.model.block_until_all_units_idle() From c6e54dd50526efea014324173877d84ac396dbc9 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 9 Mar 2021 18:09:37 +0000 Subject: [PATCH 821/898] Clean-up/correct docstrings in vault/utils.py --- zaza/openstack/charm_tests/vault/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 60ac429..40e33ac 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -140,12 +140,12 @@ def get_vip_client(cacert=None): def get_cluster_leader(clients): """Get Vault cluster leader. + We have to make sure we run api calls against the actual leader. + :param clients: Clients list to get leader :type clients: List of CharmVaultClient :returns: CharmVaultClient :rtype: CharmVaultClient or None - We have to make sure we run api calls - against the actual leader """ if len(clients) == 1: return clients[0] @@ -159,13 +159,13 @@ def get_cluster_leader(clients): def get_running_config(client): """Get Vault running config. + The hvac library does not support getting info from endpoint + /v1/sys/config/state/sanitized Therefore we implement it here + :param client: Client used to get config :type client: CharmVaultClient :returns: dict from Vault api response :rtype: dict - The hvac library does not support getting info - from endpoint /v1/sys/config/state/sanitized - Therefore we implement it here """ return requests.get( client.hvac_client.adapter.base_uri + '/v1/sys/config/state/sanitized', From 45146b6c45d306cb0312fc97509b5eca4c075c95 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 3 Mar 2021 14:54:47 +0000 Subject: [PATCH 822/898] Add ObjectRetrier to perform retries on openstack client calls This adds a wrapper class that detects if a callable object in any of the descendent objects raises an Exception. If so, then it retries that exception. This is to attempt to make the zaza tests a little more robust in the face of small network failures or strange restarts. This is a test, and robust logging a reporting should be used to determine whether it is covering up actual bugs rather than CI system issues. Related Bug: (zot repo)#348 --- unit_tests/utilities/test_utilities.py | 166 +++++++++++++++++++++++++ zaza/openstack/utilities/__init__.py | 128 +++++++++++++++++++ 2 files changed, 294 insertions(+) create mode 100644 unit_tests/utilities/test_utilities.py diff --git a/unit_tests/utilities/test_utilities.py b/unit_tests/utilities/test_utilities.py new file mode 100644 index 0000000..e961de1 --- /dev/null +++ b/unit_tests/utilities/test_utilities.py @@ -0,0 +1,166 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import unit_tests.utils as ut_utils + +import zaza.openstack.utilities as utilities + + +class SomeException(Exception): + pass + + +class SomeException2(Exception): + pass + + +class SomeException3(Exception): + pass + + +class TestObjectRetrier(ut_utils.BaseTestCase): + + def test_object_wrap(self): + + class A: + + def func(self, a, b=1): + return a + b + + a = A() + wrapped_a = utilities.ObjectRetrier(a) + self.assertEqual(wrapped_a.func(3), 4) + + def test_object_multilevel_wrap(self): + + class A: + + def f1(self, a, b): + return a * b + + class B: + + @property + def f2(self): + + return A() + + b = B() + wrapped_b = utilities.ObjectRetrier(b) + self.assertEqual(wrapped_b.f2.f1(5, 6), 30) + + def test_object_wrap_number(self): + + class A: + + class_a = 5 + + def __init__(self): + self.instance_a = 10 + + def f1(self, a, b): + return a * b + + a = A() + wrapped_a = utilities.ObjectRetrier(a) + self.assertEqual(wrapped_a.class_a, 5) + self.assertEqual(wrapped_a.instance_a, 10) + + @mock.patch("time.sleep") + def test_object_wrap_exception(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + # retry on a specific exception + wrapped_a = utilities.ObjectRetrier(a, num_retries=1, + retry_exceptions=[SomeException]) + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_called_once_with(5) + + # also retry on any exception if none specified + wrapped_a = utilities.ObjectRetrier(a, num_retries=1) + mock_sleep.reset_mock() + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_called_once_with(5) + + # no retry if exception isn't listed. + wrapped_a = utilities.ObjectRetrier(a, num_retries=1, + retry_exceptions=[SomeException2]) + mock_sleep.reset_mock() + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_not_called() + + @mock.patch("time.sleep") + def test_log_called(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + mock_log = mock.Mock() + wrapped_a = utilities.ObjectRetrier(a, num_retries=1, log=mock_log) + with self.assertRaises(SomeException): + wrapped_a.func() + + # there should be two calls; one for the single retry and one for the + # failure. + self.assertEqual(mock_log.call_count, 2) + + @mock.patch("time.sleep") + def test_back_off_maximum(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + wrapped_a = utilities.ObjectRetrier(a, num_retries=3, backoff=2) + with self.assertRaises(SomeException): + wrapped_a.func() + # Note third call hits maximum wait time of 15. + mock_sleep.assert_has_calls([mock.call(5), + mock.call(10), + mock.call(15)]) + + @mock.patch("time.sleep") + def test_total_wait(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + wrapped_a = utilities.ObjectRetrier(a, num_retries=3, total_wait=9) + with self.assertRaises(SomeException): + wrapped_a.func() + # Note only two calls, as total wait is 9, so a 3rd retry would exceed + # that. + mock_sleep.assert_has_calls([mock.call(5), + mock.call(5)]) diff --git a/zaza/openstack/utilities/__init__.py b/zaza/openstack/utilities/__init__.py index 35b5a14..5011689 100644 --- a/zaza/openstack/utilities/__init__.py +++ b/zaza/openstack/utilities/__init__.py @@ -13,3 +13,131 @@ # limitations under the License. """Collection of utilities to support zaza tests etc.""" + + +import time + + +class ObjectRetrier(object): + """An automatic retrier for an object. + + This is designed to be used with an instance of an object. Basically, it + wraps the object and any attributes that are fetched. Essentially, it is + used to provide retries on method calls on openstack client objects in + tests to increase robustness of tests. + + Although, technically this is bad, retries can be logged with the optional + log method. + + Usage: + + # get a client that does 3 retries, waits 5 seconds between retries and + # retries on any error. + some_client = ObjectRetrier(get_some_client) + # this gets retried up to 3 times. + things = some_client.list_things() + + Note, it is quite simple. It wraps the object and on a getattr(obj, name) + it finds the name and then returns a wrapped version of that name. On a + call, it returns the value of that call. It only wraps objects in the + chain that are either callable or have a __getattr__() method. i.e. one + that can then be retried or further fetched. This means that if a.b.c() is + a chain of objects, and we just wrap 'a', then 'b' and 'c' will both be + wrapped that the 'c' object __call__() method will be the one that is + actually retried. + + Note: this means that properties that do method calls won't be retried. + This is a limitation that may be addressed in the future, if it is needed. + """ + + def __init__(self, obj, num_retries=3, initial_interval=5.0, backoff=1.0, + max_interval=15.0, total_wait=30.0, retry_exceptions=None, + log=None): + """Initialise the retrier object. + + :param obj: The object to wrap. Ought to be an instance of something + that you want to get methods on to call or be called itself. + :type obj: Any + :param num_retries: The (maximum) number of retries. May not be hit if + the total_wait time is exceeded. + :type num_retries: int + :param initial_interval: The initial or starting interval between + retries. + :type initial_interval: float + :param backoff: The exponential backoff multiple. 1 is linear. + :type backoff: float + :param max_interval: The maximum interval between retries. + If backoff is >1 then the initial_interval will never grow larger + than max_interval. + :type max_interval: float + :param retry_exceptions: The list of exceptions to retry on, or None. + If a list, then it will only retry if the exception is one of the + ones in the list. + :type retry_exceptions: List[Exception] + """ + # Note we use semi-private variable names that shouldn't clash with any + # on the actual object. + self.__obj = obj + self.__kwargs = { + 'num_retries': num_retries, + 'initial_interval': initial_interval, + 'backoff': backoff, + 'max_interval': max_interval, + 'total_wait': total_wait, + 'retry_exceptions': retry_exceptions, + 'log': log or (lambda x: None), + } + + def __getattr__(self, name): + """Get attribute; delegates to wrapped object.""" + # Note the above may generate an attribute error; we expect this and + # will fail with an attribute error. + attr = getattr(self.__obj, name) + if callable(attr) or hasattr(attr, "__getattr__"): + return ObjectRetrier(attr, **self.__kwargs) + else: + return attr + # TODO(ajkavanagh): Note detecting a property is a bit trickier. we + # can do isinstance(attr, property), but then the act of accessing it + # is what calls it. i.e. it would fail at the getattr(self.__obj, + # name) stage. The solution is to check first, and if it's a property, + # then treat it like the retrier. However, I think this is too + # complex for the first go, and to use manual retries in that instance. + + def __call__(self, *args, **kwargs): + """Call the object; delegates to the wrapped object.""" + obj = self.__obj + retry = 0 + wait = self.__kwargs['initial_interval'] + max_interval = self.__kwargs['max_interval'] + log = self.__kwargs['log'] + backoff = self.__kwargs['backoff'] + total_wait = self.__kwargs['total_wait'] + num_retries = self.__kwargs['num_retries'] + retry_exceptions = self.__kwargs['retry_exceptions'] + wait_so_far = 0 + while True: + try: + return obj(*args, **kwargs) + except Exception as e: + # if retry_exceptions is None, or the type of the exception is + # not in the list of retries, then raise an exception + # immediately. This means that if retry_exceptions is None, + # then the method is always retried. + if (retry_exceptions is not None and + type(e) not in retry_exceptions): + raise + retry += 1 + if retry > num_retries: + log("{}: exceeded number of retries, so erroring out" + .format(str(obj))) + raise e + log("{}: call failed: retrying in {} seconds" + .format(str(obj), wait)) + time.sleep(wait) + wait_so_far += wait + if wait_so_far >= total_wait: + raise e + wait = wait * backoff + if wait > max_interval: + wait = max_interval From ed8528b76a842d5729ed6f5700d4cd4270de1c8a Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 3 Mar 2021 14:58:27 +0000 Subject: [PATCH 823/898] Add object retrier to the LB tests for octavia --- zaza/openstack/charm_tests/octavia/tests.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 26941a2..d6bae64 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -25,6 +25,8 @@ import osc_lib.exceptions import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils +from zaza.openstack import ObjectRetrier + class CharmOperationTest(test_utils.OpenStackBaseTest): """Charm operation tests.""" @@ -63,12 +65,12 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run class setup for running LBaaSv2 service tests.""" super(LBAASv2Test, cls).setUpClass() - cls.keystone_client = openstack_utils.get_keystone_session_client( - cls.keystone_session) - cls.neutron_client = openstack_utils.get_neutron_session_client( - cls.keystone_session) - cls.octavia_client = openstack_utils.get_octavia_session_client( - cls.keystone_session) + cls.keystone_client = ObjectRetrier( + openstack_utils.get_keystone_session_client(cls.keystone_session)) + cls.neutron_client = ObjectRetrier( + openstack_utils.get_neutron_session_client(cls.keystone_session)) + cls.octavia_client = ObjectRetrier( + openstack_utils.get_octavia_session_client(cls.keystone_session)) cls.RESOURCE_PREFIX = 'zaza-octavia' # NOTE(fnordahl): in the event of a test failure we do not want to run From c041042fe220fa5de33f8006734c66acb8255f11 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 8 Mar 2021 10:50:41 +0000 Subject: [PATCH 824/898] Add function to just retry on keystone ConnectFailure The main failure that seems to occur with clients is the ConnectFailure, which according to the docs, is retry-able. Thus provide a function that adds that exception condition automatically. Also fix the import problem in octavia test for the object retrier that is being used to validate the ObjectRetrier feature. --- unit_tests/utilities/test_utilities.py | 20 ++++++++++++++++++ zaza/openstack/charm_tests/octavia/tests.py | 2 +- zaza/openstack/utilities/__init__.py | 23 +++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/unit_tests/utilities/test_utilities.py b/unit_tests/utilities/test_utilities.py index e961de1..a78600c 100644 --- a/unit_tests/utilities/test_utilities.py +++ b/unit_tests/utilities/test_utilities.py @@ -164,3 +164,23 @@ class TestObjectRetrier(ut_utils.BaseTestCase): # that. mock_sleep.assert_has_calls([mock.call(5), mock.call(5)]) + + @mock.patch("time.sleep") + def test_retry_on_connect_failure(self, mock_sleep): + + class A: + + def func1(self): + raise SomeException() + + def func2(self): + raise utilities.ConnectFailure() + + a = A() + wrapped_a = utilities.retry_on_connect_failure(a, num_retries=2) + with self.assertRaises(SomeException): + wrapped_a.func1() + mock_sleep.assert_not_called() + with self.assertRaises(utilities.ConnectFailure): + wrapped_a.func2() + mock_sleep.assert_has_calls([mock.call(5)]) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index d6bae64..9a71b49 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -25,7 +25,7 @@ import osc_lib.exceptions import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils -from zaza.openstack import ObjectRetrier +from zaza.openstack.utilities import ObjectRetrier class CharmOperationTest(test_utils.OpenStackBaseTest): diff --git a/zaza/openstack/utilities/__init__.py b/zaza/openstack/utilities/__init__.py index 5011689..6db3ce3 100644 --- a/zaza/openstack/utilities/__init__.py +++ b/zaza/openstack/utilities/__init__.py @@ -17,6 +17,8 @@ import time +from keystoneauth1.exceptions.connection import ConnectFailure + class ObjectRetrier(object): """An automatic retrier for an object. @@ -141,3 +143,24 @@ class ObjectRetrier(object): wait = wait * backoff if wait > max_interval: wait = max_interval + + +def retry_on_connect_failure(client, **kwargs): + """Retry an object that eventually gets resolved to a call. + + Specifically, this uses ObjectRetrier but only against the + keystoneauth1.exceptions.connection.ConnectFailure exeception. + + :params client: the object that may throw and exception when called. + :type client: Any + :params **kwargs: the arguments supplied to the ObjectRetrier init method + :type **kwargs: Dict[Any] + :returns: client wrapped in an ObjectRetrier instance + :rtype: ObjectRetrier[client] + """ + kwcopy = kwargs.copy() + if 'retry_exceptions' not in kwcopy: + kwcopy['retry_exceptions'] = [] + if ConnectFailure not in kwcopy['retry_exceptions']: + kwcopy['retry_exceptions'].append(ConnectFailure) + return ObjectRetrier(client, **kwcopy) From 74c5c9d69c979a51b48bf8bd0eedd34dd0eb8883 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Fri, 12 Mar 2021 14:53:21 +0200 Subject: [PATCH 825/898] Fix _write_testing_file_on_instance docstring --- zaza/openstack/charm_tests/manila/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index e7cba8f..1b58f65 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -208,9 +208,9 @@ packages: ssh_private_key): """Write a file on a Manila share mounted into a Nova instance. - Mount the Manila share in the given Nova instance, and write a testing - file on it (which is meant to be validated from another instance). - These commands are executed via SSH. + Write a testing file into the already mounted Manila share from the + given Nova instance (which is meant to be validated from another + instance). These commands are executed via SSH. :param instance_ip: IP of the Nova instance. :type instance_ip: string From 7e568138857307ee445d9552a2717088b550f6f0 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Sun, 28 Feb 2021 22:39:29 +0200 Subject: [PATCH 826/898] Add Manila NetApp tests These Zaza tests are meant to be used with the new `manila-netapp` charm. --- .../charm_tests/manila_netapp/__init__.py | 17 ++++ .../charm_tests/manila_netapp/setup.py | 96 +++++++++++++++++++ .../charm_tests/manila_netapp/tests.py | 51 ++++++++++ zaza/openstack/utilities/openstack.py | 14 +++ 4 files changed, 178 insertions(+) create mode 100644 zaza/openstack/charm_tests/manila_netapp/__init__.py create mode 100644 zaza/openstack/charm_tests/manila_netapp/setup.py create mode 100644 zaza/openstack/charm_tests/manila_netapp/tests.py diff --git a/zaza/openstack/charm_tests/manila_netapp/__init__.py b/zaza/openstack/charm_tests/manila_netapp/__init__.py new file mode 100644 index 0000000..f6e2865 --- /dev/null +++ b/zaza/openstack/charm_tests/manila_netapp/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila NetApp setup and testing.""" diff --git a/zaza/openstack/charm_tests/manila_netapp/setup.py b/zaza/openstack/charm_tests/manila_netapp/setup.py new file mode 100644 index 0000000..fa1a671 --- /dev/null +++ b/zaza/openstack/charm_tests/manila_netapp/setup.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila NetApp setup.""" + +import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.neutron.setup as neutron_setup + + +MANILA_NETAPP_TYPE_NAME = "netapp-ontap" +MANILA_NETAPP_BACKEND_NAME = "netapp-ontap" + +MANILA_NETAPP_DHSS_TYPE_NAME = "netapp-ontap-dhss" +MANILA_NETAPP_DHSS_BACKEND_NAME = "netapp-ontap-dhss" + +MANILA_NETAPP_SHARE_NET_NAME = "netapp-ontap-share-network" + + +def create_netapp_share_type(manila_client=None): + """Create a share type for Manila with NetApp Data ONTAP driver. + + :param manila_client: Authenticated manilaclient + :type manila_client: manilaclient.Client + """ + if manila_client is None: + manila_client = openstack_utils.get_manila_session_client( + openstack_utils.get_overcloud_keystone_session()) + + manila_client.share_types.create( + name=MANILA_NETAPP_TYPE_NAME, + spec_driver_handles_share_servers=False, + extra_specs={ + 'vendor_name': 'NetApp', + 'share_backend_name': MANILA_NETAPP_BACKEND_NAME, + 'storage_protocol': 'NFS_CIFS', + }) + + +def create_netapp_dhss_share_type(manila_client=None): + """Create a DHSS share type for Manila with NetApp Data ONTAP driver. + + :param manila_client: Authenticated manilaclient + :type manila_client: manilaclient.Client + """ + if manila_client is None: + manila_client = openstack_utils.get_manila_session_client( + openstack_utils.get_overcloud_keystone_session()) + + manila_client.share_types.create( + name=MANILA_NETAPP_DHSS_TYPE_NAME, + spec_driver_handles_share_servers=True, + extra_specs={ + 'vendor_name': 'NetApp', + 'share_backend_name': MANILA_NETAPP_DHSS_BACKEND_NAME, + 'storage_protocol': 'NFS_CIFS', + }) + + +def create_netapp_share_network(manila_client=None): + """Create a Manila share network from the existing provider network. + + This setup function assumes that 'neutron.setup.basic_overcloud_network' + is called to have the proper tenant networks setup. + + The share network will be bound to the provider network configured by + 'neutron.setup.basic_overcloud_network'. + """ + session = openstack_utils.get_overcloud_keystone_session() + if manila_client is None: + manila_client = openstack_utils.get_manila_session_client(session) + + neutron = openstack_utils.get_neutron_session_client(session) + external_net = neutron.find_resource( + 'network', + neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_net_name']) + external_subnet = neutron.find_resource( + 'subnet', + neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_subnet_name']) + + manila_client.share_networks.create( + name=MANILA_NETAPP_SHARE_NET_NAME, + neutron_net_id=external_net['id'], + neutron_subnet_id=external_subnet['id']) diff --git a/zaza/openstack/charm_tests/manila_netapp/tests.py b/zaza/openstack/charm_tests/manila_netapp/tests.py new file mode 100644 index 0000000..f9178ba --- /dev/null +++ b/zaza/openstack/charm_tests/manila_netapp/tests.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila NetApp testing.""" + +from zaza.openstack.charm_tests.manila_netapp.setup import ( + MANILA_NETAPP_TYPE_NAME, + MANILA_NETAPP_DHSS_TYPE_NAME, + MANILA_NETAPP_SHARE_NET_NAME, +) + +import zaza.openstack.charm_tests.manila.tests as manila_tests + + +class ManilaNetAppNFSTest(manila_tests.ManilaBaseTest): + """Encapsulate Manila NetApp NFS test.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaNetAppNFSTest, cls).setUpClass() + cls.share_name = 'netapp-ontap-share' + cls.share_type_name = MANILA_NETAPP_TYPE_NAME + cls.share_protocol = 'nfs' + + +class ManilaNetAppDHSSNFSTest(manila_tests.ManilaBaseTest): + """Encapsulate Manila NetApp NFS test.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaNetAppDHSSNFSTest, cls).setUpClass() + cls.share_name = 'netapp-ontap-dhss-share' + cls.share_type_name = MANILA_NETAPP_DHSS_TYPE_NAME + cls.share_protocol = 'nfs' + cls.share_network = cls.manila_client.share_networks.find( + name=MANILA_NETAPP_SHARE_NET_NAME) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 4ed75b9..230a3f6 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -69,6 +69,7 @@ from neutronclient.v2_0 import client as neutronclient from neutronclient.common import exceptions as neutronexceptions from octaviaclient.api.v2 import octavia as octaviaclient from swiftclient import client as swiftclient +from manilaclient import client as manilaclient from juju.errors import JujuError @@ -449,6 +450,19 @@ def get_aodh_session_client(session): return aodh_client.Client(session=session) +def get_manila_session_client(session, version='2'): + """Return Manila client authenticated by keystone session. + + :param session: Keystone session object + :type session: keystoneauth1.session.Session object + :param version: Manila API version + :type version: str + :returns: Authenticated manilaclient + :rtype: manilaclient.Client + """ + return manilaclient.Client(session=session, client_version=version) + + def get_keystone_scope(model_name=None): """Return Keystone scope based on OpenStack release of the overcloud. From fae2f6319e730285866c77c98b86282511b8f432 Mon Sep 17 00:00:00 2001 From: Peter Matulis Date: Mon, 15 Mar 2021 13:17:22 -0400 Subject: [PATCH 827/898] Add upgrades doc note (#526) * Add upgrades doc note Add a note so that upgrade testing (charms and payload) remains in sync with the end-user documentation (CDG) --- zaza/openstack/utilities/upgrade_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 8e3a063..4f00592 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -26,7 +26,13 @@ from zaza.openstack.utilities.os_versions import ( OPENSTACK_RELEASES_PAIRS, ) - +""" +The below upgrade order is surfaced in end-user documentation. Any change to +it should be accompanied by an update to the OpenStack Charms Deployment Guide +for both charm upgrades and payload upgrades: +- source/upgrade-charms.rst#upgrade-order +- source/upgrade-openstack.rst#openstack_upgrade_order +""" SERVICE_GROUPS = ( ('Database Services', ['percona-cluster', 'mysql-innodb-cluster']), ('Stateful Services', ['rabbitmq-server', 'ceph-mon']), From e3fab555ccc84f886cd8c5078fbf58bf6b47bbb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Lal?= Date: Wed, 17 Mar 2021 18:52:10 +0100 Subject: [PATCH 828/898] Add tests for neutron-openvswitch and neutron-gateway charms (#523) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add support for running NeutronOvsVsctlTest against any charm and not only charm-neutron-gateway. * Add new test case for charm-neutron-gateway and charm-neutron-openvswitch to verify correct handling of conflicting ext-port and data-port configurations. Signed-off-by: Przemysław Lal --- zaza/openstack/charm_tests/neutron/tests.py | 55 ++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index bebe3b6..70def89 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -592,6 +592,58 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): logging.info('Testing pause resume') +class NeutronBridgePortMappingTest(NeutronPluginApiSharedTests): + """Test correct handling of network-bridge-port mapping functionality.""" + + def test_600_conflict_data_ext_ports(self): + """Verify proper handling of conflict between data-port and ext-port. + + Configuring ext-port and data-port at the same time should make the + charm to enter "blocked" state. After unsetting ext-port charm should + be active again. + """ + if self.application_name not in ["neutron-gateway", + "neutron-openvswitch"]: + logging.debug("Skipping test, charm under test is not " + "neutron-gateway or neutron-openvswitch") + return + + current_data_port = zaza.model.get_application_config( + self.application_name).get("data-port").get("value", "") + current_ext_port = zaza.model.get_application_config( + self.application_name).get("ext-port").get("value", "") + logging.debug("Current data-port: '{}'".format(current_data_port)) + logging.debug("Current data-port: '{}'".format(current_ext_port)) + + test_config = zaza.charm_lifecycle.utils.get_charm_config( + fatal=False) + current_state = test_config.get("target_deploy_status", {}) + blocked_state = copy.deepcopy(current_state) + blocked_state[self.application_name] = { + "workload-status": "blocked", + "workload-status-message": + "ext-port set when data-port set: see config.yaml"} + + logging.info("Setting conflicting ext-port and data-port options") + zaza.model.set_application_config( + self.application_name, {"data-port": "br-phynet43:eth43", + "ext-port": "br-phynet43:eth43"}) + zaza.model.wait_for_application_states(states=blocked_state) + + # unset ext-port and wait for app state to return to active + logging.info("Unsetting conflicting ext-port option") + zaza.model.set_application_config( + self.application_name, {"ext-port": ""}) + zaza.model.wait_for_application_states(states=current_state) + + # restore original config + zaza.model.set_application_config( + self.application_name, {'data-port': current_data_port, + 'ext-port': current_ext_port}) + zaza.model.wait_for_application_states(states=current_state) + logging.info('OK') + + class NeutronOvsVsctlTest(NeutronPluginApiSharedTests): """Test 'ovs-vsctl'-related functionality on Neutron charms.""" @@ -610,7 +662,8 @@ class NeutronOvsVsctlTest(NeutronPluginApiSharedTests): unit.name, bridge_name ) + ' is marked as managed by us' ) - expected_external_id = 'charm-neutron-gateway=managed' + expected_external_id = 'charm-{}=managed'.format( + self.application_name) actual_external_id = zaza.model.run_on_unit( unit.entity_id, 'ovs-vsctl br-get-external-id {}'.format(bridge_name), From b428b6cf155184541ac7cf0ddd0a05d8418e34fc Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 19 Mar 2021 15:37:03 +0100 Subject: [PATCH 829/898] Remove obsolete HaclusterScalebackTest It has been superseded by HaclusterScaleBackAndForthTest and was used in the following charm gates: hacluster and mysql-router. --- zaza/openstack/charm_tests/hacluster/tests.py | 64 +------------------ 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 5f46565..9414145 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -76,70 +76,8 @@ class HaclusterTest(HaclusterBaseTest): self._toggle_maintenance_and_wait('false') -class HaclusterScalebackTest(HaclusterBaseTest): - """hacluster scaleback tests. - - Use for testing older releases where lp:1400481 wasn't fixed yet. - Superseded by HaclusterScaleBackAndForthTest. - """ - - @classmethod - def setUpClass(cls): - """Run class setup for running hacluster scaleback tests.""" - super(HaclusterScalebackTest, cls).setUpClass() - test_config = cls.test_config['tests_options']['hacluster'] - cls._principle_app_name = test_config['principle-app-name'] - cls._hacluster_charm_name = test_config['hacluster-charm-name'] - - def test_930_scaleback(self): - """Remove a unit and add a new one.""" - principle_units = sorted(zaza.model.get_status().applications[ - self._principle_app_name]['units'].keys()) - self.assertEqual(len(principle_units), 3) - doomed_principle_unit = principle_units[0] - other_principle_unit = principle_units[1] - doomed_hacluster_unit = juju_utils.get_subordinate_units( - [doomed_principle_unit], charm_name=self._hacluster_charm_name)[0] - other_hacluster_unit = juju_utils.get_subordinate_units( - [other_principle_unit], charm_name=self._hacluster_charm_name)[0] - - logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) - zaza.model.run_action( - doomed_hacluster_unit, - 'pause', - raise_on_failure=True) - logging.info('OK') - - logging.info('Removing {}'.format(doomed_principle_unit)) - zaza.model.destroy_unit( - self._principle_app_name, - doomed_principle_unit, - wait_disappear=True) - logging.info('OK') - - logging.info('Waiting for model to settle') - zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') - zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') - zaza.model.block_until_all_units_idle() - logging.info('OK') - - logging.info('Adding an hacluster unit') - zaza.model.add_unit(self._principle_app_name, wait_appear=True) - logging.info('OK') - - logging.info('Waiting for model to settle') - zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active') - # NOTE(lourot): the principle application sometimes remain blocked - # after scaling back up. - zaza.model.block_until_all_units_idle() - logging.debug('OK') - - class HaclusterScaleBackAndForthTest(HaclusterBaseTest): - """hacluster tests scaling back and forth. - - Supersedes HaclusterScalebackTest. - """ + """hacluster tests scaling back and forth.""" @classmethod def setUpClass(cls): From 1f47ef0bd769e3e7976130d9c81071d84bacd60f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 22 Mar 2021 18:43:13 +0000 Subject: [PATCH 830/898] Update package restart message afer ch change --- zaza/openstack/charm_tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 3aff816..ea134d3 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -893,7 +893,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): self.check_show_deferred_restarts_wlm(self.restart_package_service) self.check_show_deferred_restarts_action( self.restart_package_service, - 'Pkg Update') + 'Package update') logging.info("Running restart action to clear deferred restarts") self.check_clear_restarts() From 2a6517f6fd1fca128c414d29604706e54b3cfe29 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 23 Mar 2021 08:32:01 -0700 Subject: [PATCH 831/898] Add restart on change test (#527) Add the restart on change test for mysql-router. --- zaza/openstack/charm_tests/mysql/tests.py | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 15256e7..45dbf71 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -861,3 +861,38 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): assert action.data.get("results") is not None, ( "Remove instance action failed: No results: {}" .format(action.data)) + + +class MySQLRouterTests(test_utils.OpenStackBaseTest): + """MySQL Router Tests.""" + + @classmethod + def setUpClass(cls, application_name="keystone-mysql-router"): + """Run class setup for running mysql-router tests.""" + super().setUpClass(application_name=application_name) + cls.application = application_name + cls.services = ["mysqlrouter"] + # Config file affected by juju set config change + cls.conf_file = ( + "/var/lib/mysql/{}-mysql-router/mysqlrouter.conf" + .format(application_name)) + + def test_910_restart_on_config_change(self): + """Checking restart happens on config change. + + Change max connections and assert that change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + set_default = {"ttl": ".5"} + set_alternate = {"ttl": "7"} + + # Make config change, check for service restarts + logging.info("Setting TTL ...") + self.restart_on_changed( + self.conf_file, + set_default, + set_alternate, + {}, {}, + self.services) + logging.info("Passed restart on changed test.") From 085d5a1513afbf17828ae2bbf3e2c60bd8f0eb02 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 23 Mar 2021 15:57:21 +0000 Subject: [PATCH 832/898] Add RabbitMQDeferredRestartTest + fixes --- .../charm_tests/rabbitmq_server/tests.py | 42 +++++++++++++++++++ zaza/openstack/charm_tests/test_utils.py | 3 +- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index d4a3111..40e227c 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -428,3 +428,45 @@ class RmqTests(test_utils.OpenStackBaseTest): check_units(all_units) logging.info('OK') + + +class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass( + restart_config_file='/etc/rabbitmq/rabbitmq.config', + test_service='rabbitmq-server', + restart_package='rabbitmq-server', + restart_package_service='rabbitmq-server', + application_name='rabbitmq-server') + + def trigger_deferred_restart_via_charm(self): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be renderred in + applications config file. + + NOTE: The implementation assumes the charm has a `debug` option and + self.restart_config_file in an oslo config file where that + debug option is renderred. If that is not true the specaliasation + class should override this method. + """ + app_config = zaza.model.get_application_config(self.application_name) + logging.info("Triggering deferred restart via config change") + new_debug_value = str(int(app_config['connection-backlog'].get('value', 100) + 1)) + logging.info("Setting connection-backlog: {}".format(new_debug_value)) + zaza.model.set_application_config( + self.application_name, + {'connection-backlog': new_debug_value}) + logging.info("Waiting for connection-backlog to be {} in {}".format( + new_debug_value, + self.restart_config_file)) + zaza.model.block_until_file_matches_re( + self.application_name, + self.restart_config_file, + '{{backlog, {}}}'.format(new_debug_value)) + logging.info("Waiting for units to be idle") + zaza.model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index ea134d3..26a9917 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -770,7 +770,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): # Check workload status no longer shows deferred restarts. for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + assert 'Services queued' not in unit.workload_status_message def check_show_deferred_restarts_action(self, test_service, restart_reason): @@ -796,6 +796,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): "show-deferred-restarts action on {}").format( test_service, unit.entity_id)) + assert action.data['status'] == 'completed' for event in yaml.safe_load(action.data['results']['output']): if test_service in event and restart_reason in event: break From 0315ca680e8f84304f9f2aa0ddc0c25b65eca25a Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Thu, 11 Feb 2021 17:27:42 +0200 Subject: [PATCH 833/898] Add test for Glance with Ceph RGW backend This test shall be used for Glance testing bundles with Ceph deployed, and multi-backend enabled. It validates that images can be stored using the Ceph RGW (via the OpenStack Swift API). --- zaza/openstack/charm_tests/glance/tests.py | 50 ++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index 363032d..5d45338 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -92,3 +92,53 @@ class GlanceTest(test_utils.OpenStackBaseTest): they are started """ self.pause_resume(['glance-api']) + + +class GlanceCephRGWBackendTest(test_utils.OpenStackBaseTest): + """Encapsulate glance tests using the Ceph RGW backend. + + It validates the Ceph RGW backend in glance, which uses the Swift API. + """ + + @classmethod + def setUpClass(cls): + """Run class setup for running glance tests.""" + super(GlanceCephRGWBackendTest, cls).setUpClass() + + swift_session = openstack_utils.get_keystone_session_from_relation( + 'ceph-radosgw') + cls.swift = openstack_utils.get_swift_session_client( + swift_session) + cls.glance_client = openstack_utils.get_glance_session_client( + cls.keystone_session) + + def test_100_create_image(self): + """Create an image and do a simple validation of it. + + The OpenStack Swift API is used to do the validation, since the Ceph + Rados Gateway serves an API which is compatible with that. + """ + image_name = 'zaza-ceph-rgw-image' + openstack_utils.create_image( + glance=self.glance_client, + image_url=openstack_utils.find_cirros_image(arch='x86_64'), + image_name=image_name, + backend='swift') + headers, containers = self.swift.get_account() + self.assertEqual(len(containers), 1) + container_name = containers[0].get('name') + headers, objects = self.swift.get_container(container_name) + images = openstack_utils.get_images_by_name( + self.glance_client, + image_name) + self.assertEqual(len(images), 1) + image = images[0] + total_bytes = 0 + for ob in objects: + if '{}-'.format(image['id']) in ob['name']: + total_bytes = total_bytes + int(ob['bytes']) + logging.info( + 'Checking glance image size {} matches swift ' + 'image size {}'.format(image['size'], total_bytes)) + self.assertEqual(image['size'], total_bytes) + openstack_utils.delete_image(self.glance_client, image['id']) From 77cb2d0bd84f1a095ac034be6fa428a8242fe0d8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 24 Mar 2021 13:52:34 +0000 Subject: [PATCH 834/898] Fix doc string and lint --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 40e227c..14fadc4 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -449,14 +449,13 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): Set the charm debug option and wait for that change to be renderred in applications config file. - NOTE: The implementation assumes the charm has a `debug` option and - self.restart_config_file in an oslo config file where that - debug option is renderred. If that is not true the specaliasation - class should override this method. + This overrides the base class version as the rabbit charm does not + have a debug option and the config file is not in oslo config format. """ app_config = zaza.model.get_application_config(self.application_name) logging.info("Triggering deferred restart via config change") - new_debug_value = str(int(app_config['connection-backlog'].get('value', 100) + 1)) + new_debug_value = str( + int(app_config['connection-backlog'].get('value', 100) + 1)) logging.info("Setting connection-backlog: {}".format(new_debug_value)) zaza.model.set_application_config( self.application_name, From 8af30a5275ad6af342696bae642b03937f9a7fd1 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 25 Mar 2021 11:34:43 +0000 Subject: [PATCH 835/898] Pin lxml to 4.6.2 See https://github.com/openstack-charmers/zaza-openstack-tests/issues/530 Issue: Python 3.5. is deprecated/EOL Py35 is used on Xenial Xenial is used as the test platform to run tests. lxml is used by zaza-openstack-tests for the SAML assertions (because they are XML) lxml > 4.6.2 releases are after py3.5 has been deprecated/EOL. Therefore, no wheels are now built for lxml for py35. Thus there is a binary dependency to build lxml for testing --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index 78a41f0..adec5bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +# pin lxml < 4.6.3 for py35 as no wheels exist for 4.6.3 (deprecated platform) +# This is necessary for Xenial builders +# BUG: https://github.com/openstack-charmers/zaza-openstack-tests/issues/530 +lxml<4.6.3 aiounittest async_generator boto3 From e9f11da5df5c536eb953f4e4d6f99a68fd840079 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 25 Mar 2021 14:47:42 +0000 Subject: [PATCH 836/898] Fix typo and rename class for better understanding Fixes a typo (None -> not None) and renames the class from ObjectRetrier -> ObjectRetrierWraps to make it clearer that the class instantiation is to wrap and object with the retrier code rather than do retries at that moment. --- unit_tests/utilities/test_utilities.py | 26 +++++++++++---------- zaza/openstack/charm_tests/octavia/tests.py | 8 +++---- zaza/openstack/utilities/__init__.py | 21 +++++++++-------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/unit_tests/utilities/test_utilities.py b/unit_tests/utilities/test_utilities.py index a78600c..6af6089 100644 --- a/unit_tests/utilities/test_utilities.py +++ b/unit_tests/utilities/test_utilities.py @@ -31,7 +31,7 @@ class SomeException3(Exception): pass -class TestObjectRetrier(ut_utils.BaseTestCase): +class TestObjectRetrierWraps(ut_utils.BaseTestCase): def test_object_wrap(self): @@ -41,7 +41,7 @@ class TestObjectRetrier(ut_utils.BaseTestCase): return a + b a = A() - wrapped_a = utilities.ObjectRetrier(a) + wrapped_a = utilities.ObjectRetrierWraps(a) self.assertEqual(wrapped_a.func(3), 4) def test_object_multilevel_wrap(self): @@ -59,7 +59,7 @@ class TestObjectRetrier(ut_utils.BaseTestCase): return A() b = B() - wrapped_b = utilities.ObjectRetrier(b) + wrapped_b = utilities.ObjectRetrierWraps(b) self.assertEqual(wrapped_b.f2.f1(5, 6), 30) def test_object_wrap_number(self): @@ -75,7 +75,7 @@ class TestObjectRetrier(ut_utils.BaseTestCase): return a * b a = A() - wrapped_a = utilities.ObjectRetrier(a) + wrapped_a = utilities.ObjectRetrierWraps(a) self.assertEqual(wrapped_a.class_a, 5) self.assertEqual(wrapped_a.instance_a, 10) @@ -89,15 +89,15 @@ class TestObjectRetrier(ut_utils.BaseTestCase): a = A() # retry on a specific exception - wrapped_a = utilities.ObjectRetrier(a, num_retries=1, - retry_exceptions=[SomeException]) + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=1, retry_exceptions=[SomeException]) with self.assertRaises(SomeException): wrapped_a.func() mock_sleep.assert_called_once_with(5) # also retry on any exception if none specified - wrapped_a = utilities.ObjectRetrier(a, num_retries=1) + wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=1) mock_sleep.reset_mock() with self.assertRaises(SomeException): wrapped_a.func() @@ -105,8 +105,8 @@ class TestObjectRetrier(ut_utils.BaseTestCase): mock_sleep.assert_called_once_with(5) # no retry if exception isn't listed. - wrapped_a = utilities.ObjectRetrier(a, num_retries=1, - retry_exceptions=[SomeException2]) + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=1, retry_exceptions=[SomeException2]) mock_sleep.reset_mock() with self.assertRaises(SomeException): wrapped_a.func() @@ -123,7 +123,8 @@ class TestObjectRetrier(ut_utils.BaseTestCase): a = A() mock_log = mock.Mock() - wrapped_a = utilities.ObjectRetrier(a, num_retries=1, log=mock_log) + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=1, log=mock_log) with self.assertRaises(SomeException): wrapped_a.func() @@ -140,7 +141,7 @@ class TestObjectRetrier(ut_utils.BaseTestCase): raise SomeException() a = A() - wrapped_a = utilities.ObjectRetrier(a, num_retries=3, backoff=2) + wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=3, backoff=2) with self.assertRaises(SomeException): wrapped_a.func() # Note third call hits maximum wait time of 15. @@ -157,7 +158,8 @@ class TestObjectRetrier(ut_utils.BaseTestCase): raise SomeException() a = A() - wrapped_a = utilities.ObjectRetrier(a, num_retries=3, total_wait=9) + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=3, total_wait=9) with self.assertRaises(SomeException): wrapped_a.func() # Note only two calls, as total wait is 9, so a 3rd retry would exceed diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 9a71b49..535d7aa 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -25,7 +25,7 @@ import osc_lib.exceptions import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils -from zaza.openstack.utilities import ObjectRetrier +from zaza.openstack.utilities import ObjectRetrierWraps class CharmOperationTest(test_utils.OpenStackBaseTest): @@ -65,11 +65,11 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run class setup for running LBaaSv2 service tests.""" super(LBAASv2Test, cls).setUpClass() - cls.keystone_client = ObjectRetrier( + cls.keystone_client = ObjectRetrierWraps( openstack_utils.get_keystone_session_client(cls.keystone_session)) - cls.neutron_client = ObjectRetrier( + cls.neutron_client = ObjectRetrierWraps( openstack_utils.get_neutron_session_client(cls.keystone_session)) - cls.octavia_client = ObjectRetrier( + cls.octavia_client = ObjectRetrierWraps( openstack_utils.get_octavia_session_client(cls.keystone_session)) cls.RESOURCE_PREFIX = 'zaza-octavia' diff --git a/zaza/openstack/utilities/__init__.py b/zaza/openstack/utilities/__init__.py index 6db3ce3..5798f26 100644 --- a/zaza/openstack/utilities/__init__.py +++ b/zaza/openstack/utilities/__init__.py @@ -20,7 +20,7 @@ import time from keystoneauth1.exceptions.connection import ConnectFailure -class ObjectRetrier(object): +class ObjectRetrierWraps(object): """An automatic retrier for an object. This is designed to be used with an instance of an object. Basically, it @@ -35,7 +35,7 @@ class ObjectRetrier(object): # get a client that does 3 retries, waits 5 seconds between retries and # retries on any error. - some_client = ObjectRetrier(get_some_client) + some_client = ObjectRetrierWraps(get_some_client) # this gets retried up to 3 times. things = some_client.list_things() @@ -96,7 +96,7 @@ class ObjectRetrier(object): # will fail with an attribute error. attr = getattr(self.__obj, name) if callable(attr) or hasattr(attr, "__getattr__"): - return ObjectRetrier(attr, **self.__kwargs) + return ObjectRetrierWraps(attr, **self.__kwargs) else: return attr # TODO(ajkavanagh): Note detecting a property is a bit trickier. we @@ -122,8 +122,8 @@ class ObjectRetrier(object): try: return obj(*args, **kwargs) except Exception as e: - # if retry_exceptions is None, or the type of the exception is - # not in the list of retries, then raise an exception + # if retry_exceptions is not None, or the type of the exception + # is not in the list of retries, then raise an exception # immediately. This means that if retry_exceptions is None, # then the method is always retried. if (retry_exceptions is not None and @@ -148,19 +148,20 @@ class ObjectRetrier(object): def retry_on_connect_failure(client, **kwargs): """Retry an object that eventually gets resolved to a call. - Specifically, this uses ObjectRetrier but only against the + Specifically, this uses ObjectRetrierWraps but only against the keystoneauth1.exceptions.connection.ConnectFailure exeception. :params client: the object that may throw and exception when called. :type client: Any - :params **kwargs: the arguments supplied to the ObjectRetrier init method + :params **kwargs: the arguments supplied to the ObjectRetrierWraps init + method :type **kwargs: Dict[Any] - :returns: client wrapped in an ObjectRetrier instance - :rtype: ObjectRetrier[client] + :returns: client wrapped in an ObjectRetrierWraps instance + :rtype: ObjectRetrierWraps[client] """ kwcopy = kwargs.copy() if 'retry_exceptions' not in kwcopy: kwcopy['retry_exceptions'] = [] if ConnectFailure not in kwcopy['retry_exceptions']: kwcopy['retry_exceptions'].append(ConnectFailure) - return ObjectRetrier(client, **kwcopy) + return ObjectRetrierWraps(client, **kwcopy) From e5d2acc88c399fb549ac5d3971c8bff98775c534 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 26 Mar 2021 16:58:08 +0000 Subject: [PATCH 837/898] Fix test_vault_reload race condition The test_vault_reload test was checking for running config without giving the config-change time to complete. Simply adding an idle wait resolves the issue. Closes Issue: #528 --- zaza/openstack/charm_tests/vault/tests.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index c5211e0..29719d5 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -273,23 +273,33 @@ class VaultTest(BaseVaultTest): running_config = vault_utils.get_running_config(lead_client) value_to_set = not running_config['data']['disable_mlock'] + logging.info("Setting disable-mlock to {}".format(str(value_to_set))) zaza.model.set_application_config( 'vault', {'disable-mlock': str(value_to_set)}) - logging.info("Testing reload") + logging.info("Waiting for modle to be idle ...") + zaza.model.block_until_all_units_idle(model_name=self.model_name) + + logging.info("Testing action reload on {}".format(lead_client)) zaza.model.run_action( juju_utils.get_unit_name_from_ip_address( lead_client.addr, 'vault'), 'reload', model_name=self.model_name) + logging.info("Getting new value ...") new_value = vault_utils.get_running_config(lead_client)[ 'data']['disable_mlock'] - logging.info(new_value) + + logging.info( + "Asserting new value {} is equal to set value {}" + .format(new_value, value_to_set)) self.assertEqual( value_to_set, new_value) + + logging.info("Asserting not sealed") self.assertFalse(lead_client.hvac_client.seal_status['sealed']) def test_vault_restart(self): From 88fe0af9cb2f39ea804a958ccf5c0e1a09ccd2ae Mon Sep 17 00:00:00 2001 From: mkalcok <69471063+mkalcok@users.noreply.github.com> Date: Sat, 27 Mar 2021 09:27:02 +0100 Subject: [PATCH 838/898] Functional tests for nova-compute 'cloud' actions (#470) * Functional test for 'enable' and 'disable' actions on nova-compute * lint fixes * Put tests of new functionality into separate class * Added tests for actions `remove-from-cloud` and `register-to-cloud` * Set longer timeout for action verification * Add logging to help with CI debugging * Raise error if action fails * Cleanup running VMs before test starts * Add cleanup to tests that create VMs --- zaza/openstack/charm_tests/nova/tests.py | 143 +++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index edad683..f457ea9 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -19,6 +19,8 @@ import json import logging import unittest +from configparser import ConfigParser +from time import sleep import zaza.model import zaza.openstack.charm_tests.glance.setup as glance_setup @@ -47,6 +49,10 @@ class CirrosGuestCreateTest(test_utils.OpenStackBaseTest): self.launch_guest( 'cirros', instance_key=glance_setup.CIRROS_IMAGE_NAME) + def tearDown(self): + """Cleanup of VM guests.""" + self.resource_cleanup() + class LTSGuestCreateTest(test_utils.OpenStackBaseTest): """Tests to launch a LTS image.""" @@ -57,6 +63,10 @@ class LTSGuestCreateTest(test_utils.OpenStackBaseTest): self.launch_guest( 'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME) + def tearDown(self): + """Cleanup of VM guests.""" + self.resource_cleanup() + class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): """Tests to launch a LTS image.""" @@ -68,6 +78,139 @@ class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): 'volume-backed-ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME, use_boot_volume=True) + def tearDown(self): + """Cleanup of VM guests.""" + self.resource_cleanup() + + +class CloudActions(test_utils.OpenStackBaseTest): + """Test actions from actions/cloud.py.""" + + def fetch_nova_service_hostname(self, unit_name): + """ + Fetch hostname used to register with nova-cloud-controller. + + When nova-compute registers with nova-cloud-controller it uses either + config variable from '/etc/nova/nova.conf` or host's hostname to + identify itself. We need to fetch this value directly from the unit, + otherwise it's not possible to correlate entries from + `nova service-list` with nova-compute units. + + :param unit_name: nova-compute unit name. + :return: hostname used when registering to cloud-controller + """ + nova_cfg = ConfigParser() + + result = zaza.model.run_on_unit(unit_name, + 'cat /etc/nova/nova.conf') + nova_cfg.read_string(result['Stdout']) + + try: + nova_service_name = nova_cfg['DEFAULT']['host'] + except KeyError: + # Fallback to hostname if 'host' variable is not present in the + # config + result = zaza.model.run_on_unit(unit_name, 'hostname') + nova_service_name = result['Stdout'].rstrip('\n') + + if not nova_service_name: + self.fail("Failed to fetch nova service name from" + " nova-compute unit.") + return nova_service_name + + def test_940_enable_disable_actions(self): + """Test disable/enable actions on nova-compute units.""" + nova_units = zaza.model.get_units('nova-compute', + model_name=self.model_name) + + # Check that nova-compute services are enabled before testing + for service in self.nova_client.services.list(binary='nova-compute'): + self.assertEqual(service.status, 'enabled') + + # Run 'disable' action on units + zaza.model.run_action_on_units([unit.name for unit in nova_units], + 'disable') + + # Check action results via nova API + for service in self.nova_client.services.list(binary='nova-compute'): + self.assertEqual(service.status, 'disabled') + + # Run 'enable' action on units + zaza.model.run_action_on_units([unit.name for unit in nova_units], + 'enable') + + # Check action results via nova API + for service in self.nova_client.services.list(binary='nova-compute'): + self.assertEqual(service.status, 'enabled') + + def test_950_remove_from_cloud_actions(self): + """Test actions remove-from-cloud and register-to-cloud. + + Note (martin-kalcok): This test requires that nova-compute unit is not + running any VMs. If there are any leftover VMs from previous tests, + action `remove-from-cloud` will fail. + """ + all_units = zaza.model.get_units('nova-compute', + model_name=self.model_name) + + unit_to_remove = all_units[0] + + service_name = self.fetch_nova_service_hostname(unit_to_remove.name) + + registered_nova_services = self.nova_client.services.list( + host=service_name, binary='nova-compute') + + service_count = len(registered_nova_services) + if service_count < 1: + self.fail("Unit '{}' has no nova-compute services registered in" + " nova-cloud-controller".format(unit_to_remove.name)) + elif service_count > 1: + self.fail("Unexpected number of nova-compute services registered" + " in nova-cloud controller. Expecting: 1, found: " + "{}".format(service_count)) + + # run action remove-from-cloud and wait for the results in + # nova-cloud-controller + zaza.model.run_action_on_units([unit_to_remove.name], + 'remove-from-cloud', + raise_on_failure=True) + + # Wait for nova-compute service to be removed from the + # nova-cloud-controller + sleep_timeout = 1 # don't waste 10 seconds on the first run + + for _ in range(31): + sleep(sleep_timeout) + service_list = self.nova_client.services.list( + host=service_name, binary='nova-compute') + if len(service_list) == 0: + break + sleep_timeout = 10 + else: + self.fail("nova-compute service was not unregistered from the " + "nova-cloud-controller as expected.") + + # run action register-to-cloud to revert previous action + # and wait for the results in nova-cloud-controller + zaza.model.run_action_on_units([unit_to_remove.name], + 'register-to-cloud', + raise_on_failure=True) + + # Wait for nova-compute service to be registered to the + # nova-cloud-controller + sleep_timeout = 1 # don't waste 10 seconds on the first run + + for _ in range(31): + sleep(sleep_timeout) + service_list = self.nova_client.services.list( + host=service_name, binary='nova-compute') + if len(service_list) == 1: + break + sleep_timeout = 10 + else: + self.fail("nova-compute service was not re-registered to the " + "nova-cloud-controller as expected.") + class NovaCompute(test_utils.OpenStackBaseTest): """Run nova-compute specific tests.""" From 0fdd876213079cfd1adfdea2d29ee23eb87bc2a0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 29 Mar 2021 10:41:46 +0000 Subject: [PATCH 839/898] Update inline with recent charm changes --- .../charm_tests/rabbitmq_server/tests.py | 13 +- zaza/openstack/charm_tests/test_utils.py | 129 ++++++++++++------ 2 files changed, 93 insertions(+), 49 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 14fadc4..13d0e52 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -443,7 +443,7 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): restart_package_service='rabbitmq-server', application_name='rabbitmq-server') - def trigger_deferred_restart_via_charm(self): + def trigger_deferred_hook_via_charm(self): """Set charm config option which requires a service start. Set the charm debug option and wait for that change to be renderred in @@ -460,12 +460,9 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): zaza.model.set_application_config( self.application_name, {'connection-backlog': new_debug_value}) - logging.info("Waiting for connection-backlog to be {} in {}".format( - new_debug_value, - self.restart_config_file)) - zaza.model.block_until_file_matches_re( - self.application_name, - self.restart_config_file, - '{{backlog, {}}}'.format(new_debug_value)) logging.info("Waiting for units to be idle") + test_unit = zaza.model.get_units(self.application_name)[0] + zaza.model.block_until_unit_wl_message_match( + test_unit.entity_id, + status_pattern='.*config-changed.*') zaza.model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 26a9917..94ac235 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -723,7 +723,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): NOTE: The test has been broken into various class methods which may require specialisation if the charm being tested is not a standard OpenStack - charm e.g. `trigger_deferred_restart_via_charm` if the charm is not + charm e.g. `trigger_deferred_hook_via_charm` if the charm is not an oslo config or does not have a debug option. """ @@ -770,10 +770,36 @@ class BaseDeferredRestartTest(OpenStackBaseTest): # Check workload status no longer shows deferred restarts. for unit in model.get_units(self.application_name): - assert 'Services queued' not in unit.workload_status_message + assert unit.workload_status_message == 'Unit is ready' - def check_show_deferred_restarts_action(self, test_service, - restart_reason): + def check_clear_hooks(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in model.get_units(self.application_name): + model.run_action( + unit.entity_id, + 'run-deferred-hooks') + + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + assert unit.workload_status_message == 'Unit is ready' + + def run_show_deferred_events_action(self): + """Run show-deferred-events and return results. + + :returns: Data from action run + :rtype: Dict + """ + unit = model.get_units(self.application_name)[0] + action = model.run_action(unit.entity_id, 'show-deferred-events') + return yaml.safe_load(action.data['results']['output']) + + def check_show_deferred_events_action_restart(self, test_service, + restart_reason): """Check the output from the action to list deferred restarts. Run the action to list any deferred restarts and check it has entry for @@ -787,24 +813,44 @@ class BaseDeferredRestartTest(OpenStackBaseTest): :type restart_reason: str """ # Ensure that the deferred restart and cause are listed via action - for unit in model.get_units(self.application_name): - action = model.run_action( - unit.entity_id, - 'show-deferred-restarts') - logging.info( - ("Checking {} is marked as needing restart in " - "show-deferred-restarts action on {}").format( - test_service, - unit.entity_id)) - assert action.data['status'] == 'completed' - for event in yaml.safe_load(action.data['results']['output']): - if test_service in event and restart_reason in event: - break - else: - msg = 'No entry for restart of {} for reason {} found'.format( - test_service, - restart_reason) - raise Exception(msg) + logging.info( + ("Checking {} is marked as needing restart in " + "show-deferred-events action").format( + test_service)) + for event in self.run_show_deferred_events_action()['restarts']: + logging.info("{} in {} and {} in {}".format( + test_service, + event, + restart_reason, + event)) + if test_service in event and restart_reason in event: + break + else: + msg = 'No entry for restart of {} for reason {} found'.format( + test_service, + restart_reason) + raise Exception(msg) + + def check_show_deferred_events_action_hook(self, hook): + """Check the output from the action to list deferred eveents. + + Run the action to list any deferred events and check it has entry for + the given hook. + + :param hook: Hook name + :type hook: str + """ + # Ensure that the deferred restart and cause are listed via action + logging.info( + ("Checking {} is marked as skipped in " + "show-deferred-events action").format(hook)) + for event in self.run_show_deferred_events_action()['hooks']: + logging.info("{} in {}".format(hook, event)) + if hook in event: + break + else: + msg = '{} not found in {}'.format(hook, event) + raise Exception(msg) def check_show_deferred_restarts_wlm(self, test_service): """Check the workload status message lists deferred restart. @@ -820,7 +866,16 @@ class BaseDeferredRestartTest(OpenStackBaseTest): "message of {}".format(test_service, unit.entity_id))) assert test_service in unit.workload_status_message - def trigger_deferred_restart_via_charm(self): + def check_deferred_hook_wlm(self): + """Check the workload status message lists deferred event.""" + # Ensure that the deferred restarts are visible in Juju status + for unit in model.get_units(self.application_name): + logging.info( + ("Checking {} is marked as having deferred hook in workload " + "message".format(unit.entity_id))) + assert 'config-changed' in unit.workload_status_message + + def trigger_deferred_hook_via_charm(self): """Set charm config option which requires a service start. Set the charm debug option and wait for that change to be renderred in @@ -838,17 +893,11 @@ class BaseDeferredRestartTest(OpenStackBaseTest): model.set_application_config( self.application_name, {'debug': new_debug_value}) - expected_contents = { - 'DEFAULT': { - 'debug': [new_debug_value]}} - logging.info("Waiting for debug to be {} in {}".format( - new_debug_value, - self.restart_config_file)) - model.block_until_oslo_config_entries_match( - self.application_name, - self.restart_config_file, - expected_contents) logging.info("Waiting for units to be idle") + test_unit = model.get_units(self.application_name)[0] + model.block_until_unit_wl_message_match( + test_unit.entity_id, + status_pattern='.*config-changed.*') model.block_until_all_units_idle() def trigger_deferred_restart_via_package(self): @@ -870,16 +919,14 @@ class BaseDeferredRestartTest(OpenStackBaseTest): NOTE: If this test is not relevant for the target charm then override this method and raise unittest.SkipTest """ - self.trigger_deferred_restart_via_charm() + self.trigger_deferred_hook_via_charm() - self.check_show_deferred_restarts_wlm(self.test_service) - self.check_show_deferred_restarts_action( - self.test_service, - self.restart_config_file) + self.check_deferred_hook_wlm() + self.check_show_deferred_events_action_hook('config-changed') # Rerunning to flip config option back to previous value. - self.trigger_deferred_restart_via_charm() + self.trigger_deferred_hook_via_charm() logging.info("Running restart action to clear deferred restarts") - self.check_clear_restarts() + self.check_clear_hooks() def run_package_change_test(self): """Trigger a deferred restart by updating a package. @@ -892,7 +939,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): self.trigger_deferred_restart_via_package() self.check_show_deferred_restarts_wlm(self.restart_package_service) - self.check_show_deferred_restarts_action( + self.check_show_deferred_events_action_restart( self.restart_package_service, 'Package update') logging.info("Running restart action to clear deferred restarts") From f78105a6460f9338d756bab160ddf13b295627a0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 29 Mar 2021 14:46:04 +0000 Subject: [PATCH 840/898] Allow status message to be overridden --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 7 +++++++ zaza/openstack/charm_tests/test_utils.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 13d0e52..c3f03fb 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -443,6 +443,13 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): restart_package_service='rabbitmq-server', application_name='rabbitmq-server') + def check_status_message_is_clear(self): + """Check each units status message show no defeerred events.""" + for unit in zaza.model.get_units(self.application_name): + assert unit.workload_status_message in [ + 'Unit is ready', + 'Unit is ready and clustered'] + def trigger_deferred_hook_via_charm(self): """Set charm config option which requires a service start. diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 94ac235..dec0cd0 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -755,6 +755,12 @@ class BaseDeferredRestartTest(OpenStackBaseTest): super(BaseDeferredRestartTest, cls).setUpClass( application_name=cls.application_name) + def check_status_message_is_clear(self): + """Check each units status message show no defeerred events.""" + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + assert unit.workload_status_message == 'Unit is ready' + def check_clear_restarts(self): """Clear and deferred restarts and check status. @@ -769,8 +775,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): action_params={'deferred': True}) # Check workload status no longer shows deferred restarts. - for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + self.check_status_message_is_clear() def check_clear_hooks(self): """Clear and deferred restarts and check status. @@ -785,8 +790,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): 'run-deferred-hooks') # Check workload status no longer shows deferred restarts. - for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + self.check_status_message_is_clear() def run_show_deferred_events_action(self): """Run show-deferred-events and return results. From 50c781fb53455badbc8fa724ba85bf641882be0b Mon Sep 17 00:00:00 2001 From: mkalcok <69471063+mkalcok@users.noreply.github.com> Date: Tue, 30 Mar 2021 08:44:46 +0200 Subject: [PATCH 841/898] Remove repetitive code. (#534) --- zaza/openstack/charm_tests/nova/tests.py | 42 +++++++++++------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index f457ea9..f2c68e0 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -150,6 +150,24 @@ class CloudActions(test_utils.OpenStackBaseTest): running any VMs. If there are any leftover VMs from previous tests, action `remove-from-cloud` will fail. """ + def wait_for_nova_compute_count(expected_count): + """Wait for expected number of nova compute services to be present. + + Returns True or False based on whether the expected number of nova + compute services was reached within the timeout. Checks are + performed every 10 second in the span of maximum 5 minutes. + """ + sleep_timeout = 1 # don't waste 10 seconds on the first run + + for _ in range(31): + sleep(sleep_timeout) + service_list = self.nova_client.services.list( + host=service_name, binary='nova-compute') + if len(service_list) == expected_count: + return True + sleep_timeout = 10 + return False + all_units = zaza.model.get_units('nova-compute', model_name=self.model_name) @@ -177,16 +195,7 @@ class CloudActions(test_utils.OpenStackBaseTest): # Wait for nova-compute service to be removed from the # nova-cloud-controller - sleep_timeout = 1 # don't waste 10 seconds on the first run - - for _ in range(31): - sleep(sleep_timeout) - service_list = self.nova_client.services.list( - host=service_name, binary='nova-compute') - if len(service_list) == 0: - break - sleep_timeout = 10 - else: + if not wait_for_nova_compute_count(0): self.fail("nova-compute service was not unregistered from the " "nova-cloud-controller as expected.") @@ -196,18 +205,7 @@ class CloudActions(test_utils.OpenStackBaseTest): 'register-to-cloud', raise_on_failure=True) - # Wait for nova-compute service to be registered to the - # nova-cloud-controller - sleep_timeout = 1 # don't waste 10 seconds on the first run - - for _ in range(31): - sleep(sleep_timeout) - service_list = self.nova_client.services.list( - host=service_name, binary='nova-compute') - if len(service_list) == 1: - break - sleep_timeout = 10 - else: + if not wait_for_nova_compute_count(1): self.fail("nova-compute service was not re-registered to the " "nova-cloud-controller as expected.") From fe8b78b7c6640ec40b1566aa8fd2d512c2da7476 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 14:08:54 -0700 Subject: [PATCH 842/898] Functional testing for mysql-innodb-cluster This is a change with functional test for mysql-innodb-cluster implementing `force-quorum-using-partition-of` feature. It is addressing bug: https://bugs.launchpad.net/charm-mysql-innodb-cluster/+bug/1917332 --- zaza/openstack/charm_tests/mysql/tests.py | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 45dbf71..7af9c75 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -617,6 +617,77 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) + def test_110_force_quorum_using_partition_of(self): + """Force quorum using partition of given address. + + After outage, cluster can end up without quorum. Force it. + """ + _machines = sorted( + juju_utils.get_machine_uuids_for_application(self.application)) + # Wait until update-status hooks have completed + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # Block all traffic across mysql instances: 0<-1, 1<-2 and 2<-0 + mysql_units = [unit for unit in zaza.model.get_units(self.application)] + no_of_units = len(mysql_units) + for index, unit in enumerate(mysql_units): + next_unit = mysql_units[(index+1)%no_of_units] + ip_address = next_unit.public_address + cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) + zaza.model.async_run_on_unit(unit, cmd) + + logging.info( + "Wait till all {} units are in state 'blocked' ..." + .format(self.application)) + for unit in zaza.model.get_units(self.application): + zaza.model.block_until_unit_wl_status( + unit.entity_id, + 'blocked', + negate_match=True) + + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + + # Unblock all traffic across mysql instances + for unit in zaza.model.get_units(self.application): + cmd = "sudo iptables -F" + zaza.model.async_run_on_unit(unit, cmd) + + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + + logging.info("Execute force-quorum-using-partition-of action ...") + + # Select "quorum leader" unit + leader_unit, other_units = mysql_units[0], mysq_units[1:] + action = zaza.model.run_action( + leader_unit.entity_id, + "force-quorum-using-partition-of", + action_params={"address": leader_unit.public_ip}) + + assert "Success" in action.data["results"]["outcome"], ( + "Force quorum using partition of action failed: {}" + .format(action.data)) + + # Rejoin other units to cluster + for unit in other_units: + zaza.model.run_action( + leader_unit.entity_id, + "rejoin-instance", + action_params={"address": unit.public_ip}) + + for unit in zaza.model.get_units(self.application): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" From a891ff28d5131c0e6bc02f38b3336bfe4eb61a0d Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 16:38:21 -0700 Subject: [PATCH 843/898] Added 'i-really-mean-it' tick to force-quorum --- zaza/openstack/charm_tests/mysql/tests.py | 29 +++++++---------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 7af9c75..22a6b5c 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -618,7 +618,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): states=test_config.get("target_deploy_status", {})) def test_110_force_quorum_using_partition_of(self): - """Force quorum using partition of given address. + """Force quorum using partition of instance with given address. After outage, cluster can end up without quorum. Force it. """ @@ -653,18 +653,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): self.resolve_update_status_errors() zaza.model.block_until_all_units_idle() - # Unblock all traffic across mysql instances - for unit in zaza.model.get_units(self.application): - cmd = "sudo iptables -F" - zaza.model.async_run_on_unit(unit, cmd) - - logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() - logging.info("Execute force-quorum-using-partition-of action ...") # Select "quorum leader" unit @@ -672,18 +660,19 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): action = zaza.model.run_action( leader_unit.entity_id, "force-quorum-using-partition-of", - action_params={"address": leader_unit.public_ip}) + action_params={ + "address": leader_unit.public_ip, + 'i-really-mean-it': True + }) assert "Success" in action.data["results"]["outcome"], ( "Force quorum using partition of action failed: {}" .format(action.data)) - # Rejoin other units to cluster - for unit in other_units: - zaza.model.run_action( - leader_unit.entity_id, - "rejoin-instance", - action_params={"address": unit.public_ip}) + # Unblock all traffic across mysql instances + for unit in zaza.model.get_units(self.application): + cmd = "sudo iptables -F" + zaza.model.async_run_on_unit(unit, cmd) for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") From e8157f0df8fab193de6ac70e2ddcd5e21145206f Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 20:44:23 -0700 Subject: [PATCH 844/898] Addressed flake8 violations --- zaza/openstack/charm_tests/mysql/tests.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 22a6b5c..4e9da67 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -622,9 +622,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): After outage, cluster can end up without quorum. Force it. """ - _machines = sorted( - juju_utils.get_machine_uuids_for_application(self.application)) - # Wait until update-status hooks have completed logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() @@ -632,7 +629,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): mysql_units = [unit for unit in zaza.model.get_units(self.application)] no_of_units = len(mysql_units) for index, unit in enumerate(mysql_units): - next_unit = mysql_units[(index+1)%no_of_units] + next_unit = mysql_units[(index+1) % no_of_units] ip_address = next_unit.public_address cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) zaza.model.async_run_on_unit(unit, cmd) @@ -656,7 +653,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): logging.info("Execute force-quorum-using-partition-of action ...") # Select "quorum leader" unit - leader_unit, other_units = mysql_units[0], mysq_units[1:] + leader_unit = mysql_units[0] action = zaza.model.run_action( leader_unit.entity_id, "force-quorum-using-partition-of", From 66fd299f16eaf5f57f6db855370faa146b59daa2 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 21:30:13 -0700 Subject: [PATCH 845/898] Fixed typo --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 4e9da67..6923bc0 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -658,7 +658,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): leader_unit.entity_id, "force-quorum-using-partition-of", action_params={ - "address": leader_unit.public_ip, + "address": leader_unit.public_address, 'i-really-mean-it': True }) From 51a53d4e864ed558f4ffafc033b683d8ca5f56a7 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 07:12:06 -0700 Subject: [PATCH 846/898] Verify at the end that cluster is in operable state --- zaza/openstack/charm_tests/mysql/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 6923bc0..6e0de25 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -671,8 +671,12 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): cmd = "sudo iptables -F" zaza.model.async_run_on_unit(unit, cmd) + logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {})) class MySQL8MigrationTests(MySQLBaseTest): From 8b47c8a37f4e9a94f0db1d6755988ccdb507f8ea Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 07:40:00 -0700 Subject: [PATCH 847/898] Change the way how I'm checking state --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 6e0de25..2226893 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -662,7 +662,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): 'i-really-mean-it': True }) - assert "Success" in action.data["results"]["outcome"], ( + assert action.data.get("results")["outcome"] is not Noe, ( "Force quorum using partition of action failed: {}" .format(action.data)) From 795272269f3f1548a9692404ea7ac5760276dd97 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 07:52:18 -0700 Subject: [PATCH 848/898] Fixed typo --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 2226893..311158a 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -662,7 +662,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): 'i-really-mean-it': True }) - assert action.data.get("results")["outcome"] is not Noe, ( + assert action.data.get("results") is not None, ( "Force quorum using partition of action failed: {}" .format(action.data)) From 9a5e334b444f1ab626aa8225f1884f6185bf7822 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 11:05:52 -0700 Subject: [PATCH 849/898] Addressed review comments --- zaza/openstack/charm_tests/mysql/tests.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 311158a..546fef3 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -644,11 +644,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): negate_match=True) logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() + zaza.model.block_until_all_units_idle() logging.info("Execute force-quorum-using-partition-of action ...") @@ -665,6 +661,9 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): assert action.data.get("results") is not None, ( "Force quorum using partition of action failed: {}" .format(action.data)) + logging.debug( + "Results from running 'force-quorum' command ...\n{}".format( + action.data) # Unblock all traffic across mysql instances for unit in zaza.model.get_units(self.application): @@ -678,6 +677,13 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" From 0858f016cdf42d9a076e9f47fe9e8d720801c261 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 11:29:44 -0700 Subject: [PATCH 850/898] Fixed TokenError --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 546fef3..219d25b 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -663,7 +663,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): .format(action.data)) logging.debug( "Results from running 'force-quorum' command ...\n{}".format( - action.data) + action.data)) # Unblock all traffic across mysql instances for unit in zaza.model.get_units(self.application): From 4df0cbee8ce6a70926d1fa953c0859409000437c Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 11:40:07 -0700 Subject: [PATCH 851/898] Moved block_until_idle immediately after force-quorum cmd --- zaza/openstack/charm_tests/mysql/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 219d25b..5cc563d 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -665,6 +665,13 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): "Results from running 'force-quorum' command ...\n{}".format( action.data)) + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + # Unblock all traffic across mysql instances for unit in zaza.model.get_units(self.application): cmd = "sudo iptables -F" @@ -677,13 +684,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) - logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() - class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" From a6cd0ea62a0124c45039e1a151dbbd4dfdecf8db Mon Sep 17 00:00:00 2001 From: Billy Olsen Date: Thu, 8 Apr 2021 13:51:33 -0700 Subject: [PATCH 852/898] Fix Ceph service action tests The ServiceTest.test_start_specific under the ceph osd tests stops all services using ceph-osd.target. This will stop all the OSDs, but the restarting of any single OSD will restart all the OSDs. This behavior causes the charm test to fail. Unfortunately, this wasn't caught in the initial review because the test is skipped when there are fewer the 2 OSDs on a unit and the final test run skipped the test due to this condition. Fixes #542 Signed-off-by: Billy Olsen --- zaza/openstack/charm_tests/ceph/osd/tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 63fb128..e598a5d 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -258,8 +258,11 @@ class ServiceTest(unittest.TestCase): to_start = should_stop.pop() should_stop = [service.name for service in should_stop] + # Note: can't stop ceph-osd.target as restarting a single OSD will + # cause this to start all of the OSDs when a single one starts. logging.info("Stopping all running ceph-osd services") - service_stop_cmd = 'systemctl stop ceph-osd.target' + service_stop_cmd = '; '.join(['systemctl stop {}'.format(service) + for service in service_names]) zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd) wait_for_service(unit_name=self.TESTED_UNIT, From 01544d22ed8af9a779b2aeb697dbae08044dc682 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 9 Apr 2021 10:48:52 +0100 Subject: [PATCH 853/898] Deferred restart test fixes (#540) * Add ovn-chassis test * Yet another refactor to reduce the amount that implementations need to override. * Add ovn dedicated chassis support * Fix race with checking wlm --- zaza/openstack/charm_tests/neutron/tests.py | 38 ++- zaza/openstack/charm_tests/ovn/tests.py | 58 +++++ .../charm_tests/rabbitmq_server/tests.py | 68 ++--- zaza/openstack/charm_tests/test_utils.py | 235 ++++++++++++------ 4 files changed, 291 insertions(+), 108 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 8a1f959..def9cda 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -1013,9 +1013,35 @@ class NeutronOVSDeferredRestartTest(test_utils.BaseDeferredRestartTest): @classmethod def setUpClass(cls): """Run setup for deferred restart tests.""" - super(NeutronOVSDeferredRestartTest, cls).setUpClass( - restart_config_file='/etc/neutron/neutron.conf', - test_service='neutron-openvswitch-agent', - restart_package='openvswitch-switch', - restart_package_service='openvswitch-switch', - application_name='neutron-openvswitch') + super().setUpClass(application_name='neutron-openvswitch') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + +class NeutronGatewayDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='neutron-gateway') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which requires a restart + self.run_charm_change_restart_test( + 'neutron-l3-agent', + '/etc/neutron/neutron.conf') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 9bea951..6b43abe 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -431,3 +431,61 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): except KeyError: # One of the applications is not in the model, which is fine pass + + +class OVNChassisDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='ovn-chassis') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('configure_ovs') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. + + :returns: Config key and new value + :rtype: (str, bool) + """ + app_config = zaza.model.get_application_config(self.application_name) + return 'enable-sriov', str(not app_config['enable-sriov']['value']) + + +class OVNDedicatedChassisDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='ovn-dedicated-chassis') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('configure_ovs') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. + + :returns: Config key and new value + :rtype: (str, bool) + """ + app_config = zaza.model.get_application_config(self.application_name) + new_value = str(not app_config['disable-mlockall']['value']) + return 'disable-mlockall', new_value diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index c3f03fb..fed72df 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -436,40 +436,50 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): @classmethod def setUpClass(cls): """Run setup for deferred restart tests.""" - super().setUpClass( - restart_config_file='/etc/rabbitmq/rabbitmq.config', - test_service='rabbitmq-server', - restart_package='rabbitmq-server', - restart_package_service='rabbitmq-server', - application_name='rabbitmq-server') + super().setUpClass(application_name='rabbitmq-server') def check_status_message_is_clear(self): """Check each units status message show no defeerred events.""" + pattern = '(Unit is ready|Unit is ready and clustered)$' for unit in zaza.model.get_units(self.application_name): - assert unit.workload_status_message in [ - 'Unit is ready', - 'Unit is ready and clustered'] + zaza.model.block_until_unit_wl_message_match( + unit.entity_id, + pattern) + zaza.model.block_until_all_units_idle() - def trigger_deferred_hook_via_charm(self): - """Set charm config option which requires a service start. + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. - Set the charm debug option and wait for that change to be renderred in - applications config file. - - This overrides the base class version as the rabbit charm does not - have a debug option and the config file is not in oslo config format. + :returns: Config key and new value + :rtype: (str, bool) """ app_config = zaza.model.get_application_config(self.application_name) - logging.info("Triggering deferred restart via config change") - new_debug_value = str( - int(app_config['connection-backlog'].get('value', 100) + 1)) - logging.info("Setting connection-backlog: {}".format(new_debug_value)) - zaza.model.set_application_config( - self.application_name, - {'connection-backlog': new_debug_value}) - logging.info("Waiting for units to be idle") - test_unit = zaza.model.get_units(self.application_name)[0] - zaza.model.block_until_unit_wl_message_match( - test_unit.entity_id, - status_pattern='.*config-changed.*') - zaza.model.block_until_all_units_idle() + new_value = str(int( + app_config['connection-backlog'].get('value', 100) + 1)) + return 'connection-backlog', new_value + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'rabbitmq-server', + 'rabbitmq-server') + + def check_clear_restarts(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in zaza.model.get_units(self.application_name): + zaza.model.run_action( + unit.entity_id, + 'restart-services', + action_params={'services': 'rabbitmq-server'}) + + # Check workload status no longer shows deferred restarts. + self.check_status_message_is_clear() diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index dec0cd0..a4fbc87 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -704,7 +704,7 @@ class OpenStackBaseTest(BaseCharmTest): return instance_1, instance_2 -class BaseDeferredRestartTest(OpenStackBaseTest): +class BaseDeferredRestartTest(BaseCharmTest): """Check deferred restarts. Example of adding a deferred restart test:: @@ -714,12 +714,17 @@ class BaseDeferredRestartTest(OpenStackBaseTest): @classmethod def setUpClass(cls): - super(NeutronOVSDeferredRestartTest, cls).setUpClass( - restart_config_file = '/etc/neutron/neutron.conf', - test_service = 'neutron-openvswitch-agent', - restart_package = 'openvswitch-switch', - restart_package_service = 'openvswitch-switch', - application_name = 'neutron-openvswitch') + super().setUpClass(application_name='neutron-openvswitch') + + def run_tests(self): + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + NOTE: The test has been broken into various class methods which may require specialisation if the charm being tested is not a standard OpenStack @@ -728,38 +733,23 @@ class BaseDeferredRestartTest(OpenStackBaseTest): """ @classmethod - def setUpClass(cls, restart_config_file, test_service, restart_package, - restart_package_service, application_name): + def setUpClass(cls, application_name): """Run test setup. - :param restart_config_file: Config file that will be changed to trigger - a service restart. - :type restart_config_file: str - :param test_service: Service that will require a restart after - restart_config_file has changed. - :type test_service: str - :param restart_package: Package that will be changed to trigger a - service restart. - :type restart_package: str - :param restart_package_service: Service that will require a restart - after restart_package has changed. - :type restart_package_service: str :param application_name: Name of application to run tests against. :type application_name: str """ - cls.restart_config_file = restart_config_file - cls.test_service = test_service - cls.restart_package = restart_package - cls.restart_package_service = restart_package_service cls.application_name = application_name - super(BaseDeferredRestartTest, cls).setUpClass( - application_name=cls.application_name) + super().setUpClass(application_name=cls.application_name) def check_status_message_is_clear(self): """Check each units status message show no defeerred events.""" # Check workload status no longer shows deferred restarts. for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + model.block_until_unit_wl_message_match( + unit.entity_id, + 'Unit is ready') + model.block_until_all_units_idle() def check_clear_restarts(self): """Clear and deferred restarts and check status. @@ -772,7 +762,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): model.run_action( unit.entity_id, 'restart-services', - action_params={'deferred': True}) + action_params={'deferred-only': True}) # Check workload status no longer shows deferred restarts. self.check_status_message_is_clear() @@ -841,7 +831,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): Run the action to list any deferred events and check it has entry for the given hook. - :param hook: Hook name + :param hook: Hook or method name :type hook: str """ # Ensure that the deferred restart and cause are listed via action @@ -870,85 +860,188 @@ class BaseDeferredRestartTest(OpenStackBaseTest): "message of {}".format(test_service, unit.entity_id))) assert test_service in unit.workload_status_message - def check_deferred_hook_wlm(self): - """Check the workload status message lists deferred event.""" + def check_deferred_hook_wlm(self, deferred_hook): + """Check the workload status message lists deferred event. + + :param deferred_hook: Hook or method name which should be showing as + deferred. + :type deferred_hook: str + """ # Ensure that the deferred restarts are visible in Juju status for unit in model.get_units(self.application_name): logging.info( ("Checking {} is marked as having deferred hook in workload " "message".format(unit.entity_id))) - assert 'config-changed' in unit.workload_status_message + assert deferred_hook in unit.workload_status_message - def trigger_deferred_hook_via_charm(self): + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. + + NOTE: The implementation assumes the charm has a `debug` option and + If that is not true the derived class should override this + method. + :returns: Config key and new value + :rtype: (str, bool) + """ + app_config = model.get_application_config(self.application_name) + return 'debug', str(not app_config['debug']['value']) + + def set_new_config(self): + """Change applications charm config.""" + logging.info("Triggering deferred restart via config change") + config_key, new_value = self.get_new_config() + logging.info("Setting {}: {}".format(config_key, new_value)) + model.set_application_config( + self.application_name, + {config_key: new_value}) + return new_value + + def trigger_deferred_restart_via_charm(self, restart_config_file): """Set charm config option which requires a service start. Set the charm debug option and wait for that change to be renderred in applications config file. - NOTE: The implementation assumes the charm has a `debug` option and - self.restart_config_file in an oslo config file where that - debug option is renderred. If that is not true the specaliasation - class should override this method. + NOTE: The implementation assumes the restart_config_file in an oslo + config file. If that is not true the derived class should + override this method. + + :param restart_config_file: Config file that updated value is expected + in. + :type restart_config_file: str """ - app_config = model.get_application_config(self.application_name) - logging.info("Triggering deferred restart via config change") - new_debug_value = str(not app_config['debug']['value']) - logging.info("Setting debug: {}".format(new_debug_value)) - model.set_application_config( + new_debug_value = self.set_new_config() + expected_contents = { + 'DEFAULT': { + 'debug': [new_debug_value]}} + logging.info("Waiting for debug to be {} in {}".format( + new_debug_value, + restart_config_file)) + model.block_until_oslo_config_entries_match( self.application_name, - {'debug': new_debug_value}) + restart_config_file, + expected_contents) logging.info("Waiting for units to be idle") - test_unit = model.get_units(self.application_name)[0] - model.block_until_unit_wl_message_match( - test_unit.entity_id, - status_pattern='.*config-changed.*') model.block_until_all_units_idle() - def trigger_deferred_restart_via_package(self): - """Update a package which requires a service restart.""" + def trigger_deferred_hook_via_charm(self, deferred_hook): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be rendered in + applications config file. + + :param deferred_hook: Hook or method name which should be showing as + deferred. + :type deferred_hook: str + :returns: New config value + :rtype: Union[str, int, float] + """ + new_debug_value = self.set_new_config() + for unit in model.get_units(self.application_name): + logging.info('Waiting for {} to show deferred hook'.format( + unit.entity_id)) + model.block_until_unit_wl_message_match( + unit.entity_id, + status_pattern='.*{}.*'.format(deferred_hook)) + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + return new_debug_value + + def trigger_deferred_restart_via_package(self, restart_package): + """Update a package which requires a service restart. + + :param restart_package: Package that will be changed to trigger a + service restart. + :type restart_package: str + """ logging.info("Triggering deferred restart via package change") # Test restart requested by package for unit in model.get_units(self.application_name): model.run_on_unit( unit.entity_id, - 'dpkg-reconfigure {}; ./hooks/update-status'.format( - self.restart_package)) + ('dpkg-reconfigure {}; ' + 'JUJU_HOOK_NAME=update-status ./hooks/update-status').format( + restart_package)) - def run_charm_change_test(self): + def run_charm_change_restart_test(self, test_service, restart_config_file): """Trigger a deferred restart by updating a config file via the charm. - Trigger a config-changed hook in the charm which will update a config - file and add a deferred restart. + Trigger a hook in the charm which the charm will defer. - NOTE: If this test is not relevant for the target charm then override - this method and raise unittest.SkipTest + :param test_service: Service that should need a restart + :type test_service: str + :param restart_config_file: Config file that updated value is expected + in. + :type restart_config_file: str """ - self.trigger_deferred_hook_via_charm() + self.trigger_deferred_restart_via_charm(restart_config_file) - self.check_deferred_hook_wlm() - self.check_show_deferred_events_action_hook('config-changed') - # Rerunning to flip config option back to previous value. - self.trigger_deferred_hook_via_charm() + self.check_show_deferred_restarts_wlm(test_service) + self.check_show_deferred_events_action_restart( + test_service, + restart_config_file) logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def run_charm_change_hook_test(self, deferred_hook): + """Trigger a deferred restart by updating a config file via the charm. + + :param deferred_hook: Hook or method name which should be showing as + defeerred. + :type deferred_hook: str + """ + self.trigger_deferred_hook_via_charm(deferred_hook) + + self.check_deferred_hook_wlm(deferred_hook) + self.check_show_deferred_events_action_hook(deferred_hook) + # Rerunning to flip config option back to previous value. + self.trigger_deferred_hook_via_charm(deferred_hook) + logging.info("Running restart action to clear deferred hooks") self.check_clear_hooks() - def run_package_change_test(self): + def run_package_change_test(self, restart_package, restart_package_svc): """Trigger a deferred restart by updating a package. Update a package which requires will add a deferred restart. - NOTE: If this test is not relevant for the target charm then override - this method and raise unittest.SkipTest + :param restart_package: Package that will be changed to trigger a + service restart. + :type restart_package: str + :param restart_package_service: Service that will require a restart + after restart_package has changed. + :type restart_package_service: str """ - self.trigger_deferred_restart_via_package() + self.trigger_deferred_restart_via_package(restart_package) - self.check_show_deferred_restarts_wlm(self.restart_package_service) + self.check_show_deferred_restarts_wlm(restart_package_svc) self.check_show_deferred_events_action_restart( - self.restart_package_service, + restart_package_svc, 'Package update') logging.info("Running restart action to clear deferred restarts") self.check_clear_restarts() + def run_tests(self): + """Run charm tests. should specify which tests to run. + + The charm test that implements this test should specify which tests to + run, for example: + + def run_tests(self): + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a config change which requires a restart + self.run_charm_change_restart_test( + 'neutron-l3-agent', + '/etc/neutron/neutron.conf') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + """ + raise NotImplementedError + def test_deferred_restarts(self): """Run deferred restart tests.""" app_config = model.get_application_config(self.application_name) @@ -977,11 +1070,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): else: logging.info("Auto restarts already disabled") - # Trigger a config change which requires a restart - self.run_charm_change_test() - - # Trigger a package change which requires a restart - self.run_package_change_test() + self.run_tests() # Finished so turn auto-restarts back on. logging.info("Turning on auto restarts") From 7d02173914be51b9fc8cb6d171fe5a918329c44d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 9 Apr 2021 17:10:09 +0000 Subject: [PATCH 854/898] Fix bug caused by disable-mlockall being unseet --- zaza/openstack/charm_tests/ovn/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 6b43abe..1f2f890 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -487,5 +487,5 @@ class OVNDedicatedChassisDeferredRestartTest( :rtype: (str, bool) """ app_config = zaza.model.get_application_config(self.application_name) - new_value = str(not app_config['disable-mlockall']['value']) + new_value = str(not app_config['disable-mlockall'].get('value', False)) return 'disable-mlockall', new_value From 8a04739912db7127866a99830076893466baa191 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 9 Apr 2021 16:31:35 +0000 Subject: [PATCH 855/898] Fix race exposed by rabbit charm. If there are multiple charms of a given application then a hook like leader-settings-changed can trigger a deferred hook. So a race can occur where the deferred hooks of unit1 are cleared then in the process of clearing unit2 a deferred hook is triggerred in unit1. To fix this do not block on the wl status message after the hook test completes. Also, make sure that an action failure is fatal to surface any issues. --- zaza/openstack/charm_tests/test_utils.py | 35 ++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index a4fbc87..15e90f8 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -759,26 +759,38 @@ class BaseDeferredRestartTest(BaseCharmTest): """ # Use action to run any deferred restarts for unit in model.get_units(self.application_name): + logging.info("Running restart-services on {}".format( + unit.entity_id)) model.run_action( unit.entity_id, 'restart-services', - action_params={'deferred-only': True}) + action_params={'deferred-only': True}, + raise_on_failure=True) # Check workload status no longer shows deferred restarts. self.check_status_message_is_clear() - def check_clear_hooks(self): - """Clear and deferred restarts and check status. + def clear_hooks(self): + """Clear and deferred hooks. - Clear and deferred restarts and then check the workload status message - for each unit. + Run any deferred hooks. """ # Use action to run any deferred restarts for unit in model.get_units(self.application_name): + logging.info("Running run-deferred-hooks on {}".format( + unit.entity_id)) model.run_action( unit.entity_id, - 'run-deferred-hooks') + 'run-deferred-hooks', + raise_on_failure=True) + def check_clear_hooks(self): + """Clear deferred hooks and check status. + + Clear deferred hooks and then check the workload status message + for each unit. + """ + self.clear_hooks() # Check workload status no longer shows deferred restarts. self.check_status_message_is_clear() @@ -789,7 +801,10 @@ class BaseDeferredRestartTest(BaseCharmTest): :rtype: Dict """ unit = model.get_units(self.application_name)[0] - action = model.run_action(unit.entity_id, 'show-deferred-events') + action = model.run_action( + unit.entity_id, + 'show-deferred-events', + raise_on_failure=True) return yaml.safe_load(action.data['results']['output']) def check_show_deferred_events_action_restart(self, test_service, @@ -997,7 +1012,10 @@ class BaseDeferredRestartTest(BaseCharmTest): # Rerunning to flip config option back to previous value. self.trigger_deferred_hook_via_charm(deferred_hook) logging.info("Running restart action to clear deferred hooks") - self.check_clear_hooks() + # If there are a number of units in the application and restarts take + # time then another deferred hook can occur so do not block on a + # clear status message. + self.clear_hooks() def run_package_change_test(self, restart_package, restart_package_svc): """Trigger a deferred restart by updating a package. @@ -1080,3 +1098,4 @@ class BaseDeferredRestartTest(BaseCharmTest): self.application_name, policy_file) model.block_until_all_units_idle() + self.check_clear_hooks() From d89a1651d2d8cefbe5e3a18fd725eadd486d0711 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 12 Apr 2021 09:28:31 +0000 Subject: [PATCH 856/898] Fix Neutron Gateway deferred restart test Do not run the deferred hooks action for neutron gateway as it is not needed and will end in an action error. Tested locally here: https://paste.ubuntu.com/p/cSmgxmrTr4/ This closes issue #553 --- zaza/openstack/charm_tests/neutron/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index def9cda..54420e8 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -1045,3 +1045,7 @@ class NeutronGatewayDeferredRestartTest(test_utils.BaseDeferredRestartTest): self.run_package_change_test( 'openvswitch-switch', 'openvswitch-switch') + + def check_clear_hooks(self): + """Gateway does not defer hooks so noop.""" + return From 669b57574b7a36e890215ffcd01049a171fff354 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 13 Apr 2021 10:24:15 +0000 Subject: [PATCH 857/898] Add deferred restart test for ovn central --- zaza/openstack/charm_tests/ovn/tests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 1f2f890..bfa0f21 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -489,3 +489,21 @@ class OVNDedicatedChassisDeferredRestartTest( app_config = zaza.model.get_application_config(self.application_name) new_value = str(not app_config['disable-mlockall'].get('value', False)) return 'disable-mlockall', new_value + + +class OVNCentralDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + """Deferred restart tests for OVN Central.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='ovn-central') + + def run_tests(self): + """Run deferred restart tests.""" + # Charm does not defer hooks so that test is not included. + # Trigger a package change which requires a restart + self.run_package_change_test( + 'ovn-central', + 'ovn-central') From 1c15672b61cd054d3f47e7d3ad2604f711ed5d55 Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 19 Apr 2021 08:00:38 -0700 Subject: [PATCH 858/898] MySQL Partition Test class Currently the new partition test_110 is breaking on master due to the test landing before the charm PR which implemented the change [0] has landed. Move the partition test to its own class which can be added as a target in the charm's tests.yaml. This will not affect master and can be trivially updated in [0]. [0] https://review.opendev.org/c/openstack/charm-mysql-innodb-cluster/+/779427 --- zaza/openstack/charm_tests/mysql/tests.py | 138 +++++++++++----------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 5cc563d..3438932 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -617,73 +617,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) - def test_110_force_quorum_using_partition_of(self): - """Force quorum using partition of instance with given address. - - After outage, cluster can end up without quorum. Force it. - """ - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() - - # Block all traffic across mysql instances: 0<-1, 1<-2 and 2<-0 - mysql_units = [unit for unit in zaza.model.get_units(self.application)] - no_of_units = len(mysql_units) - for index, unit in enumerate(mysql_units): - next_unit = mysql_units[(index+1) % no_of_units] - ip_address = next_unit.public_address - cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) - zaza.model.async_run_on_unit(unit, cmd) - - logging.info( - "Wait till all {} units are in state 'blocked' ..." - .format(self.application)) - for unit in zaza.model.get_units(self.application): - zaza.model.block_until_unit_wl_status( - unit.entity_id, - 'blocked', - negate_match=True) - - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() - - logging.info("Execute force-quorum-using-partition-of action ...") - - # Select "quorum leader" unit - leader_unit = mysql_units[0] - action = zaza.model.run_action( - leader_unit.entity_id, - "force-quorum-using-partition-of", - action_params={ - "address": leader_unit.public_address, - 'i-really-mean-it': True - }) - - assert action.data.get("results") is not None, ( - "Force quorum using partition of action failed: {}" - .format(action.data)) - logging.debug( - "Results from running 'force-quorum' command ...\n{}".format( - action.data)) - - logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() - - # Unblock all traffic across mysql instances - for unit in zaza.model.get_units(self.application): - cmd = "sudo iptables -F" - zaza.model.async_run_on_unit(unit, cmd) - - logging.info("Wait for application states ...") - for unit in zaza.model.get_units(self.application): - zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") - test_config = lifecycle_utils.get_charm_config(fatal=False) - zaza.model.wait_for_application_states( - states=test_config.get("target_deploy_status", {})) - class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" @@ -930,6 +863,77 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): .format(action.data)) +class MySQLInnoDBClusterPartitionTest(MySQLBaseTest): + """MySQL parition handling.""" + + def test_850_force_quorum_using_partition_of(self): + """Force quorum using partition of instance with given address. + + After outage, cluster can end up without quorum. Force it. + """ + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # Block all traffic across mysql instances: 0<-1, 1<-2 and 2<-0 + mysql_units = [unit for unit in zaza.model.get_units(self.application)] + no_of_units = len(mysql_units) + for index, unit in enumerate(mysql_units): + next_unit = mysql_units[(index+1) % no_of_units] + ip_address = next_unit.public_address + cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) + zaza.model.async_run_on_unit(unit, cmd) + + logging.info( + "Wait till all {} units are in state 'blocked' ..." + .format(self.application)) + for unit in zaza.model.get_units(self.application): + zaza.model.block_until_unit_wl_status( + unit.entity_id, + 'blocked', + negate_match=True) + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info("Execute force-quorum-using-partition-of action ...") + + # Select "quorum leader" unit + leader_unit = mysql_units[0] + action = zaza.model.run_action( + leader_unit.entity_id, + "force-quorum-using-partition-of", + action_params={ + "address": leader_unit.public_address, + 'i-really-mean-it': True + }) + + assert action.data.get("results") is not None, ( + "Force quorum using partition of action failed: {}" + .format(action.data)) + logging.debug( + "Results from running 'force-quorum' command ...\n{}".format( + action.data)) + + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + + # Unblock all traffic across mysql instances + for unit in zaza.model.get_units(self.application): + cmd = "sudo iptables -F" + zaza.model.async_run_on_unit(unit, cmd) + + logging.info("Wait for application states ...") + for unit in zaza.model.get_units(self.application): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {})) + + class MySQLRouterTests(test_utils.OpenStackBaseTest): """MySQL Router Tests.""" From 49929ef01885666123918b12c8a624b5294ef0b2 Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 19 Apr 2021 09:05:45 -0700 Subject: [PATCH 859/898] Fix typo --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 3438932..bd054e2 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -864,7 +864,7 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): class MySQLInnoDBClusterPartitionTest(MySQLBaseTest): - """MySQL parition handling.""" + """MySQL partition handling.""" def test_850_force_quorum_using_partition_of(self): """Force quorum using partition of instance with given address. From 0b401de17f1e2814f46184e5df0ec49d140902c8 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 19 Apr 2021 17:03:05 -0400 Subject: [PATCH 860/898] Get OpenStack codename from /etc/openstack-release In newer versions of Ubuntu and the Cloud Archive (currently Hirsute, Wallaby, and above), there is an openstack-release package that, if installed, specifies the current OpenStack release in /etc/openstack-release. Also adds wallaby definitions to non-version definitions in os_versions.py. Fixes #560 --- .../test_zaza_utilities_openstack.py | 36 ++++++++++++- zaza/openstack/utilities/openstack.py | 50 +++++++++++++++++-- zaza/openstack/utilities/os_versions.py | 4 +- 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index c42fc9d..b7f2eeb 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -884,7 +884,24 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): result = openstack_utils.get_current_os_release_pair() self.assertEqual(expected, result) - def test_get_openstack_release(self): + def test_get_current_os_versions(self): + self.patch_object(openstack_utils, "get_openstack_release") + self.patch_object(openstack_utils.generic_utils, "get_pkg_version") + + # Pre-Wallaby scenario where openstack-release package isn't installed + self.get_openstack_release.return_value = None + self.get_pkg_version.return_value = '18.0.0' + expected = {'keystone': 'victoria'} + result = openstack_utils.get_current_os_versions('keystone') + self.assertEqual(expected, result) + + # Wallaby+ scenario where openstack-release package is installed + self.get_openstack_release.return_value = 'wallaby' + expected = {'keystone': 'wallaby'} + result = openstack_utils.get_current_os_versions('keystone') + self.assertEqual(expected, result) + + def test_get_os_release(self): self.patch( 'zaza.openstack.utilities.openstack.get_current_os_release_pair', new_callable=mock.MagicMock(), @@ -936,6 +953,23 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.get_application_config_option.return_value = None self.assertEqual(openstack_utils.get_keystone_api_version(), 3) + def test_get_openstack_release(self): + self.patch_object(openstack_utils.model, "get_units") + self.patch_object(openstack_utils.juju_utils, "remote_run") + + # Test pre-Wallaby behavior where openstack-release pkg isn't installed + self.get_units.return_value = [] + self.remote_run.return_value = "OPENSTACK_CODENAME=wallaby " + + # Test Wallaby+ behavior where openstack-release package is installed + unit1 = mock.MagicMock() + unit1.entity_id = 1 + self.get_units.return_value = [unit1] + self.remote_run.return_value = "OPENSTACK_CODENAME=wallaby " + + result = openstack_utils.get_openstack_release("application", "model") + self.assertEqual(result, "wallaby") + def test_get_project_id(self): # No domain self.patch_object(openstack_utils, "get_keystone_api_version") diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 230a3f6..75ab9d7 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1753,9 +1753,44 @@ def get_os_code_info(package, pkg_version): return OPENSTACK_CODENAMES[vers] +def get_openstack_release(application, model_name=None): + """Return the openstack release codename based on /etc/openstack-release. + + This will only return a codename if the openstack-release package is + installed on the unit. + + :param application: Application name + :type application: string + :param model_name: Name of model to query. + :type model_name: str + :returns: OpenStack release codename for application + :rtype: string + """ + versions = [] + units = model.get_units(application, model_name=model_name) + for unit in units: + cmd = 'cat /etc/openstack-release | grep OPENSTACK_CODENAME' + out = juju_utils.remote_run(unit.entity_id, cmd, model_name=model_name) + codename = out.split('=')[1].strip() + versions.append(codename) + if len(set(versions)) == 0: + return None + elif len(set(versions)) > 1: + raise Exception('Unexpected mix of OpenStack releases for {}: {}', + application, versions) + return versions[0] + + def get_current_os_versions(deployed_applications, model_name=None): """Determine OpenStack codename of deployed applications. + Initially, see if the openstack-release pkg is available and use it + instead. + + If it isn't then it falls back to the existing method of checking the + version of the package passed and then resolving the version from that + using lookup tables. + :param deployed_applications: List of deployed applications :type deployed_applications: list :param model_name: Name of model to query. @@ -1769,11 +1804,16 @@ def get_current_os_versions(deployed_applications, model_name=None): continue logging.info("looking at application: {}".format(application)) - version = generic_utils.get_pkg_version(application['name'], - application['type']['pkg'], - model_name=model_name) - versions[application['name']] = ( - get_os_code_info(application['type']['pkg'], version)) + codename = get_openstack_release(application['name'], + model_name=model_name) + if codename: + versions[application['name']] = codename + else: + version = generic_utils.get_pkg_version(application['name'], + application['type']['pkg'], + model_name=model_name) + versions[application['name']] = ( + get_os_code_info(application['type']['pkg'], version)) return versions diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 420253c..fdb6f0f 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -36,6 +36,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('eoan', 'train'), ('focal', 'ussuri'), ('groovy', 'victoria'), + ('hirsute', 'wallaby'), ]) @@ -69,7 +70,8 @@ OPENSTACK_RELEASES_PAIRS = [ 'bionic_queens', 'bionic_rocky', 'cosmic_rocky', 'bionic_stein', 'disco_stein', 'bionic_train', 'eoan_train', 'bionic_ussuri', 'focal_ussuri', - 'focal_victoria', 'groovy_victoria'] + 'focal_victoria', 'groovy_victoria', + 'focal_wallaby', 'hirsute_wallaby'] SWIFT_CODENAMES = OrderedDict([ ('diablo', From 9a48757d5e214c4c0c4e14e8528a1f895a54901c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 20 Apr 2021 12:24:23 -0400 Subject: [PATCH 861/898] Fix version fallback path when /etc/openstack-release doesn't exist --- zaza/openstack/utilities/openstack.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 75ab9d7..846cca4 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1770,9 +1770,14 @@ def get_openstack_release(application, model_name=None): units = model.get_units(application, model_name=model_name) for unit in units: cmd = 'cat /etc/openstack-release | grep OPENSTACK_CODENAME' - out = juju_utils.remote_run(unit.entity_id, cmd, model_name=model_name) - codename = out.split('=')[1].strip() - versions.append(codename) + try: + out = juju_utils.remote_run(unit.entity_id, cmd, + model_name=model_name) + except model.CommandRunFailed: + logging.info('Fall back to version check for OpenStack codename') + else: + codename = out.split('=')[1].strip() + versions.append(codename) if len(set(versions)) == 0: return None elif len(set(versions)) > 1: From d70ff79b1e5e6bfc7fcffac98ef1e80f92da4c28 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 20 Apr 2021 13:05:18 -0400 Subject: [PATCH 862/898] Change log to debug to create less noise --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 846cca4..79e1c54 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1774,7 +1774,7 @@ def get_openstack_release(application, model_name=None): out = juju_utils.remote_run(unit.entity_id, cmd, model_name=model_name) except model.CommandRunFailed: - logging.info('Fall back to version check for OpenStack codename') + logging.debug('Fall back to version check for OpenStack codename') else: codename = out.split('=')[1].strip() versions.append(codename) From a05263f111ef79e134200936815bb0a8a389c2d2 Mon Sep 17 00:00:00 2001 From: Billy Olsen Date: Thu, 22 Apr 2021 04:13:40 -0700 Subject: [PATCH 863/898] Update nova tests (#563) Add test for the new nova_cc charm action syncing AZs The new action is meant to sync the availability zones from the underlying Juju provider with the OpenStack availability zones. It is useful within MAAS environments to map the MAAZ AZs to OpenStack AZs. Also fix output check logic for sync az test. The validation logic for the sync az test checks the specific output of the command text, but this is dependent on ordering. The code also validates that the hypervisors are added to the appropriate host aggregates, so this changes the test to verify the action was successful and has some output. Also use the availability-zone based on the configured settings. In the attempt to sync the az test, the availability zone is assumed to be the JUJU_AVAILABILITY_ZONE environment setting. In various scenarios, this is not what is used for the availability zone of a compute node. This change mimics the logic that's contained within the nova-compute charm to determine which availability zone should be used for the compute node. Signed-off-by: Billy Olsen Co-authored-by: Ionut Balutoiu --- zaza/openstack/charm_tests/nova/tests.py | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index f2c68e0..1e44e31 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -319,6 +319,62 @@ class NovaComputeActionTest(test_utils.OpenStackBaseTest): "The action failed: {}".format(action.data["message"])) +class NovaCloudControllerActionTest(test_utils.OpenStackBaseTest): + """Run nova-cloud-controller specific tests. + + Add this test class for new nova-cloud-controller action + to avoid breaking older version. + """ + + def test_sync_compute_az_action(self): + """Test sync-compute-availability-zones action.""" + juju_units_az_map = {} + compute_config = zaza.model.get_application_config('nova-compute') + default_az = compute_config['default-availability-zone']['value'] + use_juju_az = compute_config['customize-failure-domain']['value'] + + for unit in zaza.model.get_units('nova-compute', + model_name=self.model_name): + zone = default_az + if use_juju_az: + result = zaza.model.run_on_unit(unit.name, + 'echo $JUJU_AVAILABILITY_ZONE', + model_name=self.model_name, + timeout=60) + self.assertEqual(int(result['Code']), 0) + juju_az = result['Stdout'].strip() + if juju_az: + zone = juju_az + + juju_units_az_map[unit.public_address] = zone + continue + + session = openstack_utils.get_overcloud_keystone_session() + nova = openstack_utils.get_nova_session_client(session) + + result = zaza.model.run_action_on_leader( + 'nova-cloud-controller', + 'sync-compute-availability-zones', + model_name=self.model_name) + + # For validating the action results, we simply want to validate that + # the action was completed and we have something in the output. The + # functional validation really occurs below, in that the hosts are + # checked to be in the appropriate host aggregates. + self.assertEqual(result.status, 'completed') + self.assertNotEqual('', result.results['output']) + + unique_az_list = list(set(juju_units_az_map.values())) + aggregates = nova.aggregates.list() + self.assertEqual(len(aggregates), len(unique_az_list)) + for unit_address in juju_units_az_map: + az = juju_units_az_map[unit_address] + aggregate = nova.aggregates.find( + name='{}_az'.format(az), availability_zone=az) + hypervisor = nova.hypervisors.find(host_ip=unit_address) + self.assertIn(hypervisor.hypervisor_hostname, aggregate.hosts) + + class NovaCloudController(test_utils.OpenStackBaseTest): """Run nova-cloud-controller specific tests.""" From 5d533fba6af93395eff006e9912fea3b3cae4893 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 23 Apr 2021 09:26:52 -0700 Subject: [PATCH 864/898] Restart blocked mysql routers (#565) LP Bug #1918953 [0] was resolved on the mysql-innodb-cluster side using coordinated delayed action. Since then we have seen a similar issue in CI [1] after the pause and resume test. Mysql-router hangs with: 2021-04-21 20:42:05 metadata_cache WARNING [7f3f968d5700] Instance '192.168.254.18:3306' [72b4ac2c-a2dd-11eb-82a5-fa163e5a4b7e] of replicaset 'default' is unreachable. Increasing metadata cache refresh frequency. This cannot be fixed from the cluster side. I will be looking into solutions on the mysql-rotuer side. But in the meantime, to unblock the mysql-innodb-cluster gate this change restarts blocked MySQL routers. [0] https://bugs.launchpad.net/charm-mysql-router/+bug/1918953 [1] https://openstack-ci-reports.ubuntu.com/artifacts/test_charm_pipeline_func_full/openstack/charm-mysql-innodb-cluster/786514/3/8479/consoleText.test_charm_func_full_11494.txt --- zaza/openstack/charm_tests/mysql/tests.py | 59 +++++++++++++++++++++++ zaza/openstack/charm_tests/test_utils.py | 26 ++++++++++ 2 files changed, 85 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index bd054e2..4044c55 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -113,6 +113,56 @@ class MySQLBaseTest(test_utils.OpenStackBaseTest): if _primary_ip in unit.public_address: return unit + def get_blocked_mysql_routers(self): + """Get blocked mysql routers. + + :returns: List of blocked mysql-router unit names + :rtype: List[str] + """ + # Make sure mysql-router units are up to date + # We cannot assume they are as there is up to a five minute delay + mysql_router_units = [] + for application in self.get_applications_with_substring_in_name( + "mysql-router"): + for unit in zaza.model.get_units(application): + mysql_router_units.append(unit.entity_id) + self.run_update_status_hooks(mysql_router_units) + + # Get up to date status + status = zaza.model.get_status().applications + blocked_mysql_routers = [] + # Check if the units are blocked + for application in self.get_applications_with_substring_in_name( + "mysql-router"): + # Subordinate dance with primary + # There is no satus[applicatoin]["units"] for subordinates + _subordinate_to = status[application].subordinate_to[0] + for appunit in status[_subordinate_to].units: + for subunit in ( + status[_subordinate_to]. + units[appunit].subordinates.keys()): + if "blocked" in ( + status[_subordinate_to].units[appunit]. + subordinates[subunit].workload_status.status): + blocked_mysql_routers.append(subunit) + return blocked_mysql_routers + + def restart_blocked_mysql_routers(self): + """Restart blocked mysql routers. + + :returns: None + :rtype: None + """ + # Check for blocked mysql-router units + blocked_mysql_routers = self.get_blocked_mysql_routers() + for unit in blocked_mysql_routers: + logging.warning( + "Restarting blocked mysql-router unit {}" + .format(unit)) + zaza.model.run_on_unit( + unit, + "systemctl restart {}".format(unit.rpartition("/")[0])) + class MySQLCommonTests(MySQLBaseTest): """Common mysql charm tests.""" @@ -169,6 +219,15 @@ class MySQLCommonTests(MySQLBaseTest): """ with self.pause_resume(self.services): logging.info("Testing pause resume") + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # If there are any blocekd mysql routers restart them. + self.restart_blocked_mysql_routers() + assert not self.get_blocked_mysql_routers(), ( + "Should no longer be blocked mysql-router units") + logging.info("Passed pause and resume test.") diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 15e90f8..bc67933 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -573,6 +573,32 @@ class BaseCharmTest(unittest.TestCase): return self.test_config.get('tests_options', {}).get( '.'.join(caller_path + [key]), default) + def get_applications_with_substring_in_name(self, substring): + """Get applications with substring in name. + + :param substring: String to search for in application names + :type substring: str + :returns: List of matching applictions + :rtype: List + """ + status = model.get_status().applications + applications = [] + for application in status.keys(): + if substring in application: + applications.append(application) + return applications + + def run_update_status_hooks(self, units): + """Run update status hooks on units. + + :param units: List of unit names or unit.entity_id + :type units: List[str] + :returns: None + :rtype: None + """ + for unit in units: + model.run_on_unit(unit, "hooks/update-status") + class OpenStackBaseTest(BaseCharmTest): """Generic helpers for testing OpenStack API charms.""" From e631b9354117cae33f49f1343be0776cd4c0ca4e Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 4 May 2021 11:46:27 +0200 Subject: [PATCH 865/898] Add proper support for Octavia's tempest tests (#498) * Update tempest config for Octavia * ensure the test_server is executable * remove reference to accounts.yaml --- zaza/openstack/charm_tests/tempest/setup.py | 31 +++++++++++++++++-- .../tempest/templates/tempest_v3.j2 | 9 +++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 5c85d9c..b22ccb7 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -17,6 +17,7 @@ import jinja2 import urllib.parse import os +import subprocess import zaza.utilities.deployment_env as deployment_env import zaza.openstack.utilities.juju as juju_utils @@ -170,6 +171,27 @@ def add_keystone_config(ctxt, keystone_session): ctxt['default_domain_id'] = domain.id +def add_octavia_config(ctxt): + """Add octavia config to context. + + :param ctxt: Context dictionary + :type ctxt: dict + :returns: None + :rtype: None + :raises: subprocess.CalledProcessError + """ + subprocess.check_call([ + 'curl', + "{}:80/swift/v1/fixtures/test_server.bin".format( + ctxt['test_swift_ip']), + '-o', "{}/test_server.bin".format(ctxt['workspace_path']) + ]) + subprocess.check_call([ + 'chmod', '+x', + "{}/test_server.bin".format(ctxt['workspace_path']) + ]) + + def get_service_list(keystone_session): """Retrieve list of services from keystone. @@ -227,7 +249,7 @@ def add_auth_config(ctxt): overcloud_auth['OS_PROJECT_DOMAIN_NAME']) -def get_tempest_context(): +def get_tempest_context(workspace_path): """Generate the tempest config context. :returns: Context dictionary @@ -235,6 +257,7 @@ def get_tempest_context(): """ keystone_session = openstack_utils.get_overcloud_keystone_session() ctxt = {} + ctxt['workspace_path'] = workspace_path ctxt_funcs = { 'nova': add_nova_config, 'neutron': add_neutron_config, @@ -253,6 +276,7 @@ def get_tempest_context(): ctxt_func(ctxt, keystone_session) add_environment_var_config(ctxt, ctxt['enabled_services']) add_auth_config(ctxt) + add_octavia_config(ctxt) return ctxt @@ -289,13 +313,14 @@ def setup_tempest(tempest_template, accounts_template): workspace_name, workspace_path = tempest_utils.get_workspace() tempest_utils.destroy_workspace(workspace_name, workspace_path) tempest_utils.init_workspace(workspace_path) + context = get_tempest_context(workspace_path) render_tempest_config( os.path.join(workspace_path, 'etc/tempest.conf'), - get_tempest_context(), + context, tempest_template) render_tempest_config( os.path.join(workspace_path, 'etc/accounts.yaml'), - get_tempest_context(), + context, accounts_template) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 index b441f2d..5cb65ed 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 @@ -4,7 +4,6 @@ use_stderr = false log_file = tempest.log [auth] -test_accounts_file = accounts.yaml default_credentials_domain_name = {{ default_credentials_domain_name }} admin_username = {{ admin_username }} admin_project_name = {{ admin_project_name }} @@ -65,6 +64,7 @@ floating_network_name = {{ ext_net }} [network-feature-enabled] ipv6 = false api_extensions = {{ neutron_api_extensions }} +port_security = true {% endif %} {% if 'heat' in enabled_services %} @@ -104,3 +104,10 @@ catalog_type = {{ catalog_type }} [volume-feature-enabled] backup = false {% endif %} + +{% if 'octavia' in enabled_services %} +[load_balancer] +enable_security_groups = true +test_with_ipv6 = false +test_server_path = {{ workspace_path }}/test_server.bin +{% endif %} \ No newline at end of file From cd2df7534c33424d4881823f8cb154f2ead62765 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 5 May 2021 11:39:53 +0000 Subject: [PATCH 866/898] Seperate TrilioGhostNFSShareTest from TrilioDataMoverTest as its not needed for s3 --- zaza/openstack/charm_tests/trilio/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 581c395..0bf0cd8 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -463,7 +463,7 @@ class TrilioDMAPITest(TrilioBaseTest): services = ["dmapi-api"] -class TrilioDataMoverTest(TrilioGhostNFSShareTest): +class TrilioDataMoverTest(TrilioBaseTest): """Tests for Trilio Data Mover charm.""" conf_file = "/etc/tvault-contego/tvault-contego.conf" From 8e998c2d02b2f66522e844e83bba451d376f4e50 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 6 May 2021 08:22:10 +0000 Subject: [PATCH 867/898] More seperation --- zaza/openstack/charm_tests/trilio/tests.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 0bf0cd8..5072b29 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -463,10 +463,18 @@ class TrilioDMAPITest(TrilioBaseTest): services = ["dmapi-api"] -class TrilioDataMoverTest(TrilioBaseTest): +class TrilioDataMoverBaseTest(TrilioBaseTest): """Tests for Trilio Data Mover charm.""" conf_file = "/etc/tvault-contego/tvault-contego.conf" application_name = "trilio-data-mover" services = ["tvault-contego"] + + +class TrilioDataMoverNFSTest(TrilioDataMoverBaseTest, TrilioGhostNFSShareTest): + """Tests for Trilio Data Mover charm backed by NFS.""" + + +class TrilioDataMoverS3Test(TrilioDataMoverBaseTest): + """Tests for Trilio Data Mover charm backed by S3.""" From 779137f60fbcda8197a63d1d93380d5cad962223 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 6 May 2021 14:12:33 +0200 Subject: [PATCH 868/898] Restrict adding Octavia config Only add Octavia's config when Octavia is in the enabled services --- zaza/openstack/charm_tests/tempest/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index b22ccb7..7ae0ba3 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -276,7 +276,8 @@ def get_tempest_context(workspace_path): ctxt_func(ctxt, keystone_session) add_environment_var_config(ctxt, ctxt['enabled_services']) add_auth_config(ctxt) - add_octavia_config(ctxt) + if 'octavia' in ctxt['enabled_services']: + add_octavia_config(ctxt) return ctxt From 03ddaf0079a31277b2ca0c635f1f29454cd56612 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 10 May 2021 13:44:24 +0200 Subject: [PATCH 869/898] Add more logging around downloading/caching test images --- zaza/openstack/utilities/openstack.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 79e1c54..237e02d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2478,7 +2478,11 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], local_path = os.path.join(image_cache_dir, img_name) if not os.path.exists(local_path): + logging.info('Downloading {} ...'.format(image_url)) download_image(image_url, local_path) + else: + logging.info('Cached image found at {} - Skipping download'.format( + local_path)) image = upload_image_to_glance( glance, local_path, image_name, backend=backend, From 7dd96c8fb1ddfaf85539fa77809eef670b084fd4 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 11 May 2021 10:51:18 +0200 Subject: [PATCH 870/898] Revert "Skip OVN provider octavia test on Victoria" This reverts commit c4691ef1c74365bf2a33f4922f852247e205e8bb. --- zaza/openstack/charm_tests/octavia/tests.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 535d7aa..52083ab 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -180,14 +180,6 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): for provider in octavia_client.provider_list().get('providers', []) if provider['name'] != 'octavia' # alias for `amphora`, skip } - if (openstack_utils.get_os_release() in [ - openstack_utils.get_os_release('focal_victoria'), - openstack_utils.get_os_release('groovy_victoria')]): - logging.info("Skipping tests of ovn backed lb (Bug #1896603)") - try: - del providers['ovn'] - except KeyError: - pass return providers def _create_lb_resources(self, octavia_client, provider, vip_subnet_id, From 3aacd5c89c5ab66403036a60e02421b3207de020 Mon Sep 17 00:00:00 2001 From: mkalcok <69471063+mkalcok@users.noreply.github.com> Date: Thu, 13 May 2021 12:48:06 +0200 Subject: [PATCH 871/898] Functional tests for `instance-count` action on nova-compute charm. (#535) Launchpad: #1911011 Gerrit review: https://review.opendev.org/c/openstack/charm-nova-compute/+/783371 * Functional tests for `instance-count` action on nova-compute charm. Test for `remove_from_cloud_actions` waits at the end until the nova-compute service is available again. However there's a few seconds lag until that host shows up also in `hypervisor-list` which can cause race conditions with other tests. --- zaza/openstack/charm_tests/nova/tests.py | 34 +++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 1e44e31..5f6709a 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -143,7 +143,39 @@ class CloudActions(test_utils.OpenStackBaseTest): for service in self.nova_client.services.list(binary='nova-compute'): self.assertEqual(service.status, 'enabled') - def test_950_remove_from_cloud_actions(self): + def test_950_instance_count_action(self): + """Test that action 'instance-count' returns expected values.""" + def check_instance_count(expect_count, unit_name): + """Assert that unit with 'unit_name' has 'expect_count' of VMs. + + :param expect_count: How many VMs are expected to be running + :param unit_name: Name of the target nova-compute unit + :return: None + :raises AssertionError: If result of the 'instance-count' action + does not match 'expect_count'. + """ + logging.debug('Running "instance-count" action on unit "{}".' + 'Expecting result: {}'.format(unit_name, + expect_count)) + result = zaza.model.run_action(unit_name, 'instance-count') + self.assertEqual(result.status, 'completed') + instances = result.data.get('results', {}).get('instance-count') + self.assertEqual(instances, str(expect_count)) + + nova_unit = zaza.model.get_units('nova-compute', + model_name=self.model_name)[0] + + check_instance_count(0, nova_unit.entity_id) + + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME) + + check_instance_count(1, nova_unit.entity_id) + + self.resource_cleanup() + + def test_960_remove_from_cloud_actions(self): """Test actions remove-from-cloud and register-to-cloud. Note (martin-kalcok): This test requires that nova-compute unit is not From 1c4a117c3aa8280e0c1cd057fb09bf412c3ee257 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 17 May 2021 11:12:13 +0000 Subject: [PATCH 872/898] Add ovn codename wallaby (20.12) --- zaza/openstack/utilities/os_versions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index fdb6f0f..beaedec 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -121,6 +121,8 @@ OVN_CODENAMES = OrderedDict([ ['20.03']), ('victoria', ['20.06']), + ('wallaby', + ['20.12']), ]) # >= Liberty version->codename mapping From 620a098e8d4d714324269f94a436d3fe02556db3 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 18 May 2021 07:56:34 +0000 Subject: [PATCH 873/898] Add debugging --- zaza/openstack/charm_tests/trilio/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/trilio/setup.py b/zaza/openstack/charm_tests/trilio/setup.py index 1c8e97c..11fe187 100644 --- a/zaza/openstack/charm_tests/trilio/setup.py +++ b/zaza/openstack/charm_tests/trilio/setup.py @@ -39,12 +39,14 @@ def nfs_setup(): trilio_wlm_unit = zaza_model.get_first_unit_name("trilio-wlm") nfs_shares_conf = {"nfs-shares": "{}:/srv/testing".format(nfs_server_ip)} + logging.info("NFS share config: {}".format(nfs_shares_conf)) _trilio_services = ["trilio-wlm", "trilio-data-mover"] conf_changed = False for juju_service in _trilio_services: app_config = zaza_model.get_application_config(juju_service) if app_config["nfs-shares"] != nfs_shares_conf["nfs-shares"]: + logging.info("Updating nfs-shares config option") zaza_model.set_application_config(juju_service, nfs_shares_conf) conf_changed = True From d51a28dc0435f05a323a8fb426ab09f921d5139e Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 13 May 2021 06:15:46 +0200 Subject: [PATCH 874/898] Re-order change of expected workload status After recent changes to zaza workload status code the present ordering does no longer work. --- zaza/openstack/charm_tests/octavia/setup.py | 23 ++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/setup.py b/zaza/openstack/charm_tests/octavia/setup.py index 4b09aa0..b04b696 100644 --- a/zaza/openstack/charm_tests/octavia/setup.py +++ b/zaza/openstack/charm_tests/octavia/setup.py @@ -107,23 +107,36 @@ def configure_octavia(): 'amp-ssh-pub-key': base64.b64encode( bytes(ssh_public_key, 'utf-8')).decode('utf-8'), } - logging.info('Configuring certificates for mandatory Octavia ' - 'client/server authentication ' - '(client being the ``Amphorae`` load balancer instances)') + + # Tell Octavia charm it is safe to create cloud resources, we do this now + # because the workload status will be checked on config-change and it gets + # a bit complicated to augment test config to accept 'blocked' vs. 'active' + # in the various stages. + logging.info('Running `configure-resources` action on Octavia leader unit') + zaza.model.run_action_on_leader( + 'octavia', + 'configure-resources', + action_params={}) # Our expected workload status will change after we have configured the # certificates test_config = zaza.charm_lifecycle.utils.get_charm_config() del test_config['target_deploy_status']['octavia'] + logging.info('Configuring certificates for mandatory Octavia ' + 'client/server authentication ' + '(client being the ``Amphorae`` load balancer instances)') + _singleton = zaza.openstack.charm_tests.test_utils.OpenStackBaseTest() _singleton.setUpClass(application_name='octavia') with _singleton.config_change(charm_config, charm_config): # wait for configuration to be applied then return pass - # Tell Octavia charm it is safe to create cloud resources - logging.info('Running `configure-resources` action on Octavia leader unit') + # Should we consider making the charm attempt to create this key on + # config-change? + logging.info('Running `configure-resources` action again to ensure ' + 'Octavia Nova SSH key pair is created after config change.') zaza.model.run_action_on_leader( 'octavia', 'configure-resources', From be1706cb76d6cc2f7d7b20bbf3d69e63f21e8cd1 Mon Sep 17 00:00:00 2001 From: Nobuto Murata Date: Fri, 21 May 2021 15:19:28 +0900 Subject: [PATCH 875/898] Functional testing for glance external S3 backend Set up Ceph-Radosgw in the same model as an external S3 backend for Glance: - create an S3 user in Radosgw - create an S3 bucket for Glance - inject those parameters into Glance charm configurations as an external S3 backend - upload an image to the S3 backend via Glance, and check the size of the image in the S3 bucket directly Co-authored-by: Connor Chamberlain --- zaza/openstack/charm_tests/glance/setup.py | 59 ++++++++++++++++++++++ zaza/openstack/charm_tests/glance/tests.py | 56 +++++++++++++++++++- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index 6686d70..32f7d4e 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -14,7 +14,11 @@ """Code for configuring glance.""" +import json import logging + +import boto3 +import zaza.model as model import zaza.openstack.utilities.openstack as openstack_utils import zaza.utilities.deployment_env as deployment_env @@ -161,3 +165,58 @@ def add_lts_image(glance_client=None, image_name=None, release=None, glance_client=glance_client, image_name=image_name, properties=properties) + + +def configure_external_s3_backend(): + """Set up Ceph-radosgw as an external S3 backend for Glance.""" + logging.info("Creating a test S3 user and credentials for Glance") + username, displayname = "zaza-glance-test", "Zaza Glance Test User" + cmd = "radosgw-admin user create --uid='{}' --display-name='{}'".format( + username, displayname + ) + results = model.run_on_leader("ceph-mon", cmd) + stdout = json.loads(results["stdout"]) + keys = stdout["keys"][0] + access_key, secret_key = keys["access_key"], keys["secret_key"] + + logging.info("Getting S3 endpoint URL of Radosgw from Keystone") + keystone_auth = openstack_utils.get_overcloud_auth() + keystone_client = openstack_utils.get_keystone_client(keystone_auth) + endpoint_url = keystone_client.session.get_endpoint( + service_type="s3", + interface="public", + region="RegionOne", + ) + + logging.info("Creating a test S3 bucketfor Glance") + bucket_name = "zaza-glance-s3-test" + s3_client = boto3.client( + "s3", + endpoint_url=endpoint_url, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + ) + s3_client.create_bucket(Bucket=bucket_name) + + logging.info("Updating Glance configs with S3 endpoint infomation") + model.set_application_config( + "glance", + { + "s3-store-host": endpoint_url, + "s3-store-access-key": access_key, + "s3-store-secret-key": secret_key, + "s3-store-bucket": bucket_name, + }, + ) + model.wait_for_agent_status() + + logging.info("Waiting for units to reach target states") + model.wait_for_application_states( + states={ + "glance": { + "workload-status": "active", + "workload-status-message": "Unit is ready", + } + } + ) + model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index 5d45338..799e69a 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -18,8 +18,10 @@ import logging -import zaza.openstack.utilities.openstack as openstack_utils +import boto3 +import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils class GlanceTest(test_utils.OpenStackBaseTest): @@ -142,3 +144,55 @@ class GlanceCephRGWBackendTest(test_utils.OpenStackBaseTest): 'image size {}'.format(image['size'], total_bytes)) self.assertEqual(image['size'], total_bytes) openstack_utils.delete_image(self.glance_client, image['id']) + + +class GlanceExternalS3Test(test_utils.OpenStackBaseTest): + """Encapsulate glance tests using an external S3 backend.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running glance tests with S3 backend.""" + super(GlanceExternalS3Test, cls).setUpClass() + cls.glance_client = openstack_utils.get_glance_session_client( + cls.keystone_session + ) + + configs = model.get_application_config("glance") + cls.s3_store_host = configs["s3-store-host"]["value"] + cls.s3_store_access_key = configs["s3-store-access-key"]["value"] + cls.s3_store_secret_key = configs["s3-store-secret-key"]["value"] + cls.s3_store_bucket = configs["s3-store-bucket"]["value"] + + def test_100_create_delete_image(self): + """Create an image and do a simple validation of it. + + Validate the size of the image in both Glance API and actual S3 bucket. + """ + image_name = "zaza-s3-test-image" + openstack_utils.create_image( + glance=self.glance_client, + image_url=openstack_utils.find_cirros_image(arch="x86_64"), + image_name=image_name, + backend="s3", + ) + images = openstack_utils.get_images_by_name( + self.glance_client, image_name + ) + self.assertEqual(len(images), 1) + image = images[0] + + s3_client = boto3.client( + "s3", + endpoint_url=self.s3_store_host, + aws_access_key_id=self.s3_store_access_key, + aws_secret_access_key=self.s3_store_secret_key, + ) + response = s3_client.head_object( + Bucket=self.s3_store_bucket, Key=image["id"] + ) + logging.info( + "Checking glance image size {} matches S3 object's ContentLength " + "{}".format(image["size"], response["ContentLength"]) + ) + self.assertEqual(image["size"], response["ContentLength"]) + openstack_utils.delete_image(self.glance_client, image["id"]) From 6a688d29bb242eee8874e3c9d4c95cb438f91d4a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 25 May 2021 13:18:11 +0000 Subject: [PATCH 876/898] Add classes to collect NFS and S3 Trilio tests --- zaza/openstack/charm_tests/trilio/tests.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 5072b29..64239ae 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -440,7 +440,7 @@ class TrilioGhostNFSShareTest(TrilioBaseTest): ) -class TrilioWLMTest(TrilioGhostNFSShareTest): +class TrilioWLMBaseTest(TrilioBaseTest): """Tests for Trilio Workload Manager charm.""" conf_file = "/etc/workloadmgr/workloadmgr.conf" @@ -478,3 +478,11 @@ class TrilioDataMoverNFSTest(TrilioDataMoverBaseTest, TrilioGhostNFSShareTest): class TrilioDataMoverS3Test(TrilioDataMoverBaseTest): """Tests for Trilio Data Mover charm backed by S3.""" + + +class TrilioWLMNFSTest(TrilioWLMBaseTest, TrilioGhostNFSShareTest): + """Tests for Trilio WLM charm backed by NFS.""" + + +class TrilioWLMS3Test(TrilioWLMBaseTest): + """Tests for Trilio WLM charm backed by S3.""" From 855847339404f7cb37b63e65a7f38e1a812d4980 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 26 May 2021 13:49:46 +0200 Subject: [PATCH 877/898] Re-enable test_003_test_override_is_observed This test was disabled for releases older than groovy-victoria because of https://bugs.launchpad.net/ubuntu/+source/python-oslo.policy/+bug/1880959 This bug has now been fixed and released for xenial-queens and newer. Also made it more robust in GlanceTests // fixes #578 --- zaza/openstack/charm_tests/policyd/tests.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 4722755..a87f2ef 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -401,9 +401,10 @@ class BasePolicydSpecialization(PolicydTest, def test_003_test_override_is_observed(self): """Test that the override is observed by the underlying service.""" if (openstack_utils.get_os_release() < - openstack_utils.get_os_release('groovy_victoria')): + openstack_utils.get_os_release('xenial_queens')): raise unittest.SkipTest( - "Test skipped until Bug #1880959 is fix released") + "Test skipped because bug #1880959 won't be fixed for " + "releases older than Queens") if self._test_name is None: logging.info("Doing policyd override for {}" .format(self._service_name)) @@ -571,6 +572,13 @@ class GlanceTests(BasePolicydSpecialization): super(GlanceTests, cls).setUpClass(application_name="glance") cls.application_name = "glance" + # NOTE(lourot): Same as NeutronApiTests. There is a race between the glance + # charm signalling its readiness and the service actually being ready to + # serve requests. The test will fail intermittently unless we gracefully + # accept this. + # Issue: openstack-charmers/zaza-openstack-tests#578 + @tenacity.retry(wait=tenacity.wait_fixed(1), + reraise=True, stop=tenacity.stop_after_delay(8)) def get_client_and_attempt_operation(self, ip): """Attempt to list the images as a policyd override. From 7ba3141cdb8c530d3bf92a4743c062141169a8db Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 27 May 2021 13:26:58 +0200 Subject: [PATCH 878/898] glance_simplestreams_sync/setup: Add tenacity The very nature of the action ran by the configure step will cause it to fail frequently unless it is retried. Fixes #580 --- .../glance_simplestreams_sync/setup.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py index 06e172d..ec6107b 100644 --- a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py @@ -17,6 +17,7 @@ """Code for configuring glance-simplestreams-sync.""" import logging +import tenacity import zaza.model as zaza_model import zaza.openstack.utilities.generic as generic_utils @@ -30,11 +31,17 @@ def sync_images(): deployment. """ logging.info("Synchronising images using glance-simplestreams-sync") - generic_utils.assertActionRanOK( - zaza_model.run_action_on_leader( - "glance-simplestreams-sync", - "sync-images", - raise_on_failure=True, - action_params={}, - ) - ) + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10), + reraise=True): + with attempt: + generic_utils.assertActionRanOK( + zaza_model.run_action_on_leader( + "glance-simplestreams-sync", + "sync-images", + raise_on_failure=True, + action_params={}, + ) + ) From 2e7d52b9cda5ca4a2ca07f45ddf6f9ad4396c0b5 Mon Sep 17 00:00:00 2001 From: Nobuto Murata Date: Fri, 28 May 2021 22:12:51 +0900 Subject: [PATCH 879/898] fix typo --- zaza/openstack/charm_tests/glance/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index 32f7d4e..367c980 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -188,7 +188,7 @@ def configure_external_s3_backend(): region="RegionOne", ) - logging.info("Creating a test S3 bucketfor Glance") + logging.info("Creating a test S3 bucket for Glance") bucket_name = "zaza-glance-s3-test" s3_client = boto3.client( "s3", @@ -198,7 +198,7 @@ def configure_external_s3_backend(): ) s3_client.create_bucket(Bucket=bucket_name) - logging.info("Updating Glance configs with S3 endpoint infomation") + logging.info("Updating Glance configs with S3 endpoint information") model.set_application_config( "glance", { From e67b0ea1e11e5f7bb0a7058199a765077caabb8c Mon Sep 17 00:00:00 2001 From: Hernan Garcia Date: Wed, 26 May 2021 07:10:45 -0500 Subject: [PATCH 880/898] added glance functional test for image-conversion config --- .../test_zaza_utilities_openstack.py | 6 +++-- zaza/openstack/charm_tests/glance/tests.py | 26 +++++++++++++++++++ zaza/openstack/utilities/openstack.py | 23 +++++++++++++--- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index b7f2eeb..0434a2b 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -550,7 +550,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): backend=None, disk_format='qcow2', visibility='public', - container_format='bare') + container_format='bare', + force_import=False) def test_create_image_pass_directory(self): glance_mock = mock.MagicMock() @@ -574,7 +575,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): backend=None, disk_format='qcow2', visibility='public', - container_format='bare') + container_format='bare', + force_import=False) self.gettempdir.assert_not_called() def test_create_ssh_key(self): diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index 5d45338..d81e440 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -67,6 +67,32 @@ class GlanceTest(test_utils.OpenStackBaseTest): {'image_format': {'disk_formats': ['qcow2']}}, ['glance-api']) + def test_412_image_conversion(self): + """Check image-conversion config. + + When image-conversion config is enabled glance will convert images + to raw format, this is only performed for interoperable image import + docs.openstack.org/glance/train/admin/interoperable-image-import.html + image conversion is done at server-side for better image handling + """ + current_release = openstack_utils.get_os_release() + bionic_stein = openstack_utils.get_os_release('bionic_stein') + if current_release < bionic_stein: + self.skipTest('image-conversion config is supported since ' + 'bionic_stein or newer versions') + + with self.config_change({'image-conversion': 'false'}, + {'image-conversion': 'true'}): + image_url = openstack_utils.find_cirros_image(arch='x86_64') + image = openstack_utils.create_image( + self.glance_client, + image_url, + 'cirros-test-import', + force_import=True) + + disk_format = self.glance_client.images.get(image.id).disk_format + self.assertEqual('raw', disk_format) + def test_900_restart_on_config_change(self): """Checking restart happens on config change.""" # Config file affected by juju set config change diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 237e02d..3082068 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2407,7 +2407,7 @@ def delete_volume_backup(cinder, vol_backup_id): def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', visibility='public', container_format='bare', - backend=None): + backend=None, force_import=False): """Upload the given image to glance and apply the given label. :param glance: Authenticated glanceclient @@ -2424,6 +2424,9 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', format that also contains metadata about the actual virtual machine. :type container_format: str + :param force_import: Force the use of glance image import + instead of direct upload + :type force_import: boolean :returns: glance image pointer :rtype: glanceclient.common.utils.RequestIdProxy """ @@ -2433,7 +2436,15 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', disk_format=disk_format, visibility=visibility, container_format=container_format) - glance.images.upload(image.id, open(local_path, 'rb'), backend=backend) + + if force_import: + logging.info('Forcing image import') + glance.images.stage(image.id, open(local_path, 'rb')) + glance.images.image_import( + image.id, method='glance-direct', backend=backend) + else: + glance.images.upload( + image.id, open(local_path, 'rb'), backend=backend) resource_reaches_status( glance.images, @@ -2446,7 +2457,8 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], properties=None, backend=None, disk_format='qcow2', - visibility='public', container_format='bare'): + visibility='public', container_format='bare', + force_import=False): """Download the image and upload it to glance. Download an image from image_url and upload it to glance labelling @@ -2465,6 +2477,9 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], :type tags: list of str :param properties: Properties and values to add to image :type properties: dict + :param force_import: Force the use of glance image import + instead of direct upload + :type force_import: boolean :returns: glance image pointer :rtype: glanceclient.common.utils.RequestIdProxy """ @@ -2487,7 +2502,7 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], image = upload_image_to_glance( glance, local_path, image_name, backend=backend, disk_format=disk_format, visibility=visibility, - container_format=container_format) + container_format=container_format, force_import=force_import) for tag in tags: result = glance.image_tags.update(image.id, tag) logging.debug( From 7e46cde193cfa1f17b68cda261f40f9515825076 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 2 Jun 2021 15:12:13 +0200 Subject: [PATCH 881/898] policyd: Add keystoneauth1 403 to accepted exceptions Fixes #584 --- zaza/openstack/charm_tests/policyd/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index a87f2ef..fde3866 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -688,5 +688,6 @@ class OctaviaTests(BasePolicydSpecialization): self.get_keystone_session_admin_user(ip)) try: octavia_client.provider_list() - except octaviaclient.OctaviaClientException: + except (octaviaclient.OctaviaClientException, + keystoneauth1.exceptions.http.Forbidden): raise PolicydOperationFailedException() From 38581b70e533c16b0f191874eefac44a735b91b1 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 3 Jun 2021 08:11:54 +0200 Subject: [PATCH 882/898] gss: Log Keystone service catalog on failure The simplestreams service relies on the Keystone Service catalog. To help debug any issues occurring during test execution, log the contents of the Keystone service catalog in the event of failure. Related-Bug: #1930654 --- .../glance_simplestreams_sync/setup.py | 58 ++++++++++++++----- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py index ec6107b..96648af 100644 --- a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py @@ -18,9 +18,31 @@ import logging import tenacity +import pprint import zaza.model as zaza_model import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +def _get_catalog(): + """Retrieve the Keystone service catalog. + + :returns: The raw Keystone service catalog. + :rtype: List[Dict] + """ + keystone_session = openstack_utils.get_overcloud_keystone_session() + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + + token = keystone_session.get_token() + token_data = keystone_client.tokens.get_token_data(token) + + if 'catalog' not in token_data['token']: + raise ValueError('catalog not in token data: "{}"' + .format(pprint.pformat(token_data))) + + return token_data['token']['catalog'] def sync_images(): @@ -31,17 +53,27 @@ def sync_images(): deployment. """ logging.info("Synchronising images using glance-simplestreams-sync") - for attempt in tenacity.Retrying( - stop=tenacity.stop_after_attempt(3), - wait=tenacity.wait_exponential( - multiplier=1, min=2, max=10), - reraise=True): - with attempt: - generic_utils.assertActionRanOK( - zaza_model.run_action_on_leader( - "glance-simplestreams-sync", - "sync-images", - raise_on_failure=True, - action_params={}, + + catalog = None + try: + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10), + reraise=True): + with attempt: + # Proactively retrieve the Keystone service catalog so that we + # can log it in the event of a failure. + catalog = _get_catalog() + generic_utils.assertActionRanOK( + zaza_model.run_action_on_leader( + "glance-simplestreams-sync", + "sync-images", + raise_on_failure=True, + action_params={}, + ) ) - ) + except Exception: + logging.info('Contents of Keystone service catalog: "{}"' + .format(pprint.pformat(catalog))) + raise From 6645c60e93bd70ae5ef0302cacf63d83299f418d Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 9 Jun 2021 03:20:53 -0700 Subject: [PATCH 883/898] Handle Vault running tests twice (#588) Vault's runs its tests twice. In some circumstances the removal of the vault key from the test_config dictionary can throw a KeyError. Do not fail when this happens. --- zaza/openstack/charm_tests/vault/tests.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 29719d5..9014928 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -111,7 +111,11 @@ class UnsealVault(BaseVaultTest): vault_utils.run_charm_authorize(self.vault_creds['root_token']) if not test_config: test_config = lifecycle_utils.get_charm_config() - del test_config['target_deploy_status']['vault'] + try: + del test_config['target_deploy_status']['vault'] + except KeyError: + # Already removed + pass zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) @@ -154,7 +158,11 @@ class VaultTest(BaseVaultTest): allowed_domains='openstack.local') test_config = lifecycle_utils.get_charm_config() - del test_config['target_deploy_status']['vault'] + try: + del test_config['target_deploy_status']['vault'] + except KeyError: + # Already removed + pass zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', cacert.decode().strip()) From dee6032fc3bafe57f1098e89b1ab7a65bd7a081d Mon Sep 17 00:00:00 2001 From: Nobuto Murata Date: Sun, 13 Jun 2021 16:51:51 +0900 Subject: [PATCH 884/898] Skip test_vault_reload with enabling/disabling mlock in LXD When Vault is running in a LXD container, test_vault_reload is expected to fail because mlock cannot be enabled in LXD containers. Skip it when running in containers is detected. It's useful for local development environments with Juju's localhost/LXD provider. The test won't be skipped in the OpenStack Charm CI since the Vault units will be run in KVM, not in LXD containers. --- zaza/openstack/charm_tests/vault/tests.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 9014928..6058b21 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -18,6 +18,7 @@ import contextlib import hvac +import json import logging import time import unittest @@ -277,6 +278,15 @@ class VaultTest(BaseVaultTest): raise unittest.SkipTest("The version of charm-vault tested does " "not have reload action") + container_results = zaza.model.run_on_leader( + "vault", "systemd-detect-virt --container" + ) + container_rc = json.loads(container_results["Code"]) + if container_rc == 0: + raise unittest.SkipTest( + "Vault unit is running in a container. Cannot use mlock." + ) + lead_client = vault_utils.get_cluster_leader(self.clients) running_config = vault_utils.get_running_config(lead_client) value_to_set = not running_config['data']['disable_mlock'] @@ -286,7 +296,7 @@ class VaultTest(BaseVaultTest): 'vault', {'disable-mlock': str(value_to_set)}) - logging.info("Waiting for modle to be idle ...") + logging.info("Waiting for model to be idle ...") zaza.model.block_until_all_units_idle(model_name=self.model_name) logging.info("Testing action reload on {}".format(lead_client)) From e9cb5aea5af93e99e65cb98da39b27250e45a953 Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 14 Jun 2021 20:25:19 +0000 Subject: [PATCH 885/898] Retry request on test csr for Vault --- zaza/openstack/charm_tests/vault/tests.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 6058b21..9b898bf 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -24,6 +24,7 @@ import time import unittest import uuid import tempfile +import tenacity import requests import zaza.charm_lifecycle.utils as lifecycle_utils @@ -171,10 +172,19 @@ class VaultTest(BaseVaultTest): states=test_config.get('target_deploy_status', {})) ip = zaza.model.get_app_ips( 'keystone')[0] + with tempfile.NamedTemporaryFile(mode='w') as fp: fp.write(cacert.decode()) fp.flush() - requests.get('https://{}:5000'.format(ip), verify=fp.name) + # Avoid race condition and retry + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=2, min=2, max=10)): + with attempt: + logging.info( + "Attempting to connect to https://{}:5000".format(ip)) + requests.get('https://{}:5000'.format(ip), verify=fp.name) def test_all_clients_authenticated(self): """Check all vault clients are authenticated.""" From b7e542636f742f022fe449633789770cae67c84d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 16 Jun 2021 09:14:29 +0100 Subject: [PATCH 886/898] Extend trilio snapshot timeout (#594) Extend trilio snapshot timeout, closes issue #595. --- zaza/openstack/charm_tests/trilio/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 64239ae..ad477d8 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -262,7 +262,7 @@ class WorkloadmgrCLIHelper(object): retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=30), - stop=tenacity.stop_after_delay(720), + stop=tenacity.stop_after_delay(900), reraise=True, ) From 8ba11347870dd7c8d2be00f3b973c3326a52d9a3 Mon Sep 17 00:00:00 2001 From: Rodrigo Barbieri Date: Thu, 17 Jun 2021 15:18:29 -0300 Subject: [PATCH 887/898] Add pci-alias test for nova-compute (#587) Moved test that was previously existing for nova-cloud-controller to a common class to be run for nova-compute as well. --- zaza/openstack/charm_tests/nova/tests.py | 161 ++++++++++++----------- 1 file changed, 87 insertions(+), 74 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 5f6709a..66ca957 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -83,6 +83,79 @@ class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): self.resource_cleanup() +class NovaCommonTests(test_utils.OpenStackBaseTest): + """nova-compute and nova-cloud-controller common tests.""" + + XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') + XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') + XENIAL_QUEENS = openstack_utils.get_os_release('xenial_queens') + BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens') + BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky') + BIONIC_TRAIN = openstack_utils.get_os_release('bionic_train') + + @classmethod + def setUpClass(cls): + """Run class setup for running nova-cloud-controller tests.""" + super(NovaCommonTests, cls).setUpClass() + cls.current_release = openstack_utils.get_os_release() + + def _test_pci_alias_config(self, app_name, service_list): + logging.info('Checking pci aliases in nova config...') + + # Expected default and alternate values + current_value = zaza.model.get_application_config( + app_name)['pci-alias'] + try: + current_value = current_value['value'] + except KeyError: + current_value = None + new_value = '[{}, {}]'.format( + json.dumps({ + 'name': 'IntelNIC', + 'capability_type': 'pci', + 'product_id': '1111', + 'vendor_id': '8086', + 'device_type': 'type-PF' + }, sort_keys=True), + json.dumps({ + 'name': ' Cirrus Logic ', + 'capability_type': 'pci', + 'product_id': '0ff2', + 'vendor_id': '10de', + 'device_type': 'type-PCI' + }, sort_keys=True)) + + set_default = {'pci-alias': current_value} + set_alternate = {'pci-alias': new_value} + + expected_conf_section = 'pci' + expected_conf_key = 'alias' + + default_entry = {expected_conf_section: {}} + alternate_entry = {expected_conf_section: { + expected_conf_key: [ + ('{"capability_type": "pci", "device_type": "type-PF", ' + '"name": "IntelNIC", "product_id": "1111", ' + '"vendor_id": "8086"}'), + ('{"capability_type": "pci", "device_type": "type-PCI", ' + '"name": " Cirrus Logic ", "product_id": "0ff2", ' + '"vendor_id": "10de"}')]}} + + # Config file affected by juju set config change + conf_file = '/etc/nova/nova.conf' + + # Make config change, check for service restarts + logging.info( + 'Setting config on {} to {}'.format(app_name, set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + service_list) + + class CloudActions(test_utils.OpenStackBaseTest): """Test actions from actions/cloud.py.""" @@ -242,9 +315,19 @@ class CloudActions(test_utils.OpenStackBaseTest): "nova-cloud-controller as expected.") -class NovaCompute(test_utils.OpenStackBaseTest): +class NovaCompute(NovaCommonTests): """Run nova-compute specific tests.""" + def test_311_pci_alias_config_compute(self): + """Verify that the pci alias data is rendered properly. + + Change pci-alias and assert that change propagates to the correct + file and that services are restarted as a result + """ + # We are not touching the behavior of anything older than QUEENS + if self.current_release >= self.XENIAL_QUEENS: + self._test_pci_alias_config("nova-compute", ['nova-compute']) + def test_500_hugepagereport_action(self): """Test hugepagereport action.""" for unit in zaza.model.get_units('nova-compute', @@ -407,22 +490,9 @@ class NovaCloudControllerActionTest(test_utils.OpenStackBaseTest): self.assertIn(hypervisor.hypervisor_hostname, aggregate.hosts) -class NovaCloudController(test_utils.OpenStackBaseTest): +class NovaCloudController(NovaCommonTests): """Run nova-cloud-controller specific tests.""" - XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') - XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') - XENIAL_QUEENS = openstack_utils.get_os_release('xenial_queens') - BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens') - BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky') - BIONIC_TRAIN = openstack_utils.get_os_release('bionic_train') - - @classmethod - def setUpClass(cls): - """Run class setup for running nova-cloud-controller tests.""" - super(NovaCloudController, cls).setUpClass() - cls.current_release = openstack_utils.get_os_release() - @property def services(self): """Return a list of services for the selected OpenStack release.""" @@ -518,70 +588,13 @@ class NovaCloudController(test_utils.OpenStackBaseTest): 'filter:legacy_ratelimit': { 'limits': ["( POST, '*', .*, 9999, MINUTE );"]}}) - def test_310_pci_alias_config(self): + def test_310_pci_alias_config_ncc(self): """Verify that the pci alias data is rendered properly. Change pci-alias and assert that change propagates to the correct file and that services are restarted as a result """ - logging.info('Checking pci aliases in nova config...') - - # Expected default and alternate values - current_value = zaza.model.get_application_config( - 'nova-cloud-controller')['pci-alias'] - try: - current_value = current_value['value'] - except KeyError: - current_value = None - new_value = '[{}, {}]'.format( - json.dumps({ - 'name': 'IntelNIC', - 'capability_type': 'pci', - 'product_id': '1111', - 'vendor_id': '8086', - 'device_type': 'type-PF' - }, sort_keys=True), - json.dumps({ - 'name': ' Cirrus Logic ', - 'capability_type': 'pci', - 'product_id': '0ff2', - 'vendor_id': '10de', - 'device_type': 'type-PCI' - }, sort_keys=True)) - - set_default = {'pci-alias': current_value} - set_alternate = {'pci-alias': new_value} - - expected_conf_section = 'DEFAULT' - expected_conf_key = 'pci_alias' - if self.current_release >= self.XENIAL_OCATA: - expected_conf_section = 'pci' - expected_conf_key = 'alias' - - default_entry = {expected_conf_section: {}} - alternate_entry = {expected_conf_section: { - expected_conf_key: [ - ('{"capability_type": "pci", "device_type": "type-PF", ' - '"name": "IntelNIC", "product_id": "1111", ' - '"vendor_id": "8086"}'), - ('{"capability_type": "pci", "device_type": "type-PCI", ' - '"name": " Cirrus Logic ", "product_id": "0ff2", ' - '"vendor_id": "10de"}')]}} - - # Config file affected by juju set config change - conf_file = '/etc/nova/nova.conf' - - # Make config change, check for service restarts - logging.info( - 'Setting config on nova-cloud-controller to {}'.format( - set_alternate)) - self.restart_on_changed( - conf_file, - set_default, - set_alternate, - default_entry, - alternate_entry, - self.services) + self._test_pci_alias_config("nova-cloud-controller", self.services) def test_900_restart_on_config_change(self): """Checking restart happens on config change. From 858487e2070e40b223cf0485de7defa284d9124f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 15 Jun 2021 12:21:25 +0200 Subject: [PATCH 888/898] octavia: Grant role to ensure access to LBAASv2 API Starting with OpenStack Wallaby the LBAASv2 API will require authentication with system scoped token unless the user has the 'load-balancer_admin' role. Grant role on test setup, and remove it again on clean-up. --- zaza/openstack/charm_tests/octavia/tests.py | 78 +++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 52083ab..6208397 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -27,6 +27,74 @@ import zaza.openstack.utilities.openstack as openstack_utils from zaza.openstack.utilities import ObjectRetrierWraps +LBAAS_ADMIN_ROLE = 'load-balancer_admin' + + +def _op_role_current_user(keystone_client, keystone_session, op, role_name, + scope=None): + """Perform role operation on current user. + + :param keystone_client: Keysonte cilent object + :type keystone_client: keystoneclient.v3.Client + :param keystone_session: Keystone session object + :type keystone_session: keystoneauth1.session.Session + :param op: Operation to perform, one of ('grant', 'revoke') + :type op: str + :param role_name: Name of role + :type role_name: str + :param scope: Scope to apply role to, one of ('domain', 'project'(default)) + :type scope: Optional[str] + :returns: the granted role returned from server. + :rtype: keystoneclient.v3.roles.Role + :raises: ValueError, keystoneauth1.exceptions.* + """ + allowed_ops = ('grant', 'revoke') + if op not in allowed_ops: + raise ValueError('op "{}" not in allowed_ops "{}"' + .format(op, allowed_ops)) + scope = scope or 'project' + allowed_scope = ('domain', 'project') + if scope not in allowed_scope: + raise ValueError('scope "{}" not in allowed_scope "{}"' + .format(scope, allowed_scope)) + + logging.info('{} "{}" role {} current user with "{}" scope...' + .format(op.capitalize(), role_name, + 'to' if op == 'grant' else 'from', + scope)) + role_method = getattr(keystone_client.roles, op) + token = keystone_session.get_token() + token_data = keystone_client.tokens.get_token_data(token) + role = keystone_client.roles.find(name=role_name) + + kwargs = { + 'user': token_data['token']['user']['id'], + scope: token_data['token'][scope]['id'], + } + return role_method( + role, + **kwargs) + + +def grant_role_current_user(keystone_client, keystone_session, role_name, + scope=None): + """Grant role to current user. + + Please refer to docstring for _op_role_current_user. + """ + _op_role_current_user( + keystone_client, keystone_session, 'grant', role_name, scope=scope) + + +def revoke_role_current_user(keystone_client, keystone_session, role_name, + scope=None): + """Grant role to current user. + + Please refer to docstring for _op_role_current_user. + """ + _op_role_current_user( + keystone_client, keystone_session, 'revoke', role_name, scope=scope) + class CharmOperationTest(test_utils.OpenStackBaseTest): """Charm operation tests.""" @@ -67,6 +135,11 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): super(LBAASv2Test, cls).setUpClass() cls.keystone_client = ObjectRetrierWraps( openstack_utils.get_keystone_session_client(cls.keystone_session)) + + # add role to admin user for the duration of the test + grant_role_current_user(cls.keystone_client, cls.keystone_session, + LBAAS_ADMIN_ROLE) + cls.neutron_client = ObjectRetrierWraps( openstack_utils.get_neutron_session_client(cls.keystone_session)) cls.octavia_client = ObjectRetrierWraps( @@ -132,6 +205,11 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): pass # allow resource cleanup to be run multiple times self.loadbalancers = [] + + # revoke role from admin user added by this test + revoke_role_current_user(self.keystone_client, self.keystone_session, + LBAAS_ADMIN_ROLE) + for fip in self.fips: self.neutron_client.delete_floatingip(fip) # allow resource cleanup to be run multiple times From 8a78f9e4c49bc1d57a7cea4197a623c1404936a9 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 18 Jun 2021 11:02:33 +0200 Subject: [PATCH 889/898] policyd: Grant role to ensure access to LBAASv2 API Starting with OpenStack Wallaby the LBAASv2 API will require authentication with system scoped token unless the user has the 'load-balancer_admin' role. Grant role on test setup, and remove it again on clean-up. --- zaza/openstack/charm_tests/policyd/tests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index fde3866..87fd677 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -51,6 +51,9 @@ import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.keystone as ch_keystone import zaza.openstack.utilities.exceptions as zaza_exceptions +import zaza.openstack.charm_tests.octavia.tests as octavia_tests + +from zaza.openstack.utilities import ObjectRetrierWraps class PolicydTest(object): @@ -673,6 +676,20 @@ class OctaviaTests(BasePolicydSpecialization): """Run class setup for running OctaviaTests charm operation tests.""" super(OctaviaTests, cls).setUpClass(application_name="octavia") cls.application_name = "octavia" + cls.keystone_client = ObjectRetrierWraps( + openstack_utils.get_keystone_session_client(cls.keystone_session)) + + # add role to admin user for the duration of the test + octavia_tests.grant_role_current_user( + cls.keystone_client, cls.keystone_session, + octavia_tests.LBAAS_ADMIN_ROLE) + + def resource_cleanup(self): + """Restore changes made by test.""" + # revoke role from admin user added by this test + octavia_tests.revoke_role_current_user( + self.keystone_client, self.keystone_session, + octavia_tests.LBAAS_ADMIN_ROLE) def get_client_and_attempt_operation(self, ip): """Attempt to list available provider drivers. @@ -688,6 +705,7 @@ class OctaviaTests(BasePolicydSpecialization): self.get_keystone_session_admin_user(ip)) try: octavia_client.provider_list() + self.run_resource_cleanup = True except (octaviaclient.OctaviaClientException, keystoneauth1.exceptions.http.Forbidden): raise PolicydOperationFailedException() From 110df7f2c7b801e16da063b78c5909c6456a55c7 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 24 Jun 2021 09:19:34 +0000 Subject: [PATCH 890/898] Add very basic ceph dashboard tests. --- .../charm_tests/ceph/dashboard/tests.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 zaza/openstack/charm_tests/ceph/dashboard/tests.py diff --git a/zaza/openstack/charm_tests/ceph/dashboard/tests.py b/zaza/openstack/charm_tests/ceph/dashboard/tests.py new file mode 100644 index 0000000..7b21ab1 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/dashboard/tests.py @@ -0,0 +1,97 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulating `ceph-dashboard` testing.""" + +import collections +import os +import requests + +import zaza +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.utilities.deployment_env as deployment_env + + +class CephDashboardTest(test_utils.BaseCharmTest): + """Class for `ceph-dashboard` tests.""" + + REMOTE_CERT_FILE = ('/usr/local/share/ca-certificates/' + 'vault_ca_cert_dashboard.crt') + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph dashboard tests.""" + super().setUpClass() + cls.application_name = 'ceph-dashboard' + cls.local_ca_cert = cls.collect_ca() + + @classmethod + def collect_ca(cls): + """Collect CA from ceph-dashboard unit.""" + local_ca_cert = os.path.join( + deployment_env.get_tmpdir(), + os.path.basename(cls.REMOTE_CERT_FILE)) + if not os.path.isfile(local_ca_cert): + units = zaza.model.get_units(cls.application_name) + zaza.model.scp_from_unit( + units[0].entity_id, + cls.REMOTE_CERT_FILE, + local_ca_cert) + return local_ca_cert + + def test_dashboard_units(self): + """Check dashboard units are configured correctly.""" + # XXX: Switch to using CA for verification when + # https://bugs.launchpad.net/cloud-archive/+bug/1933410 + # is fix released. + # verify = self.local_ca_cert + verify = False + units = zaza.model.get_units(self.application_name) + rcs = collections.defaultdict(list) + for unit in units: + r = requests.get( + 'https://{}:8443'.format(unit.public_address), + verify=verify, + allow_redirects=False) + rcs[r.status_code].append(unit.public_address) + self.assertEqual(len(rcs[requests.codes.ok]), 1) + self.assertEqual(len(rcs[requests.codes.see_other]), len(units) - 1) + + def create_user(self, username, role='administrator'): + """Store the model aliases in a global. + + :param username: Username to create. + :type username: str + :param role: Role to grant to user. + :type role: str + :returns: Results from action. + :rtype: juju.action.Action + """ + action = zaza.model.run_action_on_leader( + 'ceph-dashboard', + 'add-user', + action_params={ + 'username': username, + 'role': role}) + return action + + def test_create_user(self): + """Test create user action.""" + test_user = 'marvin' + action = self.create_user(test_user) + self.assertEqual(action.status, "completed") + self.assertTrue(action.data['results']['password']) + action = self.create_user(test_user) + # Action should fail as the user already exists + self.assertEqual(action.status, "failed") From dc78fb6616d86eb8f9637f36522a3be941dd5acc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 24 Jun 2021 09:51:06 +0000 Subject: [PATCH 891/898] Add missing __init__ --- .../charm_tests/ceph/dashboard/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 zaza/openstack/charm_tests/ceph/dashboard/__init__.py diff --git a/zaza/openstack/charm_tests/ceph/dashboard/__init__.py b/zaza/openstack/charm_tests/ceph/dashboard/__init__.py new file mode 100644 index 0000000..f34c394 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/dashboard/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ``ceph-dashboard``.""" From 7cec6a27b56041b07ba616346b64a8327d0469cd Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 24 Jun 2021 11:57:21 +0000 Subject: [PATCH 892/898] Fix doc string --- zaza/openstack/charm_tests/ceph/dashboard/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/dashboard/tests.py b/zaza/openstack/charm_tests/ceph/dashboard/tests.py index 7b21ab1..e7c8863 100644 --- a/zaza/openstack/charm_tests/ceph/dashboard/tests.py +++ b/zaza/openstack/charm_tests/ceph/dashboard/tests.py @@ -69,7 +69,7 @@ class CephDashboardTest(test_utils.BaseCharmTest): self.assertEqual(len(rcs[requests.codes.see_other]), len(units) - 1) def create_user(self, username, role='administrator'): - """Store the model aliases in a global. + """Create a dashboard user. :param username: Username to create. :type username: str From 1009b70a87648fff38c84d805f92e84a76222635 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 29 Jun 2021 12:38:43 +0000 Subject: [PATCH 893/898] Ensure application name is passed in scenario tests When running Trilio tests in a scenario test there is no 'charm_name' so the application name needs to be explicitly set when calling the setup class otherwise some config, such as lead_unit, is missing. --- zaza/openstack/charm_tests/trilio/tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index ad477d8..6ee23f4 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -329,7 +329,7 @@ class TrilioBaseTest(test_utils.OpenStackBaseTest): @classmethod def setUpClass(cls): """Run class setup for running tests.""" - super().setUpClass() + super().setUpClass(application_name=cls.application_name) cls.cinder_client = openstack_utils.get_cinder_session_client( cls.keystone_session ) @@ -474,15 +474,19 @@ class TrilioDataMoverBaseTest(TrilioBaseTest): class TrilioDataMoverNFSTest(TrilioDataMoverBaseTest, TrilioGhostNFSShareTest): """Tests for Trilio Data Mover charm backed by NFS.""" + application_name = "trilio-data-mover" class TrilioDataMoverS3Test(TrilioDataMoverBaseTest): """Tests for Trilio Data Mover charm backed by S3.""" + application_name = "trilio-data-mover" class TrilioWLMNFSTest(TrilioWLMBaseTest, TrilioGhostNFSShareTest): """Tests for Trilio WLM charm backed by NFS.""" + application_name = "trilio-wlm" class TrilioWLMS3Test(TrilioWLMBaseTest): """Tests for Trilio WLM charm backed by S3.""" + application_name = "trilio-wlm" From ba1914eb8395f265440023c91b6c196e73bba703 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 29 Jun 2021 12:45:11 +0000 Subject: [PATCH 894/898] Fix lint --- zaza/openstack/charm_tests/trilio/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 6ee23f4..6fb7692 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -474,19 +474,23 @@ class TrilioDataMoverBaseTest(TrilioBaseTest): class TrilioDataMoverNFSTest(TrilioDataMoverBaseTest, TrilioGhostNFSShareTest): """Tests for Trilio Data Mover charm backed by NFS.""" + application_name = "trilio-data-mover" class TrilioDataMoverS3Test(TrilioDataMoverBaseTest): """Tests for Trilio Data Mover charm backed by S3.""" + application_name = "trilio-data-mover" class TrilioWLMNFSTest(TrilioWLMBaseTest, TrilioGhostNFSShareTest): """Tests for Trilio WLM charm backed by NFS.""" + application_name = "trilio-wlm" class TrilioWLMS3Test(TrilioWLMBaseTest): """Tests for Trilio WLM charm backed by S3.""" + application_name = "trilio-wlm" From 698d8f04e357d2024421db2b5b884064d576d92a Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 26 Apr 2021 20:23:08 +0200 Subject: [PATCH 895/898] Add support for restarting and then retesting manila share services In this change, a charm test that subclasses the ManilaBaseTest class can also opt-in to re-validating a share after restarting the share services by overriding the _restart_share_instance method. --- zaza/openstack/charm_tests/manila/tests.py | 81 ++++++++++++++++--- .../charm_tests/manila_ganesha/tests.py | 19 +++++ 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index 1b58f65..8381f2f 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -228,6 +228,33 @@ packages: self.mount_dir), verify=verify_status) + @tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) + def _clear_testing_file_on_instance(self, instance_ip, ssh_user_name, + ssh_private_key): + """Clear a file on a Manila share mounted into a Nova instance. + + Remove a testing file into the already mounted Manila share from the + given Nova instance (which is meant to be validated from another + instance). These commands are executed via SSH. + + :param instance_ip: IP of the Nova instance. + :type instance_ip: string + :param ssh_user_name: SSH user name. + :type ssh_user_name: string + :param ssh_private_key: SSH private key. + :type ssh_private_key: string + """ + openstack_utils.ssh_command( + vm_name="instance-{}".format(instance_ip), + ip=instance_ip, + username=ssh_user_name, + privkey=ssh_private_key, + command='sudo rm {}/test'.format( + self.mount_dir), + verify=verify_status) + @tenacity.retry( stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) @@ -254,24 +281,28 @@ packages: command='sudo cat {}/test'.format(self.mount_dir), verify=verify_manila_testing_file) + def _restart_share_instance(self): + """Restart the share service's provider. + + restart_share_instance is intended to be overridden with driver + specific implementations that allow verrification that the share is + still accessible after the service is restarted. + + :returns bool: If the test should re-validate + :rtype: bool + """ + return False + def test_manila_share(self): """Test that a Manila share can be accessed on two instances. - 1. Create a share - 2. Spawn two servers + 1. Spawn two servers + 2. Create a share 3. Mount it on both 4. Write a file on one 5. Read it on the other 6. Profit """ - # Create a share - share = self.manila_client.shares.create( - share_type=self.share_type_name, - name=self.share_name, - share_proto=self.share_protocol, - share_network=self.share_network, - size=1) - # Spawn Servers instance_1 = self.launch_guest( guest_name='ins-1', @@ -285,6 +316,14 @@ packages: fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] + # Create a share + share = self.manila_client.shares.create( + share_type=self.share_type_name, + name=self.share_name, + share_proto=self.share_protocol, + share_network=self.share_network, + size=1) + # Wait for the created share to become available before it gets used. openstack_utils.resource_reaches_status( self.manila_client.shares, @@ -313,3 +352,25 @@ packages: fip_2, ssh_user_name, privkey, share_path) self._validate_testing_file_from_instance( fip_2, ssh_user_name, privkey) + + # Restart the share provider + if self._restart_share_instance(): + logging.info("Verifying manila after restarting share instance") + # Read the previous testing file from instance #1 + self._mount_share_on_instance( + fip_1, ssh_user_name, privkey, share_path) + self._validate_testing_file_from_instance( + fip_1, ssh_user_name, privkey) + # Read the previous testing file from instance #1 + self._mount_share_on_instance( + fip_2, ssh_user_name, privkey, share_path) + # Reset the test! + self._clear_testing_file_on_instance( + fip_1, ssh_user_name, privkey + ) + # Write a testing file on instance #1 + self._write_testing_file_on_instance( + fip_1, ssh_user_name, privkey) + # Validate the testing file from instance #2 + self._validate_testing_file_from_instance( + fip_2, ssh_user_name, privkey) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index f5d7b63..2e9a186 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -16,11 +16,14 @@ """Encapsulate Manila Ganesha testing.""" +import logging + from zaza.openstack.charm_tests.manila_ganesha.setup import ( MANILA_GANESHA_TYPE_NAME, ) import zaza.openstack.charm_tests.manila.tests as manila_tests +import zaza.model class ManilaGaneshaTests(manila_tests.ManilaBaseTest): @@ -33,3 +36,19 @@ class ManilaGaneshaTests(manila_tests.ManilaBaseTest): cls.share_name = 'cephnfsshare1' cls.share_type_name = MANILA_GANESHA_TYPE_NAME cls.share_protocol = 'nfs' + + def _restart_share_instance(self): + logging.info('Restarting manila-share and nfs-ganesha') + # It would be better for thie to derive the application name, + # manila-ganesha-az1, from deployed instances fo the manila-ganesha + # charm; however, that functionality isn't present yet in zaza, so + # this is hard coded to the application name used in that charm's + # test bundles. + for unit in zaza.model.get_units('manila-ganesha-az1'): + # While we really only need to run this on the machine hosting + # nfs-ganesha and manila-share, running it everywhere isn't + # harmful. Pacemaker handles restarting the services + zaza.model.run_on_unit( + unit.entity_id, + "systemctl stop manila-share nfs-ganesha") + return True From acd179374179261e2675d8fc9e1f8e6efcf191f4 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 5 Jul 2021 11:41:45 +0200 Subject: [PATCH 896/898] Fix neutron-api-plugin-ovn's num-expected-units After zaza#451 Zaza's application readiness detection has been improved, which now exposes issues from the past that were hidden. In other words, we were relying on a bug on Zaza's side and need to get this straight. --- zaza/openstack/charm_tests/ovn/tests.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index bfa0f21..de8e938 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -263,8 +263,27 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): 'neutron-api-plugin-ovn', 'neutron-plugin', 'neutron-api:neutron-plugin-api-subordinate') zaza.model.wait_for_agent_status() + + # NOTE(lourot): usually in this scenario, the test bundle has been + # originally deployed with a non-related neutron-api-plugin-ovn + # subordinate application, and thus Zaza has been taught to expect + # initially no unit from this application. We are now relating it + # to a principal neutron-api application with one unit. Thus we now + # need to make sure we wait for one unit from this subordinate + # before proceeding: + target_deploy_status = self.test_config.get('target_deploy_status', + {}) + try: + target_deploy_status['neutron-api-plugin-ovn'][ + 'num-expected-units'] = 1 + except KeyError: + # num-expected-units wasn't set to 0, no expectation to be + # fixed, let's move on. + pass + zaza.model.wait_for_application_states( - states=self.test_config.get('target_deploy_status', {})) + states=target_deploy_status) + except juju.errors.JujuAPIError: # we were not able to add the relation, let's make sure it's # because it's already there From 13eb8006826e63a2e815305af31efca1422ebe89 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 5 Jul 2021 11:39:08 +0200 Subject: [PATCH 897/898] octavia: Add version fence for adding the LBAAS role --- zaza/openstack/charm_tests/octavia/tests.py | 17 +++++++++++------ zaza/openstack/charm_tests/policyd/tests.py | 20 ++++++++++++-------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 6208397..295bd51 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -136,9 +136,11 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): cls.keystone_client = ObjectRetrierWraps( openstack_utils.get_keystone_session_client(cls.keystone_session)) - # add role to admin user for the duration of the test - grant_role_current_user(cls.keystone_client, cls.keystone_session, - LBAAS_ADMIN_ROLE) + if (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('focal_wallaby')): + # add role to admin user for the duration of the test + grant_role_current_user(cls.keystone_client, cls.keystone_session, + LBAAS_ADMIN_ROLE) cls.neutron_client = ObjectRetrierWraps( openstack_utils.get_neutron_session_client(cls.keystone_session)) @@ -206,9 +208,12 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): # allow resource cleanup to be run multiple times self.loadbalancers = [] - # revoke role from admin user added by this test - revoke_role_current_user(self.keystone_client, self.keystone_session, - LBAAS_ADMIN_ROLE) + if (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('focal_wallaby')): + # revoke role from admin user added by this test + revoke_role_current_user(self.keystone_client, + self.keystone_session, + LBAAS_ADMIN_ROLE) for fip in self.fips: self.neutron_client.delete_floatingip(fip) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 87fd677..396f46e 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -679,17 +679,21 @@ class OctaviaTests(BasePolicydSpecialization): cls.keystone_client = ObjectRetrierWraps( openstack_utils.get_keystone_session_client(cls.keystone_session)) - # add role to admin user for the duration of the test - octavia_tests.grant_role_current_user( - cls.keystone_client, cls.keystone_session, - octavia_tests.LBAAS_ADMIN_ROLE) + if (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('focal_wallaby')): + # add role to admin user for the duration of the test + octavia_tests.grant_role_current_user( + cls.keystone_client, cls.keystone_session, + octavia_tests.LBAAS_ADMIN_ROLE) def resource_cleanup(self): """Restore changes made by test.""" - # revoke role from admin user added by this test - octavia_tests.revoke_role_current_user( - self.keystone_client, self.keystone_session, - octavia_tests.LBAAS_ADMIN_ROLE) + if (openstack_utils.get_os_release() >= + openstack_utils.get_os_release('focal_wallaby')): + # revoke role from admin user added by this test + octavia_tests.revoke_role_current_user( + self.keystone_client, self.keystone_session, + octavia_tests.LBAAS_ADMIN_ROLE) def get_client_and_attempt_operation(self, ip): """Attempt to list available provider drivers. From ecf5bca2687fbcbab508258e925d6c710c7ed6fd Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 7 Jul 2021 11:08:53 +0200 Subject: [PATCH 898/898] ovn: Add prefer-chassis-as-gw test --- zaza/openstack/charm_tests/ovn/tests.py | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index de8e938..2478d66 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -139,6 +139,35 @@ class ChassisCharmOperationTest(BaseCharmOperationTest): 'ovs-vswitchd', ] + def test_prefer_chassis_as_gw(self): + """Confirm effect of prefer-chassis-as-gw configuration option.""" + expected_key = 'external-ids:ovn-cms-options' + expected_value = 'enable-chassis-as-gw' + with self.config_change( + {}, {'prefer-chassis-as-gw': True}, + reset_to_charm_default=True): + for unit in zaza.model.get_units(self.application_name): + self.assertEqual( + zaza.model.run_on_unit( + unit.entity_id, + 'ovs-vsctl get open-vswitch . {}'.format(expected_key) + )['Stdout'].rstrip(), + expected_value) + logging.info( + '{}: "{}" set to "{}"' + .format(unit.entity_id, expected_key, expected_value)) + logging.info('Config restored, checking things went back to normal') + for unit in zaza.model.get_units(self.application_name): + self.assertEqual( + zaza.model.run_on_unit( + unit.entity_id, + 'ovs-vsctl get open-vswitch . ' + 'external-ids:ovn-cms-options')['Code'], + '1') + logging.info( + '{}: "{}" no longer present' + .format(unit.entity_id, expected_key)) + class OVSOVNMigrationTest(test_utils.BaseCharmTest): """OVS to OVN migration tests."""