From fd3f0bb09355d73574cb83700a7d8381e981260f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 21 Apr 2020 15:39:48 +0100 Subject: [PATCH 01/11] Migrate 499 cinder-ceph test to zaza This takes the 499 test from basic_deployment.py (Amulet) test and ports it over to zaza. --- .../charm_tests/ceph/mon/__init__.py | 15 ++++ zaza/openstack/charm_tests/ceph/mon/tests.py | 69 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 zaza/openstack/charm_tests/ceph/mon/__init__.py create mode 100644 zaza/openstack/charm_tests/ceph/mon/tests.py diff --git a/zaza/openstack/charm_tests/ceph/mon/__init__.py b/zaza/openstack/charm_tests/ceph/mon/__init__.py new file mode 100644 index 0000000..867c3af --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/mon/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2018 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ceph-mon for cinder-ceph.""" diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py new file mode 100644 index 0000000..50063c7 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -0,0 +1,69 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Ceph-mon Testing for cinder-ceph.""" + +import logging +import unittest + +import zaza.model + +from zaza.openstack.utilities import ( + generic as generic_utils, + openstack as openstack_utils, +) + + +class CinderCephMonTest(unittest.TestCase): + """Verify that the ceph mon units are healthy.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph security tests.""" + super().setUpClass() + + # ported from the cinder-ceph Amulet test + def test_499_ceph_cmds_exit_zero(self): + """Verify expected state with security-checklist.""" + logging.info("Checking exit values are 0 on ceph commands.") + + units = zaza.model.get_units("ceph-mon", model_name=self.model_name) + current_release = openstack_utils.get_os_release() + bionic_train = openstack_utils.get_os_release('bionic_train') + if current_release < bionic_train: + units.extend(zaza.model.get_utils("cinder-ceph", + model_name=self.model_name)) + + commands = [ + 'sudo ceph health', + 'sudo ceph mds stat', + 'sudo ceph pg stat', + 'sudo ceph osd stat', + 'sudo ceph mon stat', + ] + + for unit in units: + run_commands(unit.name, commands) + + +def run_commands(self, unit_name, commands): + """Run commands on unit. + + Apply context to commands until all variables have been replaced, then + run the command on the given unit. + """ + for cmd in commands: + generic_utils.assertRemoteRunOK(zaza.model.run_on_unit( + unit_name, + cmd)) From c6f047268139b418c2e6f24b054daab57b9e0f18 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 4 May 2020 18:23:07 +0100 Subject: [PATCH 02/11] Fix the broken bits in the tests copied from amulet --- zaza/openstack/charm_tests/ceph/mon/tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index 50063c7..276ed9f 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -23,14 +23,15 @@ from zaza.openstack.utilities import ( generic as generic_utils, openstack as openstack_utils, ) +import zaza.openstack.charm_tests.test_utils as test_utils -class CinderCephMonTest(unittest.TestCase): +class CinderCephMonTest(test_utils.OpenStackBaseTest): """Verify that the ceph mon units are healthy.""" @classmethod def setUpClass(cls): - """Run class setup for running ceph security tests.""" + """Run class setup for running ceph mon tests with cinder.""" super().setUpClass() # ported from the cinder-ceph Amulet test @@ -57,7 +58,7 @@ class CinderCephMonTest(unittest.TestCase): run_commands(unit.name, commands) -def run_commands(self, unit_name, commands): +def run_commands(unit_name, commands): """Run commands on unit. Apply context to commands until all variables have been replaced, then From 294fd83e498cfaeefe8c92913626271ed6fcb162 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 4 May 2020 18:23:34 +0100 Subject: [PATCH 03/11] Fix up cinder tests to work when cinder isn't the application under test The cinder tests were written with the assumption that cinder was the charm that was under test. This modifies the test so that the cinder tests work in a model with cinder where cinder isn't the application that is being explicitly tested. --- zaza/openstack/charm_tests/cinder/tests.py | 68 ++++++++++++++++++---- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index d645eed..460ff00 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -32,7 +32,10 @@ class CinderTests(test_utils.OpenStackBaseTest): @classmethod def setUpClass(cls): """Run class setup for running tests.""" - super(CinderTests, cls).setUpClass() + super(CinderTests, cls).setUpClass(application_name='cinder') + cls.application_name = 'cinder' + cls.lead_unit = zaza.model.get_lead_unit_name( + "cinder", model_name=cls.model_name) cls.cinder_client = openstack_utils.get_cinder_session_client( cls.keystone_session) cls.nova_client = openstack_utils.get_nova_session_client( @@ -42,18 +45,61 @@ class CinderTests(test_utils.OpenStackBaseTest): def tearDown(cls): """Remove test resources.""" logging.info('Running teardown') - for snapshot in cls.cinder_client.volume_snapshots.list(): + volumes = list(cls.cinder_client.volumes.list()) + snapped_volumes = [v for v in volumes if v.name.endswith("-from-snap")] + if snapped_volumes: + logging.info("Removing volumes from snapshot") + cls._remove_volumes(snapped_volumes) + volumes = list(cls.cinder_client.volumes.list()) + + snapshots = list(cls.cinder_client.volume_snapshots.list()) + if snapshots: + logging.info("tearDown - snapshots: {}".format( + ", ".join(s.name for s in snapshots))) + cls._remove_snapshots(snapshots) + + if volumes: + logging.info("tearDown - volumes: {}".format( + ", ".join(v.name for v in volumes))) + cls._remove_volumes(volumes) + + @classmethod + def _remove_snapshots(cls, snapshots): + """Remove snapshots passed as param. + + :param volumes: the snapshots to delete + :type volumes: List[snapshot objects] + """ + for snapshot in snapshots: if snapshot.name.startswith(cls.RESOURCE_PREFIX): - openstack_utils.delete_resource( - cls.cinder_client.volume_snapshots, - snapshot.id, - msg="snapshot") - for volume in cls.cinder_client.volumes.list(): + logging.info("removing snapshot: {}".format(snapshot.name)) + try: + openstack_utils.delete_resource( + cls.cinder_client.volume_snapshots, + snapshot.id, + msg="snapshot") + except Exception as e: + logging.error("error removing snapshot: {}".format(str(e))) + raise + + @classmethod + def _remove_volumes(cls, volumes): + """Remove volumes passed as param. + + :param volumes: the volumes to delete + :type volumes: List[volume objects] + """ + for volume in volumes: if volume.name.startswith(cls.RESOURCE_PREFIX): - openstack_utils.delete_resource( - cls.cinder_client.volumes, - volume.id, - msg="volume") + logging.info("removing volume: {}".format(volume.name)) + try: + openstack_utils.delete_resource( + cls.cinder_client.volumes, + volume.id, + msg="volume") + except Exception as e: + logging.error("error removing volume: {}".format(str(e))) + raise def test_100_volume_create_extend_delete(self): """Test creating, extending a volume.""" From 3de62513161efbb0b1ea831628bff15764086a59 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 5 May 2020 15:07:44 +0100 Subject: [PATCH 04/11] Fix broken code and collect errors on ceph commands --- zaza/openstack/charm_tests/ceph/mon/tests.py | 16 ++++++++++++---- zaza/openstack/utilities/exceptions.py | 6 ++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index 276ed9f..0fea084 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -22,6 +22,7 @@ import zaza.model from zaza.openstack.utilities import ( generic as generic_utils, openstack as openstack_utils, + exceptions as zaza_exceptions ) import zaza.openstack.charm_tests.test_utils as test_utils @@ -43,7 +44,7 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): current_release = openstack_utils.get_os_release() bionic_train = openstack_utils.get_os_release('bionic_train') if current_release < bionic_train: - units.extend(zaza.model.get_utils("cinder-ceph", + units.extend(zaza.model.get_units("cinder-ceph", model_name=self.model_name)) commands = [ @@ -64,7 +65,14 @@ def run_commands(unit_name, commands): Apply context to commands until all variables have been replaced, then run the command on the given unit. """ + errors = [] for cmd in commands: - generic_utils.assertRemoteRunOK(zaza.model.run_on_unit( - unit_name, - cmd)) + try: + generic_utils.assertRemoteRunOK(zaza.model.run_on_unit( + unit_name, + cmd)) + except Exception as e: + errors.append("unit: {}, command: {}, error: {}" + .format(unit_name, cmd, str(e))) + if errors: + raise zaza_exceptions.CephGenericError("\n".join(errors)) diff --git a/zaza/openstack/utilities/exceptions.py b/zaza/openstack/utilities/exceptions.py index e9e5d23..f8673a1 100644 --- a/zaza/openstack/utilities/exceptions.py +++ b/zaza/openstack/utilities/exceptions.py @@ -168,6 +168,12 @@ class CephPoolNotConfigured(Exception): pass +class CephGenericError(Exception): + """A generic/other Ceph error occurred.""" + + pass + + class NovaGuestMigrationFailed(Exception): """Nova guest migration failed.""" From 935af95dd23c38a4d8d570482498c6f72f649be6 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 6 May 2020 12:09:22 +0100 Subject: [PATCH 05/11] Add in missing 500 test from amulet test This is the missing 500 test from the previous amulet tests that were missed on the first pass. --- zaza/openstack/charm_tests/ceph/mon/tests.py | 66 +++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index 0fea084..39cf4a2 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -15,7 +15,6 @@ """Ceph-mon Testing for cinder-ceph.""" import logging -import unittest import zaza.model @@ -58,6 +57,57 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): for unit in units: run_commands(unit.name, commands) + # ported from the cinder-ceph Amulet test + def test_500_ceph_alternatives_cleanup(self): + """Check ceph alternatives removed when ceph-mon relation is broken.""" + # Skip this test if release is less than xenial_ocata as in that case + # cinder HAS a relation with ceph directly and this test would fail + current_release = openstack_utils.get_os_release() + xenial_ocata = openstack_utils.get_os_release('xenial_ocata') + if current_release < xenial_ocata: + logging.info("Skipping test as release < xenial-ocata") + return + + units = zaza.model.get_units("cinder-ceph", + model_name=self.model_name) + + # check each unit prior to breaking relation + for unit in units: + dir_list = directory_listing(unit.name, "/etc/ceph") + if 'ceph.conf' in dir_list: + logging.debug( + "/etc/ceph/ceph.conf exists BEFORE relation-broken") + else: + raise zaza_exceptions.CephGenericError( + "unit: {} - /etc/ceph/ceph.conf does not exist " + "BEFORE relation-broken".format(unit.name)) + + # remove the relation so that /etc/ceph/ceph.conf is removed + logging.info("Removing ceph-mon:client <-> cinder-ceph:ceph relation") + zaza.model.remove_relation( + "ceph-mon", "ceph-mon:client" "cinder-ceph:ceph") + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # check each unit after to breaking relation + for unit in units: + dir_list = directory_listing(unit.name, "/etc/ceph") + if 'ceph.conf' not in dir_list: + logging.debug( + "/etc/ceph/ceph.conf removed AFTER relation-broken") + else: + raise zaza_exceptions.CephGenericError( + "unit: {} - /etc/ceph/ceph.conf still exists " + "AFTER relation-broken".format(unit.name)) + + # Restore cinder-ceph and ceph-mon relation to keep tests idempotent + logging.info("Restoring ceph-mon:client <-> cinder-ceph:ceph relation") + zaza.model.add_relation( + "ceph-mon", "ceph-mon:client" "cinder-ceph:ceph") + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + logging.info("... Done.") + def run_commands(unit_name, commands): """Run commands on unit. @@ -76,3 +126,17 @@ def run_commands(unit_name, commands): .format(unit_name, cmd, str(e))) if errors: raise zaza_exceptions.CephGenericError("\n".join(errors)) + + +def directory_listing(unit_name, directory): + """Return a list of files/directories from a directory on a unit. + + :param unit_name: the unit to fetch the directory listing from + :type unit_name: str + :param directory: the directory to fetch the listing from + :type directory: str + :returns: A listing using "ls -1" on the unit + :rtype: List[str] + """ + result = zaza.model.run_on_unit(unit_name, "ls -1 {}".format(directory)) + return result['Stdout'].splitlines() From a7df335cc84cc5fb175cb4c8df00faf7c38d838c Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 6 May 2020 15:31:53 +0100 Subject: [PATCH 06/11] Fix missing comma --- zaza/openstack/charm_tests/ceph/mon/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index 39cf4a2..dd6d7aa 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -85,7 +85,7 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): # remove the relation so that /etc/ceph/ceph.conf is removed logging.info("Removing ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.remove_relation( - "ceph-mon", "ceph-mon:client" "cinder-ceph:ceph") + "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() @@ -103,7 +103,7 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): # Restore cinder-ceph and ceph-mon relation to keep tests idempotent logging.info("Restoring ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.add_relation( - "ceph-mon", "ceph-mon:client" "cinder-ceph:ceph") + "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() logging.info("... Done.") From 8e826f6f8bc277b75b962b2516238ae1a80a3a83 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 13 May 2020 07:57:24 +0100 Subject: [PATCH 07/11] Add debug to 105 test --- zaza/openstack/charm_tests/cinder/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 460ff00..ec6caf0 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -126,12 +126,18 @@ class CinderTests(test_utils.OpenStackBaseTest): def test_105_volume_create_from_img(self): """Test creating a volume from an image.""" + logging.debug("finding image {} ..." + .format(glance_setup.LTS_IMAGE_NAME) image = self.nova_client.glance.find_image( glance_setup.LTS_IMAGE_NAME) + logging.debug("using cinder_client to create volume from image {}" + .format(image.id)) vol_img = self.cinder_client.volumes.create( name='{}-105-vol-from-img'.format(self.RESOURCE_PREFIX), size=3, imageRef=image.id) + logging.debug("now waiting for volume {} to reach available" + .format(vol_img.id)) openstack_utils.resource_reaches_status( self.cinder_client.volumes, vol_img.id, From 4c2f723332e2bfa5e67e80741c606e845050895c Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 13 May 2020 14:34:13 +0100 Subject: [PATCH 08/11] Fix missing brace --- zaza/openstack/charm_tests/cinder/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index ec6caf0..4a851ce 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -127,7 +127,7 @@ class CinderTests(test_utils.OpenStackBaseTest): def test_105_volume_create_from_img(self): """Test creating a volume from an image.""" logging.debug("finding image {} ..." - .format(glance_setup.LTS_IMAGE_NAME) + .format(glance_setup.LTS_IMAGE_NAME)) image = self.nova_client.glance.find_image( glance_setup.LTS_IMAGE_NAME) logging.debug("using cinder_client to create volume from image {}" From 5d05eb70ccc3d71613b5326a6a28407175498d8d Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 14 May 2020 19:37:45 +0100 Subject: [PATCH 09/11] Ensure model starts executing before the wait There was a race before a block_until_all_units_idle() where it could blast through the check before it started removing the relation. This ensures that it waits until something happens and then waits for it to finish. --- zaza/openstack/charm_tests/ceph/mon/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index dd6d7aa..63c4033 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -86,6 +86,7 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): logging.info("Removing ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.remove_relation( "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") + zaza.model.wait_for_agent_status() logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() @@ -104,6 +105,7 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): logging.info("Restoring ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.add_relation( "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") + zaza.model.wait_for_agent_status() logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() logging.info("... Done.") From 5e5c8e488bda1ae2d50b1c9f01ed3fe42f6d02cd Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 18 May 2020 12:52:19 +0100 Subject: [PATCH 10/11] Add retries to teardown as it seems to be sensitive to network glitches. --- zaza/openstack/charm_tests/cinder/tests.py | 41 ++++++++++++++-------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 4a851ce..2648538 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -23,6 +23,13 @@ import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.glance.setup as glance_setup +from tenacity import ( + RetryError, + Retrying, + stop_after_attempt, + wait_exponential, +) + class CinderTests(test_utils.OpenStackBaseTest): """Encapsulate Cinder tests.""" @@ -45,23 +52,27 @@ class CinderTests(test_utils.OpenStackBaseTest): def tearDown(cls): """Remove test resources.""" logging.info('Running teardown') - volumes = list(cls.cinder_client.volumes.list()) - snapped_volumes = [v for v in volumes if v.name.endswith("-from-snap")] - if snapped_volumes: - logging.info("Removing volumes from snapshot") - cls._remove_volumes(snapped_volumes) - volumes = list(cls.cinder_client.volumes.list()) + for attempt in Retrying( + stop=stop_after_attempt(8), + wait=wait_exponential(multiplier=1, min=2, max=60)): + with attempt: + volumes = list(cls.cinder_client.volumes.list()) + snapped_volumes = [v for v in volumes if v.name.endswith("-from-snap")] + if snapped_volumes: + logging.info("Removing volumes from snapshot") + cls._remove_volumes(snapped_volumes) + volumes = list(cls.cinder_client.volumes.list()) - snapshots = list(cls.cinder_client.volume_snapshots.list()) - if snapshots: - logging.info("tearDown - snapshots: {}".format( - ", ".join(s.name for s in snapshots))) - cls._remove_snapshots(snapshots) + snapshots = list(cls.cinder_client.volume_snapshots.list()) + if snapshots: + logging.info("tearDown - snapshots: {}".format( + ", ".join(s.name for s in snapshots))) + cls._remove_snapshots(snapshots) - if volumes: - logging.info("tearDown - volumes: {}".format( - ", ".join(v.name for v in volumes))) - cls._remove_volumes(volumes) + if volumes: + logging.info("tearDown - volumes: {}".format( + ", ".join(v.name for v in volumes))) + cls._remove_volumes(volumes) @classmethod def _remove_snapshots(cls, snapshots): From ec40306f7b4c94409d52ce9c6d8104ede0ee7d23 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 19 May 2020 20:08:33 +0100 Subject: [PATCH 11/11] Improve detection of removing and adding relation --- zaza/openstack/charm_tests/ceph/mon/tests.py | 66 ++++++++++++++++++-- zaza/openstack/charm_tests/cinder/tests.py | 4 +- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/mon/tests.py b/zaza/openstack/charm_tests/ceph/mon/tests.py index 63c4033..4258a36 100644 --- a/zaza/openstack/charm_tests/ceph/mon/tests.py +++ b/zaza/openstack/charm_tests/ceph/mon/tests.py @@ -86,11 +86,19 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): logging.info("Removing ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.remove_relation( "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") - zaza.model.wait_for_agent_status() - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() + # zaza.model.wait_for_agent_status() + logging.info("Wait till relation is removed...") + ceph_mon_units = zaza.model.get_units("ceph-mon", + model_name=self.model_name) + conditions = [ + invert_condition( + does_relation_exist( + u.name, "ceph-mon", "cinder-ceph", "ceph", + self.model_name)) + for u in ceph_mon_units] + zaza.model.block_until(*conditions) - # check each unit after to breaking relation + logging.info("Checking each unit after breaking relation...") for unit in units: dir_list = directory_listing(unit.name, "/etc/ceph") if 'ceph.conf' not in dir_list: @@ -105,12 +113,60 @@ class CinderCephMonTest(test_utils.OpenStackBaseTest): logging.info("Restoring ceph-mon:client <-> cinder-ceph:ceph relation") zaza.model.add_relation( "ceph-mon", "ceph-mon:client", "cinder-ceph:ceph") - zaza.model.wait_for_agent_status() + conditions = [ + does_relation_exist( + u.name, "ceph-mon", "cinder-ceph", "ceph", self.model_name) + for u in ceph_mon_units] logging.info("Wait till model is idle ...") + zaza.model.block_until(*conditions) zaza.model.block_until_all_units_idle() logging.info("... Done.") +def does_relation_exist(unit_name, + application_name, + remote_application_name, + remote_interface_name, + model_name): + """For use in async blocking function, return True if it exists. + + :param unit_name: the unit (by name) that to check on. + :type unit_name: str + :param application_name: Name of application on this side of relation + :type application_name: str + :param remote_application_name: the relation name at that unit to check for + :type relation_application_name: str + :param remote_interface_name: the interface name at that unit to check for + :type relation_interface_name: str + :param model_name: the model to check on + :type model_name: str + :returns: Corouting that returns True if the relation was found + :rtype: Coroutine[[], boolean] + """ + async def _async_does_relation_exist_closure(): + async with zaza.model.run_in_model(model_name) as model: + spec = "{}:{}".format( + remote_application_name, remote_interface_name) + for rel in model.applications[application_name].relations: + if rel.matches(spec): + return True + return False + return _async_does_relation_exist_closure + + +def invert_condition(async_condition): + """Invert the condition provided so it can be provided to the blocking fn. + + :param async_condition: the async callable that is the test + :type async_condition: Callable[] + :returns: Corouting that returns not of the result of a the callable + :rtype: Coroutine[[], bool] + """ + async def _async_invert_condition_closure(): + return not(await async_condition()) + return _async_invert_condition_closure + + def run_commands(unit_name, commands): """Run commands on unit. diff --git a/zaza/openstack/charm_tests/cinder/tests.py b/zaza/openstack/charm_tests/cinder/tests.py index 2648538..4cbf7c0 100644 --- a/zaza/openstack/charm_tests/cinder/tests.py +++ b/zaza/openstack/charm_tests/cinder/tests.py @@ -24,7 +24,6 @@ import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.glance.setup as glance_setup from tenacity import ( - RetryError, Retrying, stop_after_attempt, wait_exponential, @@ -57,7 +56,8 @@ class CinderTests(test_utils.OpenStackBaseTest): wait=wait_exponential(multiplier=1, min=2, max=60)): with attempt: volumes = list(cls.cinder_client.volumes.list()) - snapped_volumes = [v for v in volumes if v.name.endswith("-from-snap")] + snapped_volumes = [v for v in volumes + if v.name.endswith("-from-snap")] if snapped_volumes: logging.info("Removing volumes from snapshot") cls._remove_volumes(snapped_volumes)