From 22763491b33484b288dc34f4d72677142acc3d8f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 1 Apr 2019 12:26:02 +0200 Subject: [PATCH 1/4] ceph/rbd_mirror: Add use of ``resync-pools`` action to test --- zaza/charm_tests/ceph/rbd_mirror/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zaza/charm_tests/ceph/rbd_mirror/tests.py b/zaza/charm_tests/ceph/rbd_mirror/tests.py index a90dd50..ac0cb47 100644 --- a/zaza/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/charm_tests/ceph/rbd_mirror/tests.py @@ -293,6 +293,14 @@ class CephRBDMirrorControlledFailoverTest(CephRBDMirrorBase): self.wait_for_mirror_state( 'up+stopped', model_name=self.site_a_model) + result = zaza.model.run_action_on_leader( + 'ceph-rbd-mirror' + self.site_b_app_suffix, + 'resync-pools', + model_name=self.site_b_model, + action_params={ + 'i-really-mean-it': True, + }) + logging.info(result.results) self.wait_for_mirror_state( 'up+replaying', application_name=self.application_name + self.site_b_app_suffix, From b7807fc5129e76f9ba0841e6a05f3f2b91b6d77d Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 1 Apr 2019 16:44:41 +0200 Subject: [PATCH 2/4] ceph/rbd_mirror: Optionally require presence of images in wait-helper When you tell Ceph to resync mirrored RBD images it will in practice remove and re-create the image. At present the image state wait helper will happilly accept no images in a pool as a positive outcome. Add optional ``require_images_in`` parameter that allows the wait helper to block even when no images are available in the pool (yet) --- zaza/charm_tests/ceph/rbd_mirror/tests.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/zaza/charm_tests/ceph/rbd_mirror/tests.py b/zaza/charm_tests/ceph/rbd_mirror/tests.py index ac0cb47..b108a62 100644 --- a/zaza/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/charm_tests/ceph/rbd_mirror/tests.py @@ -68,7 +68,8 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): def wait_for_mirror_state(self, state, application_name=None, model_name=None, - check_entries_behind_master=False): + check_entries_behind_master=False, + require_images_in=[]): """Wait until all images reach requested state. This function runs the ``status`` action and examines the data it @@ -85,6 +86,8 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): when used with state ``up+replying``. :type check_entries_behind_master: bool + :param require_images_in: List of pools to require images in + :type require_images_in: list of str :returns: True on success, never returns on failure """ rep = re.compile(r'.*entries_behind_master=(\d+)') @@ -92,7 +95,10 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): pool_status = self.run_status_action( application_name=application_name, model_name=model_name) for pool, status in pool_status.items(): - for image in status.get('images', []): + images = status.get('images', []) + if not len(images) and pool in require_images_in: + break + for image in images: if image['state'] and image['state'] != state: break if check_entries_behind_master: @@ -304,7 +310,8 @@ class CephRBDMirrorControlledFailoverTest(CephRBDMirrorBase): self.wait_for_mirror_state( 'up+replaying', application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model) + model_name=self.site_b_model, + require_images_in['cinder-ceph', 'glance']) class CephRBDMirrorDisasterFailoverTest(CephRBDMirrorBase): From ee06037050dac5f0b8592428421b88ccccf4f37d Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Mon, 1 Apr 2019 17:01:13 +0200 Subject: [PATCH 3/4] Add missing ``=`` --- zaza/charm_tests/ceph/rbd_mirror/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/charm_tests/ceph/rbd_mirror/tests.py b/zaza/charm_tests/ceph/rbd_mirror/tests.py index b108a62..65000ae 100644 --- a/zaza/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/charm_tests/ceph/rbd_mirror/tests.py @@ -311,7 +311,7 @@ class CephRBDMirrorControlledFailoverTest(CephRBDMirrorBase): 'up+replaying', application_name=self.application_name + self.site_b_app_suffix, model_name=self.site_b_model, - require_images_in['cinder-ceph', 'glance']) + require_images_in=['cinder-ceph', 'glance']) class CephRBDMirrorDisasterFailoverTest(CephRBDMirrorBase): From f38da846da38adc7ddef5775ca06574015e25c1e Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 2 Apr 2019 08:27:44 +0200 Subject: [PATCH 4/4] ceph/rbd_mirror: Encapsulate call to status action in try except The call to status action in ``wait_for_mirror_state`` helper is susceptible to Ceph bug LP: #1820976. Encapsulate in try except clause to work around it. --- zaza/charm_tests/ceph/rbd_mirror/tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zaza/charm_tests/ceph/rbd_mirror/tests.py b/zaza/charm_tests/ceph/rbd_mirror/tests.py index 65000ae..47e2e7e 100644 --- a/zaza/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/charm_tests/ceph/rbd_mirror/tests.py @@ -92,8 +92,12 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): """ rep = re.compile(r'.*entries_behind_master=(\d+)') while True: - pool_status = self.run_status_action( - application_name=application_name, model_name=model_name) + try: + # encapsulate in try except to work around LP: #1820976 + pool_status = self.run_status_action( + application_name=application_name, model_name=model_name) + except KeyError: + continue for pool, status in pool_status.items(): images = status.get('images', []) if not len(images) and pool in require_images_in: