From 14a6f0658665c5f2df443872ca9797fd789d4b2d Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 22 Apr 2020 17:32:33 +0000 Subject: [PATCH] Fix glance simplestreams sync test races We expect at least three images. Avoid various race conditions by retrying using tenacity. --- .../glance_simplestreams_sync/tests.py | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py index 6ddb1b3..648172a 100644 --- a/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/tests.py @@ -24,7 +24,7 @@ import zaza.openstack.utilities.openstack as openstack_utils @tenacity.retry( - retry=tenacity.retry_if_result(lambda images: len(images) == 0), + retry=tenacity.retry_if_result(lambda images: len(images) < 3), wait=tenacity.wait_fixed(6), # interval between retries stop=tenacity.stop_after_attempt(100)) # retry times def retry_image_sync(glance_client): @@ -33,6 +33,19 @@ def retry_image_sync(glance_client): return list(glance_client.images.list()) +@tenacity.retry( + retry=tenacity.retry_if_exception_type(json.decoder.JSONDecodeError), + wait=tenacity.wait_fixed(10), reraise=True, + stop=tenacity.stop_after_attempt(10)) +def get_product_streams(url): + """Get product streams json data with retry.""" + # There is a race between the images being available in glance and any + # metadata being written. Use tenacity to avoid this race. + client = requests.session() + json_data = client.get(url).text + return json.loads(json_data) + + class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): """Glance Simple Streams Sync Test.""" @@ -48,7 +61,7 @@ class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): cls.keystone_session) def test_010_wait_for_image_sync(self): - """Wait for images to be synced. Expect at least one.""" + """Wait for images to be synced. Expect at least three.""" self.assertTrue(retry_image_sync(self.glance_client)) def test_050_gss_permissions_regression_check_lp1611987(self): @@ -91,12 +104,21 @@ class GlanceSimpleStreamsSyncTest(test_utils.OpenStackBaseTest): catalog = self.keystone_client.service_catalog.get_endpoints() ps_interface = catalog["product-streams"][0][key] url = "{}/{}".format(ps_interface, uri) - client = requests.session() - json_data = client.get(url).text - product_streams = json.loads(json_data) - images = product_streams["products"] - for image in expected_images: - self.assertIn(image, images) + # There is a race between the images being available in glance and the + # metadata being written for each image. Use tenacity to avoid this + # race and make the test idempotent. + @tenacity.retry( + retry=tenacity.retry_if_exception_type(AssertionError), + wait=tenacity.wait_fixed(10), reraise=True, + stop=tenacity.stop_after_attempt(10)) + def _check_local_product_streams(url, expected_images): + product_streams = get_product_streams(url) + images = product_streams["products"] + + for image in expected_images: + self.assertIn(image, images) + + _check_local_product_streams(url, expected_images) logging.debug("Local product stream successful")