From d670a2914c5014dca1f05ab82e679a7e8384bafc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 3 Oct 2023 17:04:06 +0000 Subject: [PATCH] Add tactical work around for traefik Add tactical work around for traefik readyness. Traefik takes sometime to process changes to load balancer configuration. Traefik has an api but it is turned off by default and currently the charm does not expose a way to enable it. So, this change looks at the traefik config on disk to check if it has the correct number if units. --- zaza/openstack/charm_tests/tempest/tests.py | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/zaza/openstack/charm_tests/tempest/tests.py b/zaza/openstack/charm_tests/tempest/tests.py index a7b063d..3bf9696 100644 --- a/zaza/openstack/charm_tests/tempest/tests.py +++ b/zaza/openstack/charm_tests/tempest/tests.py @@ -17,6 +17,7 @@ import logging import os import subprocess +import yaml import zaza import zaza.charm_lifecycle.utils @@ -184,6 +185,39 @@ class TempestTestScaleK8SBase(TempestTestBase): if ustatus.agent_status.life == 'dying'] assert len(dead_units) == dead_count + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), + reraise=True, stop=tenacity.stop_after_attempt(8)) + def wait_for_traefik(self, application_name): + """Wait for traefk to finish processing lb changes.""" + logging.warning( + "Waiting for traefik to process changes. This is a temporary " + "workaround and should be removed when there is a way to " + "determine when traefik has processed all requests") + units_count = len(zaza.model.get_units(application_name)) + container_cmd = ( + "cat /opt/traefik/juju/juju_ingress_ingress_*_{}.yaml").format( + application_name) + container_name = "traefik" + for unit in zaza.model.get_units("traefik"): + config = subprocess.check_output([ + "juju", + "ssh", + "-m", zaza.model.get_juju_model(), + "--container", + container_name, + unit.entity_id, + container_cmd]).decode() + service_config = yaml.safe_load(config) + loadBalancers = [ + lb['loadBalancer'] + for lb in service_config['http']['services'].values()] + assert len(loadBalancers) == 1 + unit_count_in_lb = len(loadBalancers[0]['servers']) + logging.info("Traefik LB server count: {} unit count: {}".format( + unit_count_in_lb, + units_count)) + assert unit_count_in_lb == units_count + def run(self): """Run tempest tests as specified in tests/tests.yaml. @@ -197,6 +231,7 @@ class TempestTestScaleK8SBase(TempestTestBase): render_tempest_config_keystone_v3 = tenacity.retry( wait=tenacity.wait_fixed(10), stop=tenacity.stop_after_attempt(3) )(tempest_utils.render_tempest_config_keystone_v3) + self.wait_for_traefik(self.application_name) zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints() render_tempest_config_keystone_v3(minimal=True) if not super().run(): @@ -208,6 +243,7 @@ class TempestTestScaleK8SBase(TempestTestBase): zaza.model.block_until_all_units_idle() logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.expected_statuses) + self.wait_for_traefik(self.application_name) zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints() render_tempest_config_keystone_v3(minimal=True) if not super().run(): @@ -222,6 +258,7 @@ class TempestTestScaleK8SBase(TempestTestBase): zaza.model.block_until_all_units_idle() logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.expected_statuses) + self.wait_for_traefik(self.application_name) zaza.openstack.charm_tests.keystone.setup.wait_for_all_endpoints() render_tempest_config_keystone_v3(minimal=True) return super().run()