Merge pull request #979 from ajkavanagh/bug/2002156/rabbitmq-intermittent-tests

Fix race in test 408 for rabbitmq

pip.sh

@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-pip install pip==20.2.3
-pip "$@"

requirements.txt

@@ -11,8 +11,8 @@ async_generator
 pyopenssl<22.1.0
 
 boto3<1.25
-PyYAML<=4.2,>=3.0; python_version < '3.10'
-PyYAML>=5.1; python_version >= '3.10'
+PyYAML<=4.2,>=3.0; python_version < '3.9'
+PyYAML>=5.1; python_version >= '3.9'
 flake8>=2.2.4
 flake8-docstrings
 flake8-per-file-ignores
@@ -21,7 +21,7 @@ coverage<6.0.0  # coverage 6.0+ drops support for py3.5/py2.7
 mock>=1.2
 pytest
 pytest-cov
-pbr>=1.8.0,<1.9.0
+pbr>=1.8.0
 simplejson>=2.2.0
 netifaces>=0.10.4
 netaddr>=0.7.12,!=0.7.16

tox.ini

@@ -5,51 +5,19 @@ skipsdist = True
 sitepackages = False
 # NOTE: Avoid false positives by not skipping missing interpreters.
 skip_missing_interpreters = False
-# NOTES:
-# * We avoid the new dependency resolver by pinning pip < 20.3, see
-#   https://github.com/pypa/pip/issues/9187
-# * Pinning dependencies requires tox >= 3.2.0, see
-#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
-# * It is also necessary to pin virtualenv as a newer virtualenv would still
-#   lead to fetching the latest pip in the func* tox targets, see
-#   https://stackoverflow.com/a/38133283
-requires = pip < 20.3
-           virtualenv < 20.0
-# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
-minversion = 3.2.0
+ignore_basepython_conflict = True
 
 [testenv]
+basepython = python3
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
-install_command =
-  {toxinidir}/pip.sh install {opts} {packages}
-
 commands = pytest --cov=zaza.openstack {posargs} {toxinidir}/unit_tests
-
-[testenv:py3]
-basepython = python3
-deps = -r{toxinidir}/requirements.txt
-
-[testenv:py3.8]
-basepython = python3.8
-deps = -r{toxinidir}/requirements.txt
-
-[testenv:py3.9]
-basepython = python3.9
-deps = -r{toxinidir}/requirements.txt
-
-[testenv:py3.10]
-basepython = python3.10
 deps = -r{toxinidir}/requirements.txt
 
 [testenv:pep8]
-basepython = python3
-deps = -r{toxinidir}/requirements.txt
 commands = flake8 {posargs} zaza unit_tests
 
 [testenv:venv]
-basepython = python3
-deps = -r{toxinidir}/requirements.txt
 commands = /bin/true
 
 [flake8]
@@ -58,8 +26,5 @@ per-file-ignores =
   unit_tests/**: D
 
 [testenv:docs]
-basepython = python3
 changedir = doc/source
-deps =
-    -r{toxinidir}/requirements.txt
 commands = sphinx-build -W -b html -d {toxinidir}/doc/build/doctrees . {toxinidir}/doc/build/html

unit_tests/utilities/test_zaza_utilities_series_upgrade.py

@@ -77,7 +77,7 @@ class TestSeriesUpgrade(ut_utils.BaseTestCase):
             _unit, _machine_num, origin=_origin,
             to_series=_to_series, from_series=_from_series,
             workaround_script=_workaround_script, files=_files)
-        self.block_until_all_units_idle.called_with()
+        self.block_until_all_units_idle.assert_called()
         self.prepare_series_upgrade.assert_called_once_with(
             _machine_num, to_series=_to_series)
         self.wrap_do_release_upgrade.assert_called_once_with(

zaza/openstack/charm_tests/rabbitmq_server/utils.py

@@ -427,17 +427,25 @@ def connect_amqp_by_unit(unit, ssl=False,
                   '{}...'.format(host, port, unit_name, username))
 
     try:
-        credentials = pika.PlainCredentials(username, password)
-        parameters = pika.ConnectionParameters(host=host, port=port,
-                                               credentials=credentials,
-                                               ssl_options=ssl_options,
-                                               connection_attempts=3,
-                                               retry_delay=5,
-                                               socket_timeout=1)
-        connection = pika.BlockingConnection(parameters)
-        assert connection.is_open is True
-        logging.debug('Connect OK')
-        return connection
+        # retry connections; it's possible during the testing that a
+        # leader-setting-change hook will be running on the unit (which takes
+        # up to 30s to run) and results in a restart of the underlying rabbitmq
+        # process.  This retry get's past the restart.
+        for attempt in tenacity.Retrying(
+                stop=tenacity.stop_after_attempt(5),
+                wait=tenacity.wait_exponential(multiplier=1, min=2, max=10)):
+            with attempt:
+                credentials = pika.PlainCredentials(username, password)
+                parameters = pika.ConnectionParameters(host=host, port=port,
+                                                       credentials=credentials,
+                                                       ssl_options=ssl_options,
+                                                       connection_attempts=3,
+                                                       retry_delay=5,
+                                                       socket_timeout=1)
+                connection = pika.BlockingConnection(parameters)
+                assert connection.is_open is True
+                logging.debug('Connect OK')
+                return connection
     except Exception as e:
         msg = ('amqp connection failed to {}:{} as '
                '{} ({})'.format(host, port, username, str(e)))