From 9ad7e55a5f559790041271d2e5964dfb5f8bcc86 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 6 Jan 2023 20:12:08 +0000 Subject: [PATCH 1/6] Fix race in test 408 for rabbitmq There is a race in the 408 test for rabbitmq where the config-change to enable ssl causes a leader-settings-changed hook in the non-leader units which results in a rabbitmq service restart. This can happen at exactly the same time as the test attempts to establish a connection with the that unit. This patch retries the connection attempt. Note that this may only be a partial fix as it's possible that a restart will happen just after the connection is made, which would then result in a test failure. Related-Bug: LP#2002156 --- pip.sh | 4 -- tox.ini | 39 +------------------ .../charm_tests/rabbitmq_server/utils.py | 30 ++++++++------ 3 files changed, 21 insertions(+), 52 deletions(-) delete mode 100755 pip.sh diff --git a/pip.sh b/pip.sh deleted file mode 100755 index 8a71ce4..0000000 --- a/pip.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -pip install pip==20.2.3 -pip "$@" diff --git a/tox.ini b/tox.ini index c1e4a91..fbce34a 100644 --- a/tox.ini +++ b/tox.ini @@ -5,51 +5,19 @@ skipsdist = True sitepackages = False # NOTE: Avoid false positives by not skipping missing interpreters. skip_missing_interpreters = False -# NOTES: -# * We avoid the new dependency resolver by pinning pip < 20.3, see -# https://github.com/pypa/pip/issues/9187 -# * Pinning dependencies requires tox >= 3.2.0, see -# https://tox.readthedocs.io/en/latest/config.html#conf-requires -# * It is also necessary to pin virtualenv as a newer virtualenv would still -# lead to fetching the latest pip in the func* tox targets, see -# https://stackoverflow.com/a/38133283 -requires = pip < 20.3 - virtualenv < 20.0 -# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci -minversion = 3.2.0 +ignore_basepython_conflict = True [testenv] +basepython = python3 setenv = VIRTUAL_ENV={envdir} PYTHONHASHSEED=0 -install_command = - {toxinidir}/pip.sh install {opts} {packages} - commands = pytest --cov=zaza.openstack {posargs} {toxinidir}/unit_tests - -[testenv:py3] -basepython = python3 -deps = -r{toxinidir}/requirements.txt - -[testenv:py3.8] -basepython = python3.8 -deps = -r{toxinidir}/requirements.txt - -[testenv:py3.9] -basepython = python3.9 -deps = -r{toxinidir}/requirements.txt - -[testenv:py3.10] -basepython = python3.10 deps = -r{toxinidir}/requirements.txt [testenv:pep8] -basepython = python3 -deps = -r{toxinidir}/requirements.txt commands = flake8 {posargs} zaza unit_tests [testenv:venv] -basepython = python3 -deps = -r{toxinidir}/requirements.txt commands = /bin/true [flake8] @@ -58,8 +26,5 @@ per-file-ignores = unit_tests/**: D [testenv:docs] -basepython = python3 changedir = doc/source -deps = - -r{toxinidir}/requirements.txt commands = sphinx-build -W -b html -d {toxinidir}/doc/build/doctrees . {toxinidir}/doc/build/html diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index a0ced05..edbd951 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -427,17 +427,25 @@ def connect_amqp_by_unit(unit, ssl=False, '{}...'.format(host, port, unit_name, username)) try: - credentials = pika.PlainCredentials(username, password) - parameters = pika.ConnectionParameters(host=host, port=port, - credentials=credentials, - ssl_options=ssl_options, - connection_attempts=3, - retry_delay=5, - socket_timeout=1) - connection = pika.BlockingConnection(parameters) - assert connection.is_open is True - logging.debug('Connect OK') - return connection + # retry connections; it's possible during the testing that a + # leader-setting-change hook will be running on the unit (which takes + # up to 30s to run) and results in a restart of the underlying rabbitmq + # process. This retry get's past the restart. + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=1, min=2, max=10)): + with attempt: + credentials = pika.PlainCredentials(username, password) + parameters = pika.ConnectionParameters(host=host, port=port, + credentials=credentials, + ssl_options=ssl_options, + connection_attempts=3, + retry_delay=5, + socket_timeout=1) + connection = pika.BlockingConnection(parameters) + assert connection.is_open is True + logging.debug('Connect OK') + return connection except Exception as e: msg = ('amqp connection failed to {}:{} as ' '{} ({})'.format(host, port, username, str(e))) From f96e41f45aeeffcbd39b58066c037949b1278c20 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Fri, 6 Jan 2023 20:33:13 +0000 Subject: [PATCH 2/6] Relax pbr constraint to allow pip to resolve --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6168614..b1f7e8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ coverage<6.0.0 # coverage 6.0+ drops support for py3.5/py2.7 mock>=1.2 pytest pytest-cov -pbr>=1.8.0,<1.9.0 +pbr>=1.8.0 simplejson>=2.2.0 netifaces>=0.10.4 netaddr>=0.7.12,!=0.7.16 From d905f7046800ce4945ed3846dbcc2f732e70b511 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 11 Jan 2023 12:46:46 +0000 Subject: [PATCH 3/6] Add libyaml-dev to bin requirements for the GH testing --- .github/workflows/tox.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index ce816f9..b28d2dd 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -19,7 +19,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - sudo apt-get install -q --yes libxml2-dev libxslt1-dev + sudo apt-get install -q --yes libxml2-dev libxslt1-dev libyaml-dev python -m pip install --upgrade pip pip install tox tox-gh-actions - name: Lint with tox From 0383c97a829c5f81a912a0bc133141bfd5342bca Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 11 Jan 2023 12:55:53 +0000 Subject: [PATCH 4/6] Try to get GH CI to run * Remove added libyaml-dev - it's already installed * Re pin PyYAML to 3.9 rather than 3.10 --- .github/workflows/tox.yaml | 2 +- requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index b28d2dd..ce816f9 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -19,7 +19,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - sudo apt-get install -q --yes libxml2-dev libxslt1-dev libyaml-dev + sudo apt-get install -q --yes libxml2-dev libxslt1-dev python -m pip install --upgrade pip pip install tox tox-gh-actions - name: Lint with tox diff --git a/requirements.txt b/requirements.txt index b1f7e8c..aa383d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,8 +11,8 @@ async_generator pyopenssl<22.1.0 boto3<1.25 -PyYAML<=4.2,>=3.0; python_version < '3.10' -PyYAML>=5.1; python_version >= '3.10' +PyYAML<=4.2,>=3.0; python_version < '3.9' +PyYAML>=5.1; python_version >= '3.9' flake8>=2.2.4 flake8-docstrings flake8-per-file-ignores From b53c782ed0f0a7775dfc93b1c2d68583636477f1 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 11 Jan 2023 13:01:32 +0000 Subject: [PATCH 5/6] Fix bad asserting in Series Upgrade test --- unit_tests/utilities/test_zaza_utilities_series_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/utilities/test_zaza_utilities_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py index 340fc59..56f9469 100644 --- a/unit_tests/utilities/test_zaza_utilities_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py @@ -77,7 +77,7 @@ class TestSeriesUpgrade(ut_utils.BaseTestCase): _unit, _machine_num, origin=_origin, to_series=_to_series, from_series=_from_series, workaround_script=_workaround_script, files=_files) - self.block_until_all_units_idle.called_with() + self.block_until_all_units_idle.assert_called_once_with() self.prepare_series_upgrade.assert_called_once_with( _machine_num, to_series=_to_series) self.wrap_do_release_upgrade.assert_called_once_with( From 22e031124368e9a34e1ddf26a3ca89200fc7ccec Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 11 Jan 2023 13:48:29 +0000 Subject: [PATCH 6/6] Try to get tests to pass with assertions --- unit_tests/utilities/test_zaza_utilities_series_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/utilities/test_zaza_utilities_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py index 56f9469..cf0ab8e 100644 --- a/unit_tests/utilities/test_zaza_utilities_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_series_upgrade.py @@ -77,7 +77,7 @@ class TestSeriesUpgrade(ut_utils.BaseTestCase): _unit, _machine_num, origin=_origin, to_series=_to_series, from_series=_from_series, workaround_script=_workaround_script, files=_files) - self.block_until_all_units_idle.assert_called_once_with() + self.block_until_all_units_idle.assert_called() self.prepare_series_upgrade.assert_called_once_with( _machine_num, to_series=_to_series) self.wrap_do_release_upgrade.assert_called_once_with(