From 9bceee55c8c1aa04b545b878e5e7c8187f2fdc11 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 12:17:55 +1000 Subject: [PATCH 01/50] Add function that returns a map of unit names -> hostnames --- zaza/openstack/utilities/generic.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 7eb175c..b2f7354 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -75,6 +75,16 @@ def get_network_config(net_topology, ignore_env_vars=False, return net_info +def get_unit_hostnames(units): + """Return a dict of juju unit names to hostnames.""" + host_names = {} + for unit in units: + output = model.run_on_unit(unit.entity_id, 'hostname') + hostname = output['Stdout'].strip() + host_names[unit.entity_id] = hostname + return host_names + + def get_pkg_version(application, pkg): """Return package version. From 27bd15a9e0cac4958a85b7eba9999e387dc7e808 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 12:18:43 +1000 Subject: [PATCH 02/50] Add a function that returns contents of a file from a unit --- zaza/openstack/utilities/generic.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index b2f7354..8b562ab 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -716,6 +716,11 @@ def get_ubuntu_release(ubuntu_name): return index +def get_file_contents(unit, f): + return model.run_on_unit(unit.entity_id, + "cat {}".format(f))['Stdout'] + + def is_port_open(port, address): """Determine if TCP port is accessible. From 2888cb5e92615f9f8c44c1be7fe6c099f0919472 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 12:19:25 +1000 Subject: [PATCH 03/50] Add a function that checks port access given a list of units --- zaza/openstack/utilities/generic.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 8b562ab..c8e942e 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -744,3 +744,21 @@ def is_port_open(port, address): logging.error("connection refused connecting" " to {}:{}".format(address, port)) return False + + +def port_knock_units(units, port=22, expect_success=True): + """Open a TCP socket to check for a listening sevice on each + listed juju unit. + :param units: list of unit pointers + :param port: TCP port number, default to 22 + :param timeout: Connect timeout, default to 15 seconds + :expect_success: True by default, set False to invert logic + :returns: None if successful, Failure message otherwise + """ + for u in units: + host = u.public_address + connected = is_port_open(port, host) + if not connected and expect_success: + return 'Socket connect failed.' + elif connected and not expect_success: + return 'Socket connected unexpectedly.' From 31c71f4061def1a4ed363a534c1788d58087d8d3 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 12:20:26 +1000 Subject: [PATCH 04/50] Add function that runs/checks a command on all units, expecting exit 0 --- zaza/openstack/utilities/generic.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index c8e942e..f026321 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -513,6 +513,31 @@ def dist_upgrade(unit_name): model.run_on_unit(unit_name, dist_upgrade_cmd) +def check_commands_on_units(commands, units): + """Check that all commands in a list exit zero on all + units in a list. + :param commands: list of bash commands + :param units: list of unit pointers + :returns: None if successful; Failure message otherwise + """ + logging.debug('Checking exit codes for {} commands on {} ' + 'units...'.format(len(commands), + len(units))) + + for u in units: + for cmd in commands: + output = model.run_on_unit(u.entity_id, cmd) + if int(output['Code']) == 0: + logging.debug('{} `{}` returned {} ' + '(OK)'.format(u.entity_id, + cmd, output['Code'])) + else: + return ('{} `{}` returned {} ' + '{}'.format(u.entity_id, + cmd, output['Code'], output)) + return None + + def do_release_upgrade(unit_name): """Run do-release-upgrade noninteractive. From cf91a9dab940f7dfdd305eacc416e03a0d05f6e9 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:07:22 +1000 Subject: [PATCH 05/50] Create rabbitmq_server.utils file (add a license as a starter) --- .../charm_tests/rabbitmq_server/__init__.py | 15 +++++++++++++++ .../charm_tests/rabbitmq_server/utils.py | 13 +++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 zaza/openstack/charm_tests/rabbitmq_server/__init__.py create mode 100644 zaza/openstack/charm_tests/rabbitmq_server/utils.py diff --git a/zaza/openstack/charm_tests/rabbitmq_server/__init__.py b/zaza/openstack/charm_tests/rabbitmq_server/__init__.py new file mode 100644 index 0000000..cdac408 --- /dev/null +++ b/zaza/openstack/charm_tests/rabbitmq_server/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing rabbitmq server.""" diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py new file mode 100644 index 0000000..e429521 --- /dev/null +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -0,0 +1,13 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 2fb7dcbdfc76626a949ea8267e53ea4b47d8c441 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:09:09 +1000 Subject: [PATCH 06/50] Add function that waits until Rmq units are clustered and ready --- .../charm_tests/rabbitmq_server/utils.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index e429521..7599511 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -11,3 +11,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import zaza.model + + +def wait_for_cluster(model_name=None, timeout=1200): + """Wait for rmq units extended status to show cluster readiness, + after an optional initial sleep period. Initial sleep is likely + necessary to be effective following a config change, as status + message may not instantly update to non-ready.""" + states = { + 'rabbitmq-server': { + 'workload-status-messages': 'Unit is ready and clustered' + } + } + + zaza.model.wait_for_application_states(model_name=model_name, + states=states, + timeout=timeout) + + From 47657d6113a2686323aec00c97970632c25be13f Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:12:10 +1000 Subject: [PATCH 07/50] Add a function for getting Rmq cluster_status info from a unit --- .../openstack/charm_tests/rabbitmq_server/utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 7599511..3ef9836 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import zaza.model @@ -31,3 +32,16 @@ def wait_for_cluster(model_name=None, timeout=1200): timeout=timeout) +def get_cluster_status(unit): + """Execute rabbitmq cluster status command on a unit and return + the full output. + :param unit: unit + :returns: String containing console output of cluster status command + """ + cmd = 'rabbitmqctl cluster_status' + output = zaza.model.run_on_unit(unit.entity_id, cmd)['Stdout'].strip() + logging.debug('{} cluster_status:\n{}'.format( + unit.entity_id, output)) + return str(output) + + From fa68f09e721282e05701ad6fcaff55d3276e26ac Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:13:56 +1000 Subject: [PATCH 08/50] Add a function providing all units/nodes Rmq cluster_status in JSON --- .../charm_tests/rabbitmq_server/utils.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 3ef9836..66cb1c8 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import zaza.model @@ -45,3 +46,22 @@ def get_cluster_status(unit): return str(output) +def get_cluster_running_nodes(unit): + """Parse rabbitmqctl cluster_status output string, return list of + running rabbitmq cluster nodes. + :param unit: unit pointer + :returns: List containing node names of running nodes + """ + # NOTE(beisner): rabbitmqctl cluster_status output is not + # json-parsable, do string chop foo, then json.loads that. + str_stat = get_cluster_status(unit) + if 'running_nodes' in str_stat: + pos_start = str_stat.find("{running_nodes,") + 15 + pos_end = str_stat.find("]},", pos_start) + 1 + str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"') + run_nodes = json.loads(str_run_nodes) + return run_nodes + else: + return [] + + From 6749ff763ed9574b60a31305ea62fc9a0d9a092b Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:25:32 +1000 Subject: [PATCH 09/50] Add a function for creating connection to Rmq instance on unit --- .../charm_tests/rabbitmq_server/utils.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 66cb1c8..734c8bb 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -14,8 +14,11 @@ import json import logging + +import pika import zaza.model +import ssl as libssl def wait_for_cluster(model_name=None, timeout=1200): """Wait for rmq units extended status to show cluster readiness, @@ -65,3 +68,55 @@ def get_cluster_running_nodes(unit): return [] +def connect_amqp_by_unit(unit, ssl=False, + port=None, fatal=True, + username="testuser1", password="changeme"): + """Establish and return a pika amqp connection to the rabbitmq service + running on a rmq juju unit. + :param unit: unit pointer + :param ssl: boolean, default to False + :param port: amqp port, use defaults if None + :param fatal: boolean, default to True (raises on connect error) + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :returns: pika amqp connection pointer or None if failed and non-fatal + """ + host = unit.public_address + unit_name = unit.entity_id + + if ssl: + ssl_options = pika.SSLOptions(libssl.SSLContext()) + else: + ssl_options = None + + # Default port logic if port is not specified + if ssl and not port: + port = 5671 + elif not ssl and not port: + port = 5672 + + logging.debug('Connecting to amqp on {}:{} ({}) as ' + '{}...'.format(host, port, unit_name, username)) + + try: + credentials = pika.PlainCredentials(username, password) + parameters = pika.ConnectionParameters(host=host, port=port, + credentials=credentials, + ssl_options=ssl_options, + connection_attempts=3, + retry_delay=5, + socket_timeout=1) + connection = pika.BlockingConnection(parameters) + assert connection.is_open is True + logging.debug('Connect OK') + return connection + except Exception as e: + msg = ('amqp connection failed to {}:{} as ' + '{} ({})'.format(host, port, username, str(e))) + if fatal: + raise Exception(msg) + else: + logging.warn(msg) + return None + + From efb781b04817d5c09c7afb67f7e951f5abd80c4d Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:39:35 +1000 Subject: [PATCH 10/50] Create rabbitmq_server.tests file (add a license as a starter) --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 zaza/openstack/charm_tests/rabbitmq_server/tests.py diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py new file mode 100644 index 0000000..e429521 --- /dev/null +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -0,0 +1,13 @@ +# Copyright 2019 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From e0707fac044562bb2aa1aabfd5f481e7d03a904e Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:41:29 +1000 Subject: [PATCH 11/50] Add an RmqTests class (tests to follow...) --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index e429521..d50d8c2 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -11,3 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import zaza.openstack.charm_tests.test_utils as test_utils + + +class RmqTests(test_utils.OpenStackBaseTest): + """Zaza tests on a basic rabbitmq cluster deployment. Verify + relations, service status, users and endpoint service catalog.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(RmqTests, cls).setUpClass() + From 313f07bd0c267034efb4f5fdb14c6b04db5bbefe Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:42:08 +1000 Subject: [PATCH 12/50] Add test confirming that all Rmq units are part of cluster --- .../charm_tests/rabbitmq_server/tests.py | 17 ++++++++++ .../charm_tests/rabbitmq_server/utils.py | 32 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index d50d8c2..1168759 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -12,8 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging +import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils +from . import utils as rmq_utils + class RmqTests(test_utils.OpenStackBaseTest): """Zaza tests on a basic rabbitmq cluster deployment. Verify @@ -24,3 +28,16 @@ class RmqTests(test_utils.OpenStackBaseTest): """Run class setup for running tests.""" super(RmqTests, cls).setUpClass() + def test_400_rmq_cluster_running_nodes(self): + """Verify that cluster status from each rmq juju unit shows + every cluster node as a running member in that cluster.""" + logging.debug('Checking that all units are in cluster_status ' + 'running nodes...') + + units = zaza.model.get_units(self.application_name) + + ret = rmq_utils.validate_cluster_running_nodes(units) + self.assertIsNone(ret) + + logging.info('OK\n') + diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 734c8bb..1335dd6 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -19,6 +19,8 @@ import pika import zaza.model import ssl as libssl +import zaza.openstack.utilities.generic as generic_utils + def wait_for_cluster(model_name=None, timeout=1200): """Wait for rmq units extended status to show cluster readiness, @@ -68,6 +70,36 @@ def get_cluster_running_nodes(unit): return [] +def validate_cluster_running_nodes(units): + """Check that all rmq unit hostnames are represented in the + cluster_status output of all units. + :param host_names: dict of juju unit names to host names + :param units: list of unit pointers (all rmq units) + :returns: None if successful, otherwise return error message + """ + host_names = generic_utils.get_unit_hostnames(units) + errors = [] + + # Query every unit for cluster_status running nodes + for query_unit in units: + query_unit_name = query_unit.entity_id + running_nodes = get_cluster_running_nodes(query_unit) + + # Confirm that every unit is represented in the queried unit's + # cluster_status running nodes output. + for validate_unit in units: + val_host_name = host_names[validate_unit.entity_id] + val_node_name = 'rabbit@{}'.format(val_host_name) + + if val_node_name not in running_nodes: + errors.append('Cluster member check failed on {}: {} not ' + 'in {}\n'.format(query_unit_name, + val_node_name, + running_nodes)) + if errors: + return ''.join(errors) + + def connect_amqp_by_unit(unit, ssl=False, port=None, fatal=True, username="testuser1", password="changeme"): From 3a64bec97aebd178901ff71285679e1d8da8ca56 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:55:31 +1000 Subject: [PATCH 13/50] Add function for adding a user to an Rmq cluster --- .../charm_tests/rabbitmq_server/utils.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 1335dd6..2ccebe3 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -38,6 +38,43 @@ def wait_for_cluster(model_name=None, timeout=1200): timeout=timeout) +def add_user(units, username="testuser1", password="changeme"): + """Add a user via the first rmq juju unit, check connection as + the new user against all units. + :param units: list of unit pointers + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :returns: None if successful. Raise on error. + """ + logging.debug('Adding rmq user ({})...'.format(username)) + + # Check that user does not already exist + cmd_user_list = 'rabbitmqctl list_users' + cmd_result = zaza.model.run_on_unit(units[0].entity_id, cmd_user_list) + output = cmd_result['Stdout'].strip() + if username in output: + logging.warning('User ({}) already exists, returning ' + 'gracefully.'.format(username)) + return + + perms = '".*" ".*" ".*"' + cmds = ['rabbitmqctl add_user {} {}'.format(username, password), + 'rabbitmqctl set_permissions {} {}'.format(username, perms)] + + # Add user via first unit + for cmd in cmds: + cmd_result = zaza.model.run_on_unit(units[0].entity_id, cmd) + output = cmd_result['Stdout'].strip() + + # Check connection against the other units + logging.debug('Checking user connect against units...') + for u in units: + connection = connect_amqp_by_unit(u, ssl=False, + username=username, + password=password) + connection.close() + + def get_cluster_status(unit): """Execute rabbitmq cluster status command on a unit and return the full output. From 6d3714aa06da7b56f8c075b81528680cd0a0a49a Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 14:55:59 +1000 Subject: [PATCH 14/50] Add a function for deleting a user from a Rmq cluster --- .../charm_tests/rabbitmq_server/utils.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 2ccebe3..0380d70 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -75,6 +75,30 @@ def add_user(units, username="testuser1", password="changeme"): connection.close() +def delete_user(units, username="testuser1"): + """Delete a rabbitmq user via the first rmq juju unit. + :param units: list of unit pointers + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :returns: None if successful or no such user. + """ + logging.debug('Deleting rmq user ({})...'.format(username)) + + # Check that the user exists + cmd_user_list = 'rabbitmqctl list_users' + output = zaza.model.run_on_unit(units[0].entity_id, + cmd_user_list)['Stdout'].strip() + + if username not in output: + logging.warning('User ({}) does not exist, returning ' + 'gracefully.'.format(username)) + return + + # Delete the user + cmd_user_del = 'rabbitmqctl delete_user {}'.format(username) + output = zaza.model.run_on_unit(units[0].entity_id, cmd_user_del) + + def get_cluster_status(unit): """Execute rabbitmq cluster status command on a unit and return the full output. From 79a9f179dfd962447b6cc159061ac5708c8404fb Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:05:28 +1000 Subject: [PATCH 15/50] Add function for determing if SSL is enabled in unit's Rmq config --- .../charm_tests/rabbitmq_server/utils.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 0380d70..e52479c 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -161,6 +161,42 @@ def validate_cluster_running_nodes(units): return ''.join(errors) +def is_ssl_enabled_on_unit(unit, port=None): + """Check a single juju rmq unit for ssl and port in the config file.""" + host = unit.public_address + unit_name = unit.entity_id + + conf_file = '/etc/rabbitmq/rabbitmq.config' + conf_contents = str(generic_utils.get_file_contents(unit, + conf_file)) + # Checks + conf_ssl = 'ssl' in conf_contents + conf_port = str(port) in conf_contents + + # Port explicitly checked in config + if port and conf_port and conf_ssl: + logging.debug('SSL is enabled @{}:{} ' + '({})'.format(host, port, unit_name)) + return True + elif port and not conf_port and conf_ssl: + logging.debug('SSL is enabled @{} but not on port {} ' + '({})'.format(host, port, unit_name)) + return False + # Port not checked (useful when checking that ssl is disabled) + elif not port and conf_ssl: + logging.debug('SSL is enabled @{}:{} ' + '({})'.format(host, port, unit_name)) + return True + elif not conf_ssl: + logging.debug('SSL not enabled @{}:{} ' + '({})'.format(host, port, unit_name)) + return False + else: + msg = ('Unknown condition when checking SSL status @{}:{} ' + '({})'.format(host, port, unit_name)) + raise ValueError(msg) + + def connect_amqp_by_unit(unit, ssl=False, port=None, fatal=True, username="testuser1", password="changeme"): From c917c5b370c72a9b9b77553f01344fe1d0b17d93 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:06:51 +1000 Subject: [PATCH 16/50] Add function validating if SSL is enabled on Rmq units --- zaza/openstack/charm_tests/rabbitmq_server/utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index e52479c..8ee5b17 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -161,6 +161,19 @@ def validate_cluster_running_nodes(units): return ''.join(errors) +def validate_ssl_enabled_units(units, port=None): + """Check that ssl is enabled on rmq juju units. + :param units: list of all rmq units + :param port: optional ssl port override to validate + :returns: None if successful, otherwise return error message + """ + for u in units: + if not is_ssl_enabled_on_unit(u, port=port): + return ('Unexpected condition: ssl is disabled on unit ' + '({})'.format(u.info['unit_name'])) + return None + + def is_ssl_enabled_on_unit(unit, port=None): """Check a single juju rmq unit for ssl and port in the config file.""" host = unit.public_address From 97cc2b8e6040ddff9bb01c235234fb437c244d3e Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:07:25 +1000 Subject: [PATCH 17/50] Add function validating is SSL is disabled on Rmq units --- zaza/openstack/charm_tests/rabbitmq_server/utils.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 8ee5b17..99545e1 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -174,6 +174,18 @@ def validate_ssl_enabled_units(units, port=None): return None +def validate_ssl_disabled_units(units): + """Check that ssl is enabled on listed rmq juju units. + :param units: list of all rmq units + :returns: True if successful. Raise on error. + """ + for u in units: + if is_ssl_enabled_on_unit(u): + return ('Unexpected condition: ssl is enabled on unit ' + '({})'.format(u.entity_id)) + return None + + def is_ssl_enabled_on_unit(unit, port=None): """Check a single juju rmq unit for ssl and port in the config file.""" host = unit.public_address From 79d70e4f8d9ee3710e221504b532a34be97659dc Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:12:00 +1000 Subject: [PATCH 18/50] Add a function for enabling SSL on Rmq units --- .../charm_tests/rabbitmq_server/utils.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 99545e1..2c945e7 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -14,6 +14,7 @@ import json import logging +import time import pika import zaza.model @@ -186,6 +187,43 @@ def validate_ssl_disabled_units(units): return None +def configure_ssl_on(units, model_name=None, + port=None, max_wait=60): + """Turn ssl charm config option on, with optional non-default + ssl port specification. Confirm that it is enabled on every + unit. + :param units: list of units + :param port: amqp port, use defaults if None + :param max_wait: maximum time to wait in seconds to confirm + :returns: None if successful. Raise on error. + """ + logging.debug('Setting ssl charm config option: on') + + # Enable RMQ SSL + config = {'ssl': 'on'} + if port: + config['ssl_port'] = str(port) + + zaza.model.set_application_config('rabbitmq-server', + config, + model_name=model_name) + + # Wait for unit status + wait_for_cluster(model_name) + + # Confirm + tries = 0 + ret = validate_ssl_enabled_units(units, port=port) + while ret and tries < (max_wait / 4): + time.sleep(4) + logging.debug('Attempt {}: {}'.format(tries, ret)) + ret = validate_ssl_enabled_units(units, port=port) + tries += 1 + + if ret: + raise Exception(ret) + + def is_ssl_enabled_on_unit(unit, port=None): """Check a single juju rmq unit for ssl and port in the config file.""" host = unit.public_address From e09660f7c12eb761525664e468c6bf0f96722f35 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:12:27 +1000 Subject: [PATCH 19/50] Add a function for disabling SSL on Rmq units --- .../charm_tests/rabbitmq_server/utils.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 2c945e7..450855a 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -224,6 +224,37 @@ def configure_ssl_on(units, model_name=None, raise Exception(ret) +def configure_ssl_off(units, model_name=None, max_wait=60): + """Turn ssl charm config option off, confirm that it is disabled + on every unit. + :param units: list of units + :param max_wait: maximum time to wait in seconds to confirm + :returns: None if successful. Raise on error. + """ + logging.debug('Setting ssl charm config option: off') + + # Disable RMQ SSL + config = {'ssl': 'off'} + zaza.model.set_application_config('rabbitmq-server', + config, + model_name=model_name) + + # Wait for unit status + wait_for_cluster(model_name) + + # Confirm + tries = 0 + ret = validate_ssl_disabled_units(units) + while ret and tries < (max_wait / 4): + time.sleep(4) + logging.debug('Attempt {}: {}'.format(tries, ret)) + ret = validate_ssl_disabled_units(units) + tries += 1 + + if ret: + raise Exception(ret) + + def is_ssl_enabled_on_unit(unit, port=None): """Check a single juju rmq unit for ssl and port in the config file.""" host = unit.public_address From 20aad97033def58a677e2b041051ad5e7aed286c Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:28:56 +1000 Subject: [PATCH 20/50] Add a function for publishing a message to an Rmq unit --- .../charm_tests/rabbitmq_server/utils.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 450855a..fd08ee5 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -343,3 +343,40 @@ def connect_amqp_by_unit(unit, ssl=False, return None +def publish_amqp_message_by_unit(unit, message, + queue="test", ssl=False, + username="testuser1", + password="changeme", + port=None): + """Publish an amqp message to a rmq juju unit. + :param unit: unit pointer + :param message: amqp message string + :param queue: message queue, default to test + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :param ssl: boolean, default to False + :param port: amqp port, use defaults if None + :returns: None. Raises exception if publish failed. + """ + logging.debug('Publishing message to {} queue:\n{}'.format(queue, + message)) + connection = connect_amqp_by_unit(unit, ssl=ssl, + port=port, + username=username, + password=password) + + # NOTE(beisner): extra debug here re: pika hang potential: + # https://github.com/pika/pika/issues/297 + # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw + logging.debug('Defining channel...') + channel = connection.channel() + logging.debug('Declaring queue...') + channel.queue_declare(queue=queue, auto_delete=False, durable=True) + logging.debug('Publishing message...') + channel.basic_publish(exchange='', routing_key=queue, body=message) + logging.debug('Closing channel...') + channel.close() + logging.debug('Closing connection...') + connection.close() + + From 1acbfd874163112fbd7a681f152b348d1f08fc41 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:29:56 +1000 Subject: [PATCH 21/50] Add a function for retrieving a message to an Rmq unit --- .../charm_tests/rabbitmq_server/utils.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index fd08ee5..2447f8f 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -380,3 +380,34 @@ def publish_amqp_message_by_unit(unit, message, connection.close() +def get_amqp_message_by_unit(unit, queue="test", + username="testuser1", + password="changeme", + ssl=False, port=None): + """Get an amqp message from a rmq juju unit. + :param unit: unit pointer + :param queue: message queue, default to test + :param username: amqp user name, default to testuser1 + :param password: amqp user password + :param ssl: boolean, default to False + :param port: amqp port, use defaults if None + :returns: amqp message body as string. Raise if get fails. + """ + connection = connect_amqp_by_unit(unit, ssl=ssl, + port=port, + username=username, + password=password) + channel = connection.channel() + method_frame, _, body = channel.basic_get(queue) + body = body.decode() + + if method_frame: + logging.debug('Retreived message from {} queue:\n{}'.format(queue, + body)) + channel.basic_ack(method_frame.delivery_tag) + channel.close() + connection.close() + return body + else: + msg = 'No message retrieved.' + raise Exception(msg) From 1baff7281cba7d29b4f0aca60acb1ab92a667775 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:31:52 +1000 Subject: [PATCH 22/50] Helper function for testing if messages move across cluster successfully --- .../charm_tests/rabbitmq_server/tests.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 1168759..9808150 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -13,8 +13,13 @@ # limitations under the License. import logging +import time +import uuid + import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.generic as generic_utils + from . import utils as rmq_utils @@ -28,6 +33,86 @@ class RmqTests(test_utils.OpenStackBaseTest): """Run class setup for running tests.""" super(RmqTests, cls).setUpClass() + def _get_uuid_epoch_stamp(self): + """Returns a stamp string based on uuid4 and epoch time. Useful in + generating test messages which need to be unique-ish.""" + return '[{}-{}]'.format(uuid.uuid4(), time.time()) + + def _test_rmq_amqp_messages_all_units(self, units, + ssl=False, port=None): + """Reusable test to send amqp messages to every listed rmq unit + and check every listed rmq unit for messages. + + :param units: list of units + :returns: None if successful. Raise on error. + """ + + # Add test user if it does not already exist + rmq_utils.add_user(units) + + # Handle ssl (includes wait-for-cluster) + if ssl: + rmq_utils.configure_ssl_on(units, port=port) + else: + rmq_utils.configure_ssl_off(units) + + # Publish and get amqp messages in all possible unit combinations. + # Qty of checks == (qty of units) ^ 2 + amqp_msg_counter = 1 + host_names = generic_utils.get_unit_hostnames(units) + + for dest_unit in units: + dest_unit_name = dest_unit.entity_id + dest_unit_host = dest_unit.public_address + dest_unit_host_name = host_names[dest_unit_name] + + for check_unit in units: + check_unit_name = check_unit.entity_id + check_unit_host = check_unit.public_address + check_unit_host_name = host_names[check_unit_name] + + amqp_msg_stamp = self._get_uuid_epoch_stamp() + amqp_msg = ('Message {}@{} {}'.format(amqp_msg_counter, + dest_unit_host, + amqp_msg_stamp)).upper() + # Publish amqp message + logging.debug('Publish message to: {} ' + '({} {})'.format(dest_unit_host, + dest_unit_name, + dest_unit_host_name)) + + rmq_utils.publish_amqp_message_by_unit(dest_unit, + amqp_msg, ssl=ssl, + port=port) + + # Wait a bit before checking for message + time.sleep(10) + + # Get amqp message + logging.debug('Get message from: {} ' + '({} {})'.format(check_unit_host, + check_unit_name, + check_unit_host_name)) + + amqp_msg_rcvd = rmq_utils.get_amqp_message_by_unit(check_unit, + ssl=ssl, + port=port) + + # Validate amqp message content + if amqp_msg == amqp_msg_rcvd: + logging.debug('Message {} received ' + 'OK.'.format(amqp_msg_counter)) + else: + logging.error('Expected: {}'.format(amqp_msg)) + logging.error('Actual: {}'.format(amqp_msg_rcvd)) + msg = 'Message {} mismatch.'.format(amqp_msg_counter) + raise Exception(msg) + + amqp_msg_counter += 1 + + # Delete the test user + rmq_utils.delete_user(units) + def test_400_rmq_cluster_running_nodes(self): """Verify that cluster status from each rmq juju unit shows every cluster node as a running member in that cluster.""" From 79c816ed0f49358c51dd2c3f9cc411c9995cd03c Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:47:50 +1000 Subject: [PATCH 23/50] Test: messages can be sent/retrieved to/from Rmq cluster without SSL --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 9808150..37375d2 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -126,3 +126,13 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') + def test_406_rmq_amqp_messages_all_units_ssl_off(self): + """Send amqp messages to every rmq unit and check every rmq unit + for messages. Standard amqp tcp port, no ssl.""" + logging.debug('Checking amqp message publish/get on all units ' + '(ssl off)...') + + units = zaza.model.get_units(self.application_name) + self._test_rmq_amqp_messages_all_units(units, ssl=False) + logging.info('OK\n') + From 405cdf63885b5da6642c08d55dbce597c598f873 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 15:50:18 +1000 Subject: [PATCH 24/50] Test: RMQ management plugin enabling/disabling exposes/hides ports --- .../charm_tests/rabbitmq_server/tests.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 37375d2..33d52f5 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -136,3 +136,54 @@ class RmqTests(test_utils.OpenStackBaseTest): self._test_rmq_amqp_messages_all_units(units, ssl=False) logging.info('OK\n') + def test_412_rmq_management_plugin(self): + """Enable and check management plugin.""" + logging.debug('Checking tcp socket connect to management plugin ' + 'port on all rmq units...') + + units = zaza.model.get_units(self.application_name) + mgmt_port = 15672 + + # Enable management plugin + logging.debug('Enabling management_plugin charm config option...') + config = {'management_plugin': 'True'} + zaza.model.set_application_config('rabbitmq-server', config) + rmq_utils.wait_for_cluster() + + # Check tcp connect to management plugin port + max_wait = 600 + tries = 0 + ret = generic_utils.port_knock_units(units, mgmt_port) + while ret and tries < (max_wait / 30): + time.sleep(30) + logging.debug('Attempt {}: {}'.format(tries, ret)) + ret = generic_utils.port_knock_units(units, mgmt_port) + tries += 1 + + self.assertIsNone(ret) + logging.debug('Connect to all units (OK)\n') + + # Disable management plugin + logging.debug('Disabling management_plugin charm config option...') + config = {'management_plugin': 'False'} + zaza.model.set_application_config('rabbitmq-server', config) + rmq_utils.wait_for_cluster() + + # Negative check - tcp connect to management plugin port + logging.info('Expect tcp connect fail since charm config ' + 'option is disabled.') + tries = 0 + ret = generic_utils.port_knock_units(units, + mgmt_port, + expect_success=False) + + while ret and tries < (max_wait / 30): + time.sleep(30) + logging.debug('Attempt {}: {}'.format(tries, ret)) + ret = generic_utils.port_knock_units(units, mgmt_port, + expect_success=False) + tries += 1 + + self.assertIsNone(ret) + logging.info('Confirm mgmt port closed on all units (OK)\n') + From c5aba589ce0f8223bec012b2b4d1b289d0b012bf Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:14:04 +1000 Subject: [PATCH 25/50] Test: existence NRPE configs/data on Rmq units --- .../charm_tests/rabbitmq_server/tests.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 33d52f5..6faf015 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -187,3 +187,42 @@ class RmqTests(test_utils.OpenStackBaseTest): self.assertIsNone(ret) logging.info('Confirm mgmt port closed on all units (OK)\n') + def test_414_rmq_nrpe_monitors(self): + """Check rabbimq-server nrpe monitor basic functionality.""" + units = zaza.model.get_units(self.application_name) + host_names = generic_utils.get_unit_hostnames(units) + + # check_rabbitmq monitor + logging.debug('Checking nrpe check_rabbitmq on units...') + cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_rabbitmq.cfg'] + ret = generic_utils.check_commands_on_units(cmds, units) + self.assertIsNone(ret) + + logging.debug('Sleeping 2ms for 1m cron job to run...') + time.sleep(120) + + # check_rabbitmq_queue monitor + logging.debug('Checking nrpe check_rabbitmq_queue on units...') + cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_rabbitmq_queue.cfg'] + ret = generic_utils.check_commands_on_units(cmds, units) + self.assertIsNone(ret) + + # check dat file existence + logging.debug('Checking nrpe dat file existence on units...') + for u in units: + unit_host_name = host_names[u.entity_id] + + cmds = [ + 'stat /var/lib/rabbitmq/data/{}_general_stats.dat'.format( + unit_host_name), + 'stat /var/lib/rabbitmq/data/{}_queue_stats.dat'.format( + unit_host_name) + ] + + ret = generic_utils.check_commands_on_units(cmds, [u]) + self.assertIsNone(ret) + + logging.info('OK\n') + From 9e7d76df2f655bd2d0a810780ec41428a712e306 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:17:34 +1000 Subject: [PATCH 26/50] Test: Rmq unit action 'cluster-status' returns a Juju action --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 6faf015..58ab127 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -16,6 +16,7 @@ import logging import time import uuid +import juju import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils @@ -226,3 +227,13 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') + def test_911_cluster_status(self): + """ rabbitmqctl cluster_status action can be returned. """ + logging.debug('Checking cluster status action...') + + unit = zaza.model.get_units(self.application_name)[0] + action = zaza.model.run_action(unit.entity_id, "cluster-status") + self.assertIsInstance(action, juju.action.Action) + + logging.debug('OK') + From 2a0b7bf8e28bb8cbd0396a54aa4164374ffa5f5b Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:18:47 +1000 Subject: [PATCH 27/50] Test: Rmq unit action 'check-queues' returns a Juju action --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 58ab127..6127522 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -237,3 +237,11 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('OK') + def test_912_check_queues(self): + """ rabbitmqctl check_queues action can be returned. """ + logging.debug('Checking cluster status action...') + + unit = zaza.model.get_units(self.application_name)[0] + action = zaza.model.run_action(unit.entity_id, "check-queues") + self.assertIsInstance(action, juju.action.Action) + From a738efd7f215fe1477f7419b107e04314c672708 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:24:32 +1000 Subject: [PATCH 28/50] Test: Rmq unit action 'list-unconsumed-queues' returns a Juju action --- .../charm_tests/rabbitmq_server/tests.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 6127522..32ff23c 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging import time import uuid @@ -245,3 +246,31 @@ class RmqTests(test_utils.OpenStackBaseTest): action = zaza.model.run_action(unit.entity_id, "check-queues") self.assertIsInstance(action, juju.action.Action) + def test_913_list_unconsumed_queues(self): + """ rabbitmqctl list-unconsumed-queues action can be returned. """ + logging.debug('Checking list-unconsumed-queues action...') + + unit = zaza.model.get_units(self.application_name)[0] + self._test_rmq_amqp_messages_all_units([unit]) + action = zaza.model.run_action(unit.entity_id, + 'list-unconsumed-queues') + self.assertIsInstance(action, juju.action.Action) + + queue_count = int(action.results['unconsumed-queue-count']) + assert queue_count > 0, 'Did not find any unconsumed queues.' + + queue_name = 'test' # publish_amqp_message_by_unit default queue name + for i in range(queue_count): + queue_data = json.loads( + action.results['unconsumed-queues'][str(i)]) + if queue_data['name'] == queue_name: + break + else: + assert False, 'Did not find expected queue in result.' + + # Since we just reused _test_rmq_amqp_messages_all_units, we should + # have created the queue if it didn't already exist, but all messages + # should have already been consumed. + assert queue_data['messages'] == 0, 'Found unexpected message count.' + + logging.debug('OK') From 19c9e86b192633ed8a3a031317ea6cef43315dec Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:26:31 +1000 Subject: [PATCH 29/50] Test: Rmq pause/resume. Note: This test may have exposed a bug, where the `block_until_unit_wl_status` returns once it reaches the "maintenance" state, but subsequent queries to `unit.workload_status == "maintenance"` fail. Recreating the unit object (via `zaza.model.get_unit_from_name`) returns the correct workload_status when queried. --- .../charm_tests/rabbitmq_server/tests.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 32ff23c..dae8009 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -228,6 +228,30 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') + def test_910_pause_and_resume(self): + """The services can be paused and resumed. """ + + logging.debug('Checking pause and resume actions...') + + unit = zaza.model.get_units(self.application_name)[0] + assert unit.workload_status == "active" + + zaza.model.run_action(unit.entity_id, "pause") + zaza.model.block_until_unit_wl_status(unit.entity_id, "maintenance") + # TODO: investigate possible bug (the following line is + # required, otherwise it looks like workload_status is + # reporting cached information, no matter how long you sleep) + unit = zaza.model.get_unit_from_name(unit.entity_id) + assert unit.workload_status == "maintenance" + + zaza.model.run_action(unit.entity_id, "resume") + zaza.model.block_until_unit_wl_status(unit.entity_id, "active") + unit = zaza.model.get_unit_from_name(unit.entity_id) + assert unit.workload_status == "active" + + rmq_utils.wait_for_cluster() + logging.debug('OK') + def test_911_cluster_status(self): """ rabbitmqctl cluster_status action can be returned. """ logging.debug('Checking cluster status action...') From ce45a47dab9200841244daabf914096b8a75db7a Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:32:01 +1000 Subject: [PATCH 30/50] Test: messages can be sent/retrieved to/from Rmq cluster with SSL on As per the code: is there a function to determine unit's release? Otherwise, I'll just implement a generic function that run_on_unit lsb_release -cs --- .../charm_tests/rabbitmq_server/tests.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index dae8009..b13f186 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -138,6 +138,26 @@ class RmqTests(test_utils.OpenStackBaseTest): self._test_rmq_amqp_messages_all_units(units, ssl=False) logging.info('OK\n') + def test_408_rmq_amqp_messages_all_units_ssl_on(self): + """Send amqp messages with ssl enabled, to every rmq unit and + check every rmq unit for messages. Standard ssl tcp port.""" + # http://pad.lv/1625044 + # TODO: exsdev: find out if there's a function to determine unit's release + # Otherwise run_on_unit: lsb_release -cs + # if (CompareHostReleases(self.client_series) >= 'xenial' and + # CompareHostReleases(self.series) <= 'trusty'): + # logging.info('SKIP') + # logging.info('Skipping SSL tests due to client' + # ' compatibility issues') + # return + logging.debug('Checking amqp message publish/get on all units ' + '(ssl on)...') + + units = zaza.model.get_units(self.application_name) + self._test_rmq_amqp_messages_all_units(units, + ssl=True, port=5671) + logging.info('OK\n') + def test_412_rmq_management_plugin(self): """Enable and check management plugin.""" logging.debug('Checking tcp socket connect to management plugin ' From 1835d957a1c9a9f78c13b8cb30c2e481f0b8b476 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 19 Sep 2019 16:32:16 +1000 Subject: [PATCH 31/50] Test: msgs can be sent/retrieved to/from Rmq cluster with SSL+alt port Same message as my previous commit: As per the code: is there a function to determine unit's release? Otherwise, I'll just implement a generic function that run_on_unit lsb_release -cs --- .../charm_tests/rabbitmq_server/tests.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index b13f186..55f0049 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -158,6 +158,26 @@ class RmqTests(test_utils.OpenStackBaseTest): ssl=True, port=5671) logging.info('OK\n') + def test_410_rmq_amqp_messages_all_units_ssl_alt_port(self): + """Send amqp messages with ssl on, to every rmq unit and check + every rmq unit for messages. Custom ssl tcp port.""" + # http://pad.lv/1625044 + # TODO: exsdev: find out if there's a function to determine unit's release + # Otherwise run_on_unit: lsb_release -cs + # if (CompareHostReleases(self.client_series) >= 'xenial' and + # CompareHostReleases(self.series) <= 'trusty'): + # logging.info('SKIP') + # logging.info('Skipping SSL tests due to client' + # ' compatibility issues') + # return + logging.debug('Checking amqp message publish/get on all units ' + '(ssl on)...') + + units = zaza.model.get_units(self.application_name) + self._test_rmq_amqp_messages_all_units(units, + ssl=True, port=5999) + logging.info('OK\n') + def test_412_rmq_management_plugin(self): """Enable and check management plugin.""" logging.debug('Checking tcp socket connect to management plugin ' From 0e163593306dd3407895ebb468d51c473ffa5bd5 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 20 Sep 2019 13:57:05 +1000 Subject: [PATCH 32/50] Check client+unit OS versions before running Rmq tests with SSL enabled --- .../charm_tests/rabbitmq_server/tests.py | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 55f0049..3e3e0f0 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -22,6 +22,10 @@ import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils +from charmhelpers.core.host import ( + lsb_release, + CompareHostReleases, +) from . import utils as rmq_utils @@ -138,18 +142,26 @@ class RmqTests(test_utils.OpenStackBaseTest): self._test_rmq_amqp_messages_all_units(units, ssl=False) logging.info('OK\n') + def _series(self, unit): + result = zaza.model.run_on_unit(unit.entity_id, + "lsb_release -cs") + return result['Stdout'].strip() + + def _client_series(self): + return lsb_release()['DISTRIB_CODENAME'] + def test_408_rmq_amqp_messages_all_units_ssl_on(self): """Send amqp messages with ssl enabled, to every rmq unit and check every rmq unit for messages. Standard ssl tcp port.""" + units = zaza.model.get_units(self.application_name) + # http://pad.lv/1625044 - # TODO: exsdev: find out if there's a function to determine unit's release - # Otherwise run_on_unit: lsb_release -cs - # if (CompareHostReleases(self.client_series) >= 'xenial' and - # CompareHostReleases(self.series) <= 'trusty'): - # logging.info('SKIP') - # logging.info('Skipping SSL tests due to client' - # ' compatibility issues') - # return + if (CompareHostReleases(self._client_series()) >= 'xenial' and + CompareHostReleases(self._series(units[0])) <= 'trusty'): + logging.info('SKIP') + logging.info('Skipping SSL tests due to client' + ' compatibility issues') + return logging.debug('Checking amqp message publish/get on all units ' '(ssl on)...') @@ -161,15 +173,15 @@ class RmqTests(test_utils.OpenStackBaseTest): def test_410_rmq_amqp_messages_all_units_ssl_alt_port(self): """Send amqp messages with ssl on, to every rmq unit and check every rmq unit for messages. Custom ssl tcp port.""" + units = zaza.model.get_units(self.application_name) + # http://pad.lv/1625044 - # TODO: exsdev: find out if there's a function to determine unit's release - # Otherwise run_on_unit: lsb_release -cs - # if (CompareHostReleases(self.client_series) >= 'xenial' and - # CompareHostReleases(self.series) <= 'trusty'): - # logging.info('SKIP') - # logging.info('Skipping SSL tests due to client' - # ' compatibility issues') - # return + if (CompareHostReleases(self._client_series()) >= 'xenial' and + CompareHostReleases(self._series(units[0])) <= 'trusty'): + logging.info('SKIP') + logging.info('Skipping SSL tests due to client' + ' compatibility issues') + return logging.debug('Checking amqp message publish/get on all units ' '(ssl on)...') From 4792a527c9503ec27ebc8d15f1c89e5bb473d22c Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Tue, 24 Sep 2019 16:00:05 +1000 Subject: [PATCH 33/50] PROTOCOL_TLS is not available until Py3.6, use alternative --- zaza/openstack/charm_tests/rabbitmq_server/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 2447f8f..37c70e5 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -308,7 +308,10 @@ def connect_amqp_by_unit(unit, ssl=False, unit_name = unit.entity_id if ssl: - ssl_options = pika.SSLOptions(libssl.SSLContext()) + # TODO: when Python3.5 support is removed, investigate + # changing protocol to PROTOCOL_TLS + context = libssl.SSLContext(protocol=libssl.PROTOCOL_TLSv1_2) + ssl_options = pika.SSLOptions(context) else: ssl_options = None From 18726c341a94a129e5c79e2a8efb662ed84eccaa Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Wed, 25 Sep 2019 10:29:25 +1000 Subject: [PATCH 34/50] Fix lint errors found in docstrings --- .../charm_tests/rabbitmq_server/tests.py | 58 ++++++++++++------- .../charm_tests/rabbitmq_server/utils.py | 45 ++++++++++---- zaza/openstack/utilities/generic.py | 10 ++-- 3 files changed, 77 insertions(+), 36 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 3e3e0f0..898c078 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""RabbitMQ Testing.""" + import json import logging import time @@ -31,8 +33,7 @@ from . import utils as rmq_utils class RmqTests(test_utils.OpenStackBaseTest): - """Zaza tests on a basic rabbitmq cluster deployment. Verify - relations, service status, users and endpoint service catalog.""" + """Zaza tests on a basic rabbitmq cluster deployment.""" @classmethod def setUpClass(cls): @@ -40,19 +41,22 @@ class RmqTests(test_utils.OpenStackBaseTest): super(RmqTests, cls).setUpClass() def _get_uuid_epoch_stamp(self): - """Returns a stamp string based on uuid4 and epoch time. Useful in - generating test messages which need to be unique-ish.""" + """Return a string based on uuid4 and epoch time. + + Useful in generating test messages which need to be unique-ish. + """ return '[{}-{}]'.format(uuid.uuid4(), time.time()) def _test_rmq_amqp_messages_all_units(self, units, ssl=False, port=None): - """Reusable test to send amqp messages to every listed rmq unit - and check every listed rmq unit for messages. + """Reusable test to send/check amqp messages to every listed rmq unit. + Reusable test to send amqp messages to every listed rmq + unit. Checks every listed rmq unit for messages. :param units: list of units :returns: None if successful. Raise on error. - """ + """ # Add test user if it does not already exist rmq_utils.add_user(units) @@ -120,8 +124,7 @@ class RmqTests(test_utils.OpenStackBaseTest): rmq_utils.delete_user(units) def test_400_rmq_cluster_running_nodes(self): - """Verify that cluster status from each rmq juju unit shows - every cluster node as a running member in that cluster.""" + """Verify cluster status shows every cluster node as running member.""" logging.debug('Checking that all units are in cluster_status ' 'running nodes...') @@ -133,8 +136,12 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') def test_406_rmq_amqp_messages_all_units_ssl_off(self): - """Send amqp messages to every rmq unit and check every rmq unit - for messages. Standard amqp tcp port, no ssl.""" + """Send (and check) amqp messages to every rmq unit. + + Sends amqp messages to every rmq unit, and check every rmq + unit for messages. Uses Standard amqp tcp port, no ssl. + + """ logging.debug('Checking amqp message publish/get on all units ' '(ssl off)...') @@ -151,8 +158,12 @@ class RmqTests(test_utils.OpenStackBaseTest): return lsb_release()['DISTRIB_CODENAME'] def test_408_rmq_amqp_messages_all_units_ssl_on(self): - """Send amqp messages with ssl enabled, to every rmq unit and - check every rmq unit for messages. Standard ssl tcp port.""" + """Send (and check) amqp messages to every rmq unit (ssl enabled). + + Sends amqp messages to every rmq unit, and check every rmq + unit for messages. Uses Standard ssl tcp port. + + """ units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 @@ -160,7 +171,7 @@ class RmqTests(test_utils.OpenStackBaseTest): CompareHostReleases(self._series(units[0])) <= 'trusty'): logging.info('SKIP') logging.info('Skipping SSL tests due to client' - ' compatibility issues') + ' compatibility issues') return logging.debug('Checking amqp message publish/get on all units ' '(ssl on)...') @@ -171,8 +182,12 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') def test_410_rmq_amqp_messages_all_units_ssl_alt_port(self): - """Send amqp messages with ssl on, to every rmq unit and check - every rmq unit for messages. Custom ssl tcp port.""" + """Send (and check) amqp messages to every rmq unit (alt ssl port). + + Send amqp messages with ssl on, to every rmq unit and check + every rmq unit for messages. Custom ssl tcp port. + + """ units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 @@ -180,7 +195,7 @@ class RmqTests(test_utils.OpenStackBaseTest): CompareHostReleases(self._series(units[0])) <= 'trusty'): logging.info('SKIP') logging.info('Skipping SSL tests due to client' - ' compatibility issues') + ' compatibility issues') return logging.debug('Checking amqp message publish/get on all units ' '(ssl on)...') @@ -281,8 +296,7 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.info('OK\n') def test_910_pause_and_resume(self): - """The services can be paused and resumed. """ - + """The services can be paused and resumed.""" logging.debug('Checking pause and resume actions...') unit = zaza.model.get_units(self.application_name)[0] @@ -305,7 +319,7 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('OK') def test_911_cluster_status(self): - """ rabbitmqctl cluster_status action can be returned. """ + """Test rabbitmqctl cluster_status action can be returned.""" logging.debug('Checking cluster status action...') unit = zaza.model.get_units(self.application_name)[0] @@ -315,7 +329,7 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('OK') def test_912_check_queues(self): - """ rabbitmqctl check_queues action can be returned. """ + """Test rabbitmqctl check_queues action can be returned.""" logging.debug('Checking cluster status action...') unit = zaza.model.get_units(self.application_name)[0] @@ -323,7 +337,7 @@ class RmqTests(test_utils.OpenStackBaseTest): self.assertIsInstance(action, juju.action.Action) def test_913_list_unconsumed_queues(self): - """ rabbitmqctl list-unconsumed-queues action can be returned. """ + """Test rabbitmqctl list-unconsumed-queues action can be returned.""" logging.debug('Checking list-unconsumed-queues action...') unit = zaza.model.get_units(self.application_name)[0] diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 37c70e5..5a942e1 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +"""RabbitMQ Testing utility functions.""" + import json import logging import time @@ -24,10 +26,13 @@ import zaza.openstack.utilities.generic as generic_utils def wait_for_cluster(model_name=None, timeout=1200): - """Wait for rmq units extended status to show cluster readiness, + """Wait for Rmq cluster status to show cluster readiness. + + Wait for rmq units extended status to show cluster readiness, after an optional initial sleep period. Initial sleep is likely necessary to be effective following a config change, as status - message may not instantly update to non-ready.""" + message may not instantly update to non-ready. + """ states = { 'rabbitmq-server': { 'workload-status-messages': 'Unit is ready and clustered' @@ -40,7 +45,9 @@ def wait_for_cluster(model_name=None, timeout=1200): def add_user(units, username="testuser1", password="changeme"): - """Add a user via the first rmq juju unit, check connection as + """Add a user to a RabbitMQ cluster. + + Add a user via the first rmq juju unit, check connection as the new user against all units. :param units: list of unit pointers :param username: amqp user name, default to testuser1 @@ -77,7 +84,9 @@ def add_user(units, username="testuser1", password="changeme"): def delete_user(units, username="testuser1"): - """Delete a rabbitmq user via the first rmq juju unit. + """Delete a user from a RabbitMQ cluster. + + Delete a rabbitmq user via the first rmq juju unit. :param units: list of unit pointers :param username: amqp user name, default to testuser1 :param password: amqp user password @@ -101,7 +110,9 @@ def delete_user(units, username="testuser1"): def get_cluster_status(unit): - """Execute rabbitmq cluster status command on a unit and return + """Get RabbitMQ cluster status output. + + Execute rabbitmq cluster status command on a unit and return the full output. :param unit: unit :returns: String containing console output of cluster status command @@ -114,7 +125,9 @@ def get_cluster_status(unit): def get_cluster_running_nodes(unit): - """Parse rabbitmqctl cluster_status output string, return list of + """Get a list of RabbitMQ cluster's running nodes. + + Parse rabbitmqctl cluster_status output string, return list of running rabbitmq cluster nodes. :param unit: unit pointer :returns: List containing node names of running nodes @@ -133,7 +146,9 @@ def get_cluster_running_nodes(unit): def validate_cluster_running_nodes(units): - """Check that all rmq unit hostnames are represented in the + """Check all rmq unit hostnames are represented in cluster_status. + + Check that all rmq unit hostnames are represented in the cluster_status output of all units. :param host_names: dict of juju unit names to host names :param units: list of unit pointers (all rmq units) @@ -164,6 +179,7 @@ def validate_cluster_running_nodes(units): def validate_ssl_enabled_units(units, port=None): """Check that ssl is enabled on rmq juju units. + :param units: list of all rmq units :param port: optional ssl port override to validate :returns: None if successful, otherwise return error message @@ -177,6 +193,7 @@ def validate_ssl_enabled_units(units, port=None): def validate_ssl_disabled_units(units): """Check that ssl is enabled on listed rmq juju units. + :param units: list of all rmq units :returns: True if successful. Raise on error. """ @@ -189,7 +206,9 @@ def validate_ssl_disabled_units(units): def configure_ssl_on(units, model_name=None, port=None, max_wait=60): - """Turn ssl charm config option on, with optional non-default + """Turn RabbitMQ charm SSL config option on. + + Turn ssl charm config option on, with optional non-default ssl port specification. Confirm that it is enabled on every unit. :param units: list of units @@ -225,7 +244,9 @@ def configure_ssl_on(units, model_name=None, def configure_ssl_off(units, model_name=None, max_wait=60): - """Turn ssl charm config option off, confirm that it is disabled + """Turn RabbitMQ charm SSL config option off. + + Turn ssl charm config option off, confirm that it is disabled on every unit. :param units: list of units :param max_wait: maximum time to wait in seconds to confirm @@ -294,7 +315,9 @@ def is_ssl_enabled_on_unit(unit, port=None): def connect_amqp_by_unit(unit, ssl=False, port=None, fatal=True, username="testuser1", password="changeme"): - """Establish and return a pika amqp connection to the rabbitmq service + """Establish and return a pika amqp connection to the rabbitmq service. + + Establish and return a pika amqp connection to the rabbitmq service running on a rmq juju unit. :param unit: unit pointer :param ssl: boolean, default to False @@ -352,6 +375,7 @@ def publish_amqp_message_by_unit(unit, message, password="changeme", port=None): """Publish an amqp message to a rmq juju unit. + :param unit: unit pointer :param message: amqp message string :param queue: message queue, default to test @@ -388,6 +412,7 @@ def get_amqp_message_by_unit(unit, queue="test", password="changeme", ssl=False, port=None): """Get an amqp message from a rmq juju unit. + :param unit: unit pointer :param queue: message queue, default to test :param username: amqp user name, default to testuser1 diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index f026321..ed5aa0f 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -514,8 +514,8 @@ def dist_upgrade(unit_name): def check_commands_on_units(commands, units): - """Check that all commands in a list exit zero on all - units in a list. + """Check that all commands in a list exit zero on all units in a list. + :param commands: list of bash commands :param units: list of unit pointers :returns: None if successful; Failure message otherwise @@ -742,6 +742,7 @@ def get_ubuntu_release(ubuntu_name): def get_file_contents(unit, f): + """Get contents of a file on a remote unit.""" return model.run_on_unit(unit.entity_id, "cat {}".format(f))['Stdout'] @@ -772,8 +773,9 @@ def is_port_open(port, address): def port_knock_units(units, port=22, expect_success=True): - """Open a TCP socket to check for a listening sevice on each - listed juju unit. + """Check if specific port is open on units. + + Open a TCP socket to check for a listening sevice on each listed juju unit. :param units: list of unit pointers :param port: TCP port number, default to 22 :param timeout: Connect timeout, default to 15 seconds From b5f60126bcc87942cc328a6b44f18cd62eb261d5 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Mon, 30 Sep 2019 09:09:29 +1000 Subject: [PATCH 35/50] Remove duplicate call to function --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 898c078..18d6461 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -176,7 +176,6 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('Checking amqp message publish/get on all units ' '(ssl on)...') - units = zaza.model.get_units(self.application_name) self._test_rmq_amqp_messages_all_units(units, ssl=True, port=5671) logging.info('OK\n') From 60e84e4c276955a47ea0d38cd583241e3f5b8fd8 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Mon, 30 Sep 2019 09:12:47 +1000 Subject: [PATCH 36/50] Port Rmq charm func test removing a unit from a Rmq cluster This function was previously called test_901_remove_unit, but had to be renamed (moved to the end of the func tests); The way in which unit removal is now performed (by running the "stop" hook) puts the the removed unit in a "waiting" state -- which consequently causes wait_for_cluster() (e.g. used in 910) to fail (timeout). --- .../charm_tests/rabbitmq_server/tests.py | 41 +++++++++++++++++++ .../charm_tests/rabbitmq_server/utils.py | 23 +++++++++++ 2 files changed, 64 insertions(+) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 18d6461..3a144ff 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -363,3 +363,44 @@ class RmqTests(test_utils.OpenStackBaseTest): assert queue_data['messages'] == 0, 'Found unexpected message count.' logging.debug('OK') + + def test_921_remove_unit(self): + """Test if unit cleans up when removed from Rmq cluster. + + Test if a unit correctly cleans up by removing itself from the + RabbitMQ cluster on removal + + """ + logging.debug('Checking that units correctly clean up after ' + 'themselves on unit removal...') + config = {'min-cluster-size': '2'} + zaza.model.set_application_config('rabbitmq-server', config) + rmq_utils.wait_for_cluster() + + units = zaza.model.get_units(self.application_name) + removed_unit = units[-1] + left_units = units[:-1] + + zaza.model.run_on_unit(removed_unit.entity_id, 'hooks/stop') + zaza.model.block_until_unit_wl_status(removed_unit.entity_id, + "waiting") + + unit_host_names = generic_utils.get_unit_hostnames(left_units) + unit_node_names = [] + for unit in unit_host_names: + unit_node_names.append('rabbit@{}'.format(unit_host_names[unit])) + errors = [] + + for u in left_units: + e = rmq_utils.check_unit_cluster_nodes(u, unit_node_names) + if e: + # NOTE: cluster status may not have been updated yet so wait a + # little and try one more time. Need to find a better way to do + # this. + time.sleep(10) + e = rmq_utils.check_unit_cluster_nodes(u, unit_node_names) + if e: + errors.append(e) + + self.assertFalse(errors) + logging.debug('OK') diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 5a942e1..027ca50 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -439,3 +439,26 @@ def get_amqp_message_by_unit(unit, queue="test", else: msg = 'No message retrieved.' raise Exception(msg) + + +def check_unit_cluster_nodes(unit, unit_node_names): + """Check if unit exists in list of Rmq cluster node names.""" + unit_name = unit.entity_id + nodes = [] + errors = [] + str_stat = get_cluster_status(unit) + # make the interesting part of rabbitmqctl cluster_status output + # json-parseable. + if 'nodes,[{disc,' in str_stat: + pos_start = str_stat.find('nodes,[{disc,') + 13 + pos_end = str_stat.find(']}]},', pos_start) + 1 + str_nodes = str_stat[pos_start:pos_end].replace("'", '"') + nodes = json.loads(str_nodes) + for node in nodes: + if node not in unit_node_names: + errors.append('Cluster registration check failed on {}: ' + '{} should not be registered with RabbitMQ ' + 'after unit removal.\n' + ''.format(unit_name, node)) + + return errors From 1a9449766a58bb6aa74238e6cf252459deb75895 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Wed, 2 Oct 2019 08:51:06 +1000 Subject: [PATCH 37/50] Remove superfluous newlines at the end of debug messages --- .../openstack/charm_tests/rabbitmq_server/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 3a144ff..150a39b 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -133,7 +133,7 @@ class RmqTests(test_utils.OpenStackBaseTest): ret = rmq_utils.validate_cluster_running_nodes(units) self.assertIsNone(ret) - logging.info('OK\n') + logging.info('OK') def test_406_rmq_amqp_messages_all_units_ssl_off(self): """Send (and check) amqp messages to every rmq unit. @@ -147,7 +147,7 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) self._test_rmq_amqp_messages_all_units(units, ssl=False) - logging.info('OK\n') + logging.info('OK') def _series(self, unit): result = zaza.model.run_on_unit(unit.entity_id, @@ -178,7 +178,7 @@ class RmqTests(test_utils.OpenStackBaseTest): self._test_rmq_amqp_messages_all_units(units, ssl=True, port=5671) - logging.info('OK\n') + logging.info('OK') def test_410_rmq_amqp_messages_all_units_ssl_alt_port(self): """Send (and check) amqp messages to every rmq unit (alt ssl port). @@ -202,7 +202,7 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) self._test_rmq_amqp_messages_all_units(units, ssl=True, port=5999) - logging.info('OK\n') + logging.info('OK') def test_412_rmq_management_plugin(self): """Enable and check management plugin.""" @@ -229,7 +229,7 @@ class RmqTests(test_utils.OpenStackBaseTest): tries += 1 self.assertIsNone(ret) - logging.debug('Connect to all units (OK)\n') + logging.debug('Connect to all units (OK)') # Disable management plugin logging.debug('Disabling management_plugin charm config option...') @@ -253,7 +253,7 @@ class RmqTests(test_utils.OpenStackBaseTest): tries += 1 self.assertIsNone(ret) - logging.info('Confirm mgmt port closed on all units (OK)\n') + logging.info('Confirm mgmt port closed on all units (OK)') def test_414_rmq_nrpe_monitors(self): """Check rabbimq-server nrpe monitor basic functionality.""" @@ -292,7 +292,7 @@ class RmqTests(test_utils.OpenStackBaseTest): ret = generic_utils.check_commands_on_units(cmds, [u]) self.assertIsNone(ret) - logging.info('OK\n') + logging.info('OK') def test_910_pause_and_resume(self): """The services can be paused and resumed.""" From a0769e5dcb33b6ca3a713cccc61ecc15c2c965ed Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Wed, 2 Oct 2019 09:14:10 +1000 Subject: [PATCH 38/50] Remove TODO (behaviour confirmed not to be a bug) --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 150a39b..f93bd2d 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -303,9 +303,6 @@ class RmqTests(test_utils.OpenStackBaseTest): zaza.model.run_action(unit.entity_id, "pause") zaza.model.block_until_unit_wl_status(unit.entity_id, "maintenance") - # TODO: investigate possible bug (the following line is - # required, otherwise it looks like workload_status is - # reporting cached information, no matter how long you sleep) unit = zaza.model.get_unit_from_name(unit.entity_id) assert unit.workload_status == "maintenance" From 4eb068ab236449d9f76a01bc6a130cef7b414246 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Wed, 2 Oct 2019 09:12:43 +1000 Subject: [PATCH 39/50] Provide a message if assert fails --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index f93bd2d..a5a764a 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -131,7 +131,7 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) ret = rmq_utils.validate_cluster_running_nodes(units) - self.assertIsNone(ret) + self.assertIsNone(ret, msg=ret) logging.info('OK') @@ -228,7 +228,7 @@ class RmqTests(test_utils.OpenStackBaseTest): ret = generic_utils.port_knock_units(units, mgmt_port) tries += 1 - self.assertIsNone(ret) + self.assertIsNone(ret, msg=ret) logging.debug('Connect to all units (OK)') # Disable management plugin @@ -252,7 +252,7 @@ class RmqTests(test_utils.OpenStackBaseTest): expect_success=False) tries += 1 - self.assertIsNone(ret) + self.assertIsNone(ret, msg=ret) logging.info('Confirm mgmt port closed on all units (OK)') def test_414_rmq_nrpe_monitors(self): @@ -275,7 +275,7 @@ class RmqTests(test_utils.OpenStackBaseTest): cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' 'check_rabbitmq_queue.cfg'] ret = generic_utils.check_commands_on_units(cmds, units) - self.assertIsNone(ret) + self.assertIsNone(ret, msg=ret) # check dat file existence logging.debug('Checking nrpe dat file existence on units...') @@ -290,7 +290,7 @@ class RmqTests(test_utils.OpenStackBaseTest): ] ret = generic_utils.check_commands_on_units(cmds, [u]) - self.assertIsNone(ret) + self.assertIsNone(ret, msg=ret) logging.info('OK') From 5e170cd1ba87814473ced9681db0fe6f14ff1ab1 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 09:50:45 +1000 Subject: [PATCH 40/50] Make get_series/get_client_series available as public util functions --- .../charm_tests/rabbitmq_server/tests.py | 24 +++++++------------ zaza/openstack/utilities/generic.py | 13 ++++++++++ 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index a5a764a..2ae080a 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -24,9 +24,11 @@ import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils -from charmhelpers.core.host import ( - lsb_release, - CompareHostReleases, +from charmhelpers.core.host import CompareHostReleases + +from zaza.openstack.utilities.generic import ( + get_series, + get_client_series, ) from . import utils as rmq_utils @@ -149,14 +151,6 @@ class RmqTests(test_utils.OpenStackBaseTest): self._test_rmq_amqp_messages_all_units(units, ssl=False) logging.info('OK') - def _series(self, unit): - result = zaza.model.run_on_unit(unit.entity_id, - "lsb_release -cs") - return result['Stdout'].strip() - - def _client_series(self): - return lsb_release()['DISTRIB_CODENAME'] - def test_408_rmq_amqp_messages_all_units_ssl_on(self): """Send (and check) amqp messages to every rmq unit (ssl enabled). @@ -167,8 +161,8 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 - if (CompareHostReleases(self._client_series()) >= 'xenial' and - CompareHostReleases(self._series(units[0])) <= 'trusty'): + if (CompareHostReleases(get_client_series()) >= 'xenial' and + CompareHostReleases(get_series(units[0])) <= 'trusty'): logging.info('SKIP') logging.info('Skipping SSL tests due to client' ' compatibility issues') @@ -190,8 +184,8 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 - if (CompareHostReleases(self._client_series()) >= 'xenial' and - CompareHostReleases(self._series(units[0])) <= 'trusty'): + if (CompareHostReleases(get_client_series()) >= 'xenial' and + CompareHostReleases(get_series(units[0])) <= 'trusty'): logging.info('SKIP') logging.info('Skipping SSL tests due to client' ' compatibility issues') diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index ed5aa0f..6e3809d 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -22,6 +22,7 @@ import telnetlib import yaml from zaza import model +from charmhelpers.core.host import lsb_release from zaza.openstack.utilities import juju as juju_utils from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE @@ -789,3 +790,15 @@ def port_knock_units(units, port=22, expect_success=True): return 'Socket connect failed.' elif connected and not expect_success: return 'Socket connected unexpectedly.' + + +def get_series(unit): + """Ubuntu release name running on unit.""" + result = model.run_on_unit(unit.entity_id, + "lsb_release -cs") + return result['Stdout'].strip() + + +def get_client_series(): + """Ubuntu release name of machine running this function.""" + return lsb_release()['DISTRIB_CODENAME'] From 814ff7ddee24bc11373ebcade2a9c8d6197877b0 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 09:59:01 +1000 Subject: [PATCH 41/50] Don't sleep before attempting get AMQP msg. Retry w/ tenacity --- .../charm_tests/rabbitmq_server/tests.py | 20 +++++++++++++------ .../charm_tests/rabbitmq_server/utils.py | 8 +++++++- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 2ae080a..3aae1f0 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -20,6 +20,7 @@ import time import uuid import juju +import tenacity import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils @@ -32,6 +33,7 @@ from zaza.openstack.utilities.generic import ( ) from . import utils as rmq_utils +from .utils import RmqNoMessageException class RmqTests(test_utils.OpenStackBaseTest): @@ -49,6 +51,15 @@ class RmqTests(test_utils.OpenStackBaseTest): """ return '[{}-{}]'.format(uuid.uuid4(), time.time()) + @tenacity.retry( + retry=tenacity.retry_if_exception_type(RmqNoMessageException), + wait=tenacity.wait_fixed(10), + stop=tenacity.stop_after_attempt(2)) + def _retry_get_amqp_message(self, check_unit, ssl=None, port=None): + return rmq_utils.get_amqp_message_by_unit(check_unit, + ssl=ssl, + port=port) + def _test_rmq_amqp_messages_all_units(self, units, ssl=False, port=None): """Reusable test to send/check amqp messages to every listed rmq unit. @@ -97,18 +108,15 @@ class RmqTests(test_utils.OpenStackBaseTest): amqp_msg, ssl=ssl, port=port) - # Wait a bit before checking for message - time.sleep(10) - # Get amqp message logging.debug('Get message from: {} ' '({} {})'.format(check_unit_host, check_unit_name, check_unit_host_name)) - amqp_msg_rcvd = rmq_utils.get_amqp_message_by_unit(check_unit, - ssl=ssl, - port=port) + amqp_msg_rcvd = self._retry_get_amqp_message(check_unit, + ssl=ssl, + port=port) # Validate amqp message content if amqp_msg == amqp_msg_rcvd: diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index 027ca50..cfb2f4b 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -25,6 +25,12 @@ import ssl as libssl import zaza.openstack.utilities.generic as generic_utils +class RmqNoMessageException(Exception): + """Message retrieval from Rmq resulted in no message.""" + + pass + + def wait_for_cluster(model_name=None, timeout=1200): """Wait for Rmq cluster status to show cluster readiness. @@ -438,7 +444,7 @@ def get_amqp_message_by_unit(unit, queue="test", return body else: msg = 'No message retrieved.' - raise Exception(msg) + raise RmqNoMessageException(msg) def check_unit_cluster_nodes(unit, unit_node_names): From e8068a83565f3e018e0da348a9da4a8a57587eab Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 10:09:22 +1000 Subject: [PATCH 42/50] Port validating Rmq SSL on from time.sleep to Tenacity --- .../charm_tests/rabbitmq_server/utils.py | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index cfb2f4b..edae102 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -19,6 +19,7 @@ import logging import time import pika +import tenacity import zaza.model import ssl as libssl @@ -31,6 +32,11 @@ class RmqNoMessageException(Exception): pass +def _log_tenacity_retry(retry_state): + logging.info('Attempt {}: {}'.format(retry_state.attempt_number, + retry_state.outcome.result())) + + def wait_for_cluster(model_name=None, timeout=1200): """Wait for Rmq cluster status to show cluster readiness. @@ -210,8 +216,16 @@ def validate_ssl_disabled_units(units): return None -def configure_ssl_on(units, model_name=None, - port=None, max_wait=60): +@tenacity.retry( + retry=tenacity.retry_if_result(lambda errors: bool(errors)), + wait=tenacity.wait_fixed(4), + stop=tenacity.stop_after_attempt(15), + after=_log_tenacity_retry) +def _retry_validate_ssl_enabled_units(units, port=None): + return validate_ssl_enabled_units(units, port=port) + + +def configure_ssl_on(units, model_name=None, port=None): """Turn RabbitMQ charm SSL config option on. Turn ssl charm config option on, with optional non-default @@ -219,7 +233,6 @@ def configure_ssl_on(units, model_name=None, unit. :param units: list of units :param port: amqp port, use defaults if None - :param max_wait: maximum time to wait in seconds to confirm :returns: None if successful. Raise on error. """ logging.debug('Setting ssl charm config option: on') @@ -236,15 +249,7 @@ def configure_ssl_on(units, model_name=None, # Wait for unit status wait_for_cluster(model_name) - # Confirm - tries = 0 - ret = validate_ssl_enabled_units(units, port=port) - while ret and tries < (max_wait / 4): - time.sleep(4) - logging.debug('Attempt {}: {}'.format(tries, ret)) - ret = validate_ssl_enabled_units(units, port=port) - tries += 1 - + ret = _retry_validate_ssl_enabled_units(units, port=port) if ret: raise Exception(ret) From d780f76797233b9298d15776388f194753681076 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 10:12:36 +1000 Subject: [PATCH 43/50] Port validating Rmq SSL off from time.sleep to Tenacity --- .../charm_tests/rabbitmq_server/utils.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/utils.py b/zaza/openstack/charm_tests/rabbitmq_server/utils.py index edae102..dba2d92 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/utils.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/utils.py @@ -16,7 +16,6 @@ import json import logging -import time import pika import tenacity @@ -254,6 +253,15 @@ def configure_ssl_on(units, model_name=None, port=None): raise Exception(ret) +@tenacity.retry( + retry=tenacity.retry_if_result(lambda errors: bool(errors)), + wait=tenacity.wait_fixed(4), + stop=tenacity.stop_after_attempt(15), + after=_log_tenacity_retry) +def _retry_validate_ssl_disabled_units(units): + return validate_ssl_disabled_units(units) + + def configure_ssl_off(units, model_name=None, max_wait=60): """Turn RabbitMQ charm SSL config option off. @@ -274,14 +282,7 @@ def configure_ssl_off(units, model_name=None, max_wait=60): # Wait for unit status wait_for_cluster(model_name) - # Confirm - tries = 0 - ret = validate_ssl_disabled_units(units) - while ret and tries < (max_wait / 4): - time.sleep(4) - logging.debug('Attempt {}: {}'.format(tries, ret)) - ret = validate_ssl_disabled_units(units) - tries += 1 + ret = _retry_validate_ssl_disabled_units(units) if ret: raise Exception(ret) From 215b3c87e3b10e56d9ef11f447b2e006954fff67 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 10:14:36 +1000 Subject: [PATCH 44/50] Don't time.sleep waiting for Nagios. Port to tenacity. --- .../charm_tests/rabbitmq_server/tests.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 3aae1f0..3d30d8a 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -257,6 +257,14 @@ class RmqTests(test_utils.OpenStackBaseTest): self.assertIsNone(ret, msg=ret) logging.info('Confirm mgmt port closed on all units (OK)') + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + # sleep for 2mins to allow 1min cron job to run... + wait=tenacity.wait_fixed(120), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_commands_on_units(self, cmds, units): + return generic_utils.check_commands_on_units(cmds, units) + def test_414_rmq_nrpe_monitors(self): """Check rabbimq-server nrpe monitor basic functionality.""" units = zaza.model.get_units(self.application_name) @@ -266,17 +274,14 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('Checking nrpe check_rabbitmq on units...') cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' 'check_rabbitmq.cfg'] - ret = generic_utils.check_commands_on_units(cmds, units) - self.assertIsNone(ret) - - logging.debug('Sleeping 2ms for 1m cron job to run...') - time.sleep(120) + ret = self._retry_check_commands_on_units(cmds, units) + self.assertIsNone(ret, msg=ret) # check_rabbitmq_queue monitor logging.debug('Checking nrpe check_rabbitmq_queue on units...') cmds = ['egrep -oh /usr/local.* /etc/nagios/nrpe.d/' 'check_rabbitmq_queue.cfg'] - ret = generic_utils.check_commands_on_units(cmds, units) + ret = self._retry_check_commands_on_units(cmds, units) self.assertIsNone(ret, msg=ret) # check dat file existence From 060f7392d91fe44278f38cadb220207fcb8b61f7 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 10:32:51 +1000 Subject: [PATCH 45/50] Don't sleep while waiting for cluster status to update. Use tenacity --- .../charm_tests/rabbitmq_server/tests.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 3d30d8a..0c28074 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -368,6 +368,13 @@ class RmqTests(test_utils.OpenStackBaseTest): logging.debug('OK') + @tenacity.retry( + retry=tenacity.retry_if_result(lambda errors: bool(errors)), + wait=tenacity.wait_fixed(10), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_unit_cluster_nodes(self, u, unit_node_names): + return rmq_utils.check_unit_cluster_nodes(u, unit_node_names) + def test_921_remove_unit(self): """Test if unit cleans up when removed from Rmq cluster. @@ -396,15 +403,10 @@ class RmqTests(test_utils.OpenStackBaseTest): errors = [] for u in left_units: - e = rmq_utils.check_unit_cluster_nodes(u, unit_node_names) + e = self._retry_check_unit_cluster_nodes(u, + unit_node_names) if e: - # NOTE: cluster status may not have been updated yet so wait a - # little and try one more time. Need to find a better way to do - # this. - time.sleep(10) - e = rmq_utils.check_unit_cluster_nodes(u, unit_node_names) - if e: - errors.append(e) + errors.append(e) - self.assertFalse(errors) + self.assertFalse(errors, msg=errors) logging.debug('OK') From 1ea0be044058ce0809dcdf57c99a971cbcf4b6fa Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Fri, 4 Oct 2019 10:37:55 +1000 Subject: [PATCH 46/50] Don't time.sleep when checking Rmq ports after toggling mgmt plugin. --- .../charm_tests/rabbitmq_server/tests.py | 32 ++++++++----------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 0c28074..c1ef9de 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -206,6 +206,15 @@ class RmqTests(test_utils.OpenStackBaseTest): ssl=True, port=5999) logging.info('OK') + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + wait=tenacity.wait_fixed(30), + stop=tenacity.stop_after_attempt(20), + after=rmq_utils._log_tenacity_retry) + def _retry_port_knock_units(self, units, port, expect_success=True): + return generic_utils.port_knock_units(units, port, + expect_success=expect_success) + def test_412_rmq_management_plugin(self): """Enable and check management plugin.""" logging.debug('Checking tcp socket connect to management plugin ' @@ -221,14 +230,7 @@ class RmqTests(test_utils.OpenStackBaseTest): rmq_utils.wait_for_cluster() # Check tcp connect to management plugin port - max_wait = 600 - tries = 0 - ret = generic_utils.port_knock_units(units, mgmt_port) - while ret and tries < (max_wait / 30): - time.sleep(30) - logging.debug('Attempt {}: {}'.format(tries, ret)) - ret = generic_utils.port_knock_units(units, mgmt_port) - tries += 1 + ret = self._retry_port_knock_units(units, mgmt_port) self.assertIsNone(ret, msg=ret) logging.debug('Connect to all units (OK)') @@ -242,17 +244,9 @@ class RmqTests(test_utils.OpenStackBaseTest): # Negative check - tcp connect to management plugin port logging.info('Expect tcp connect fail since charm config ' 'option is disabled.') - tries = 0 - ret = generic_utils.port_knock_units(units, - mgmt_port, - expect_success=False) - - while ret and tries < (max_wait / 30): - time.sleep(30) - logging.debug('Attempt {}: {}'.format(tries, ret)) - ret = generic_utils.port_knock_units(units, mgmt_port, - expect_success=False) - tries += 1 + ret = self._retry_port_knock_units(units, + mgmt_port, + expect_success=False) self.assertIsNone(ret, msg=ret) logging.info('Confirm mgmt port closed on all units (OK)') From d2e12abb9153cea04d7563fd6b81a6fb140a842d Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Thu, 10 Oct 2019 11:06:26 +1100 Subject: [PATCH 47/50] Remove condition: check client series will always be true. All CI test runners are >= xenial. The condition will always be true. --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 12 +++--------- zaza/openstack/utilities/generic.py | 6 ------ 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index c1ef9de..b114314 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -26,11 +26,7 @@ import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils from charmhelpers.core.host import CompareHostReleases - -from zaza.openstack.utilities.generic import ( - get_series, - get_client_series, -) +from zaza.openstack.utilities.generic import get_series from . import utils as rmq_utils from .utils import RmqNoMessageException @@ -169,8 +165,7 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 - if (CompareHostReleases(get_client_series()) >= 'xenial' and - CompareHostReleases(get_series(units[0])) <= 'trusty'): + if CompareHostReleases(get_series(units[0])) <= 'trusty': logging.info('SKIP') logging.info('Skipping SSL tests due to client' ' compatibility issues') @@ -192,8 +187,7 @@ class RmqTests(test_utils.OpenStackBaseTest): units = zaza.model.get_units(self.application_name) # http://pad.lv/1625044 - if (CompareHostReleases(get_client_series()) >= 'xenial' and - CompareHostReleases(get_series(units[0])) <= 'trusty'): + if CompareHostReleases(get_series(units[0])) <= 'trusty': logging.info('SKIP') logging.info('Skipping SSL tests due to client' ' compatibility issues') diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index 6e3809d..174b47a 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -22,7 +22,6 @@ import telnetlib import yaml from zaza import model -from charmhelpers.core.host import lsb_release from zaza.openstack.utilities import juju as juju_utils from zaza.openstack.utilities import exceptions as zaza_exceptions from zaza.openstack.utilities.os_versions import UBUNTU_OPENSTACK_RELEASE @@ -797,8 +796,3 @@ def get_series(unit): result = model.run_on_unit(unit.entity_id, "lsb_release -cs") return result['Stdout'].strip() - - -def get_client_series(): - """Ubuntu release name of machine running this function.""" - return lsb_release()['DISTRIB_CODENAME'] From 009f37ab0b4e357d16eaa178b71b60deb217f5a5 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Tue, 15 Oct 2019 12:48:21 +1100 Subject: [PATCH 48/50] Add unit test for generic util function: get_unit_hostnames --- .../utilities/test_zaza_utilities_generic.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index 4869ae4..a4eed09 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -572,3 +572,33 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.telnet.side_effect = generic_utils.socket.error self.assertFalse(generic_utils.is_port_open(_port, _addr)) + + def test_get_unit_hostnames(self): + self.patch( + "zaza.openstack.utilities.generic.model.run_on_unit", + new_callable=mock.MagicMock(), + name="_run" + ) + + _unit1 = mock.MagicMock() + _unit1.entity_id = "testunit/1" + _unit2 = mock.MagicMock() + _unit2.entity_id = "testunit/2" + + _hostname1 = "host1.domain" + _hostname2 = "host2.domain" + + expected = { + _unit1.entity_id: _hostname1, + _unit2.entity_id: _hostname2, + } + + _units = [_unit1, _unit2] + + self._run.side_effect = [{"Stdout": _hostname1}, + {"Stdout": _hostname2}] + + actual = generic_utils.get_unit_hostnames(_units) + + self.assertEqual(actual, expected) + From 6402d5a0212e2c4a6595b6a3b0e1071e459c5762 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Tue, 15 Oct 2019 12:50:56 +1100 Subject: [PATCH 49/50] Add unit test for generic util function: port_knock_units --- .../utilities/test_zaza_utilities_generic.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index a4eed09..c992e03 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -602,3 +602,29 @@ class TestGenericUtils(ut_utils.BaseTestCase): self.assertEqual(actual, expected) + def test_port_knock_units(self): + self.patch( + "zaza.openstack.utilities.generic.is_port_open", + new_callable=mock.MagicMock(), + name="_is_port_open" + ) + + _units = [ + mock.MagicMock(), + mock.MagicMock(), + ] + + self._is_port_open.side_effect = [True, True] + self.assertIsNone(generic_utils.port_knock_units(_units)) + self.assertEqual(self._is_port_open.call_count, len(_units)) + + self._is_port_open.side_effect = [True, False] + self.assertIsNotNone(generic_utils.port_knock_units(_units)) + + # check when func is expecting failure, i.e. should succeed + self._is_port_open.reset_mock() + self._is_port_open.side_effect = [False, False] + self.assertIsNone(generic_utils.port_knock_units(_units, + expect_success=False)) + self.assertEqual(self._is_port_open.call_count, len(_units)) + From 596c347be4a37e0b0e917e887f5f9f5c13e2b1a9 Mon Sep 17 00:00:00 2001 From: Edin Sarajlic Date: Tue, 15 Oct 2019 12:51:12 +1100 Subject: [PATCH 50/50] Add unit test for generic util function: check_commands_on_units --- .../utilities/test_zaza_utilities_generic.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/unit_tests/utilities/test_zaza_utilities_generic.py b/unit_tests/utilities/test_zaza_utilities_generic.py index c992e03..4c09d2b 100644 --- a/unit_tests/utilities/test_zaza_utilities_generic.py +++ b/unit_tests/utilities/test_zaza_utilities_generic.py @@ -628,3 +628,31 @@ class TestGenericUtils(ut_utils.BaseTestCase): expect_success=False)) self.assertEqual(self._is_port_open.call_count, len(_units)) + def test_check_commands_on_units(self): + self.patch( + "zaza.openstack.utilities.generic.model.run_on_unit", + new_callable=mock.MagicMock(), + name="_run" + ) + + num_units = 2 + _units = [mock.MagicMock() for i in range(num_units)] + + num_cmds = 3 + cmds = ["/usr/bin/fakecmd"] * num_cmds + + # Test success, all calls return 0 + # zero is a string to replicate run_on_unit return data type + _cmd_results = [{"Code": "0"}] * len(_units) * len(cmds) + self._run.side_effect = _cmd_results + + result = generic_utils.check_commands_on_units(cmds, _units) + self.assertIsNone(result) + self.assertEqual(self._run.call_count, len(_units) * len(cmds)) + + # Test failure, some call returns 1 + _cmd_results[2] = {"Code": "1"} + self._run.side_effect = _cmd_results + + result = generic_utils.check_commands_on_units(cmds, _units) + self.assertIsNotNone(result)