From 2cab47dec252cf7e030d4a553afa7bf510621b44 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Fri, 2 Oct 2020 18:06:21 +1300 Subject: [PATCH 1/4] Add NRPE check tests for Designate Adds a simple test for the Designate application to ensure that the NRPE checks for services have been created. Related-Bug: #1897809 --- zaza/openstack/charm_tests/designate/tests.py | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 3c56ee1..c01014e 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -24,6 +24,7 @@ import designateclient.v1.servers as servers import zaza.model import zaza.openstack.utilities.juju as zaza_juju +import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.designate.utils as designate_utils @@ -88,6 +89,14 @@ class BaseDesignateTest(test_utils.OpenStackBaseTest): cls.server_create = cls.designate.servers.create cls.server_delete = cls.designate.servers.delete + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + # sleep for 2mins to allow 1min cron job to run... + wait=tenacity.wait_fixed(120), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_commands_on_units(self, cmds, units): + return generic_utils.check_commands_on_units(cmds, units) + class DesignateAPITests(BaseDesignateTest): """Tests interact with designate api.""" @@ -257,7 +266,26 @@ class DesignateCharmTests(BaseDesignateTest): logging.info("Testing pause resume") -class DesignateTests(DesignateAPITests, DesignateCharmTests): +class DesignateMonitoringTests(BaseDesignateTest): + """Designate charm monitoring tests.""" + + def test_nrpe_configured(self): + """Confirm that the NRPE service check files are created.""" + units = zaza.model.get_units(self.application_name) + cmds = [] + for check_name in self.designate_svcs: + cmds.append( + 'egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_{}.cfg'.format(check_name) + ) + ret = self._retry_check_commands_on_units(cmds, units) + if ret: + logging.info(ret) + self.assertIsNone(ret, msg=ret) + + +class DesignateTests(DesignateAPITests, DesignateCharmTests, + DesignateMonitoringTests): """Collection of all Designate test classes.""" pass From 66d08c0866cdf6929d79cfba2ee79d197f4bfe30 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 13 Aug 2021 10:44:32 +0200 Subject: [PATCH 2/4] Make vault/setup/validate_ca() more robust --- zaza/openstack/charm_tests/vault/setup.py | 20 ++-------- zaza/openstack/charm_tests/vault/tests.py | 22 +---------- zaza/openstack/charm_tests/vault/utils.py | 48 ++++++++++++++++++++--- 3 files changed, 47 insertions(+), 43 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index c792508..87917bc 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -17,14 +17,12 @@ import base64 import functools import logging -import requests import tempfile import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.openstack.charm_tests.vault.utils as vault_utils import zaza.model import zaza.openstack.utilities.cert -import zaza.openstack.utilities.openstack import zaza.openstack.utilities.generic import zaza.openstack.utilities.exceptions as zaza_exceptions import zaza.utilities.juju as juju_utils @@ -95,7 +93,7 @@ def mojo_unseal_by_unit(): def unseal_by_unit(cacert=None): """Unseal any units reported as sealed using mojo cacert.""" cacert = cacert or get_cacert_file() - vault_creds = vault_utils.get_credentails() + vault_creds = vault_utils.get_credentials() for client in vault_utils.get_clients(cacert=cacert): if client.hvac_client.is_sealed(): client.hvac_client.unseal(vault_creds['keys'][0]) @@ -126,7 +124,7 @@ async def async_mojo_unseal_by_unit(): async def async_unseal_by_unit(cacert=None): """Unseal any units reported as sealed using vault cacert.""" cacert = cacert or get_cacert_file() - vault_creds = vault_utils.get_credentails() + vault_creds = vault_utils.get_credentials() for client in vault_utils.get_clients(cacert=cacert): if client.hvac_client.is_sealed(): client.hvac_client.unseal(vault_creds['keys'][0]) @@ -222,16 +220,4 @@ def validate_ca(cacertificate, application="keystone", port=5000): :returns: None :rtype: None """ - zaza.openstack.utilities.openstack.block_until_ca_exists( - application, - cacertificate.decode().strip()) - vip = (zaza.model.get_application_config(application) - .get("vip").get("value")) - if vip: - ip = vip - else: - ip = zaza.model.get_app_ips(application)[0] - with tempfile.NamedTemporaryFile(mode='w') as fp: - fp.write(cacertificate.decode()) - fp.flush() - requests.get('https://{}:{}'.format(ip, str(port)), verify=fp.name) + vault_utils.validate_ca(cacertificate, application, port) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index a83440f..4ba2e58 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -21,9 +21,7 @@ import json import logging import unittest import uuid -import tempfile -import requests import tenacity from hvac.exceptions import InternalServerError @@ -64,7 +62,7 @@ class BaseVaultTest(test_utils.OpenStackBaseTest): cls.vip_client = vault_utils.get_vip_client() if cls.vip_client: cls.clients.append(cls.vip_client) - cls.vault_creds = vault_utils.get_credentails() + cls.vault_creds = vault_utils.get_credentials() vault_utils.unseal_all(cls.clients, cls.vault_creds['keys'][0]) vault_utils.auth_all(cls.clients, cls.vault_creds['root_token']) vault_utils.ensure_secret_backend(cls.clients[0]) @@ -180,26 +178,10 @@ class VaultTest(BaseVaultTest): except KeyError: # Already removed pass - zaza.openstack.utilities.openstack.block_until_ca_exists( - 'keystone', - cacert.decode().strip()) zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) - ip = zaza.model.get_app_ips( - 'keystone')[0] - with tempfile.NamedTemporaryFile(mode='w') as fp: - fp.write(cacert.decode()) - fp.flush() - # Avoid race condition and retry - for attempt in tenacity.Retrying( - stop=tenacity.stop_after_attempt(3), - wait=tenacity.wait_exponential( - multiplier=2, min=2, max=10)): - with attempt: - logging.info( - "Attempting to connect to https://{}:5000".format(ip)) - requests.get('https://{}:5000'.format(ip), verify=fp.name) + vault_utils.validate_ca(cacert) def test_all_clients_authenticated(self): """Check all vault clients are authenticated.""" diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index b4b5579..c29c745 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -18,6 +18,7 @@ import base64 import hvac +import logging import requests import tempfile import urllib3 @@ -27,6 +28,7 @@ import tenacity import collections import zaza.model +import zaza.openstack.utilities.openstack import zaza.utilities.networking as network_utils AUTH_FILE = "vault_tests.yaml" @@ -70,10 +72,10 @@ class VaultFacade: def initialize(self): """Initialise vault and store resulting credentials.""" if self.is_initialized: - self.vault_creds = get_credentails() + self.vault_creds = get_credentials() else: self.vault_creds = init_vault(self.unseal_client) - store_credentails(self.vault_creds) + store_credentials(self.vault_creds) self.initialized = is_initialized(self.unseal_client) def unseal(self): @@ -294,7 +296,7 @@ def find_unit_with_creds(): return unit -def get_credentails(): +def get_credentials(): """Retrieve vault token and keys from unit. Retrieve vault token and keys from unit. These are stored on a unit @@ -315,7 +317,7 @@ def get_credentails(): return creds -def store_credentails(creds): +def store_credentials(creds): """Store the supplied credentials. Store the supplied credentials on a vault unit. ONLY USE FOR FUNCTIONAL @@ -334,7 +336,7 @@ def store_credentails(creds): '~/{}'.format(AUTH_FILE)) -def get_credentails_from_file(auth_file): +def get_credentials_from_file(auth_file): """Read the vault credentials from the auth_file. :param auth_file: Path to file with credentials @@ -347,7 +349,7 @@ def get_credentails_from_file(auth_file): return vault_creds -def write_credentails(auth_file, vault_creds): +def write_credentials(auth_file, vault_creds): """Write the vault credentials to the auth_file. :param auth_file: Path to file to write credentials @@ -434,3 +436,37 @@ def run_upload_signed_csr(pem, root_ca, allowed_domains): 'root-ca': base64.b64encode(root_ca).decode(), 'allowed-domains=': allowed_domains, 'ttl': '24h'}) + + +@tenacity.retry( + reraise=True, + wait=tenacity.wait_exponential(multiplier=2, min=2, max=10), + stop=tenacity.stop_after_attempt(3)) +def validate_ca(cacertificate, application="keystone", port=5000): + """Validate Certificate Authority against application. + + :param cacertificate: PEM formatted CA certificate + :type cacertificate: str + :param application: Which application to validate against. + :type application: str + :param port: Port to validate against. + :type port: int + :returns: None + :rtype: None + """ + zaza.openstack.utilities.openstack.block_until_ca_exists( + application, + cacertificate.decode().strip()) + vip = (zaza.model.get_application_config(application) + .get("vip").get("value")) + if vip: + ip = vip + else: + ip = zaza.model.get_app_ips(application)[0] + with tempfile.NamedTemporaryFile(mode='w') as fp: + fp.write(cacertificate.decode()) + fp.flush() + keystone_url = 'https://{}:{}'.format(ip, str(port)) + logging.info( + 'Attempting to connect to {}'.format(keystone_url)) + requests.get(keystone_url, verify=fp.name) From c759a23962d2de3c7c67d45841114ceb1ae1cf7e Mon Sep 17 00:00:00 2001 From: coreycb Date: Tue, 24 Aug 2021 09:48:55 -0400 Subject: [PATCH 3/4] Do not update external network data port if already set (#625) --- zaza/openstack/utilities/openstack.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index fc0d914..a4fcbc3 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -986,8 +986,11 @@ def configure_networking_charms(networking_data, macs, use_juju_wait=True): current_data_port = get_application_config_option( application_name, networking_data.port_config_key) - if current_data_port == config[networking_data.port_config_key]: - logging.info('Config already set to value') + if current_data_port: + logging.info("Skip update of external network data port config." + "Config '{}' already set to value: {}".format( + networking_data.port_config_key, + current_data_port)) return model.set_application_config( From 5baf16237f1a6fc6805324237f59c002ebc03e19 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 31 Aug 2021 13:04:25 +0000 Subject: [PATCH 4/4] Add certificate check for Ceph dashboard Add setup step for the dashboard which will block until the certificates are present and the model is idle. This is to prevent the tests from continuing when the certificates are not ready. Also up CephDashboardTest to use standard tools for collecting ca cert. --- .../charm_tests/ceph/dashboard/setup.py | 34 +++++++++++++++++++ .../charm_tests/ceph/dashboard/tests.py | 20 ++--------- 2 files changed, 37 insertions(+), 17 deletions(-) create mode 100644 zaza/openstack/charm_tests/ceph/dashboard/setup.py diff --git a/zaza/openstack/charm_tests/ceph/dashboard/setup.py b/zaza/openstack/charm_tests/ceph/dashboard/setup.py new file mode 100644 index 0000000..7be7e7d --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/dashboard/setup.py @@ -0,0 +1,34 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for setting up Ceph Dashboard.""" + +import logging + +import zaza.model +import zaza.openstack.utilities.openstack + + +def check_dashboard_cert(model_name=None): + """Wait for Dashboard to be ready. + + :param model_name: Name of model to query. + :type model_name: str + """ + logging.info("Check dashbaord Waiting for cacert") + zaza.openstack.utilities.openstack.block_until_ca_exists( + 'ceph-dashboard', + 'CERTIFICATE', + model_name=model_name) + zaza.model.block_until_all_units_idle(model_name=model_name) diff --git a/zaza/openstack/charm_tests/ceph/dashboard/tests.py b/zaza/openstack/charm_tests/ceph/dashboard/tests.py index e7c8863..1727699 100644 --- a/zaza/openstack/charm_tests/ceph/dashboard/tests.py +++ b/zaza/openstack/charm_tests/ceph/dashboard/tests.py @@ -15,12 +15,11 @@ """Encapsulating `ceph-dashboard` testing.""" import collections -import os import requests import zaza import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.utilities.deployment_env as deployment_env +import zaza.openstack.utilities.openstack as openstack_utils class CephDashboardTest(test_utils.BaseCharmTest): @@ -34,21 +33,8 @@ class CephDashboardTest(test_utils.BaseCharmTest): """Run class setup for running ceph dashboard tests.""" super().setUpClass() cls.application_name = 'ceph-dashboard' - cls.local_ca_cert = cls.collect_ca() - - @classmethod - def collect_ca(cls): - """Collect CA from ceph-dashboard unit.""" - local_ca_cert = os.path.join( - deployment_env.get_tmpdir(), - os.path.basename(cls.REMOTE_CERT_FILE)) - if not os.path.isfile(local_ca_cert): - units = zaza.model.get_units(cls.application_name) - zaza.model.scp_from_unit( - units[0].entity_id, - cls.REMOTE_CERT_FILE, - local_ca_cert) - return local_ca_cert + cls.local_ca_cert = openstack_utils.get_remote_ca_cert_file( + cls.application_name) def test_dashboard_units(self): """Check dashboard units are configured correctly."""