Merge pull request #621 from AurelienLourot/vault-validate-ca-more-robust

Make vault/setup/validate_ca() more robust

zaza/openstack/charm_tests/vault/setup.py

@@ -17,14 +17,12 @@
 import base64
 import functools
 import logging
-import requests
 import tempfile
 
 import zaza.charm_lifecycle.utils as lifecycle_utils
 import zaza.openstack.charm_tests.vault.utils as vault_utils
 import zaza.model
 import zaza.openstack.utilities.cert
-import zaza.openstack.utilities.openstack
 import zaza.openstack.utilities.generic
 import zaza.openstack.utilities.exceptions as zaza_exceptions
 import zaza.utilities.juju as juju_utils
@@ -95,7 +93,7 @@ def mojo_unseal_by_unit():
 def unseal_by_unit(cacert=None):
     """Unseal any units reported as sealed using mojo cacert."""
     cacert = cacert or get_cacert_file()
-    vault_creds = vault_utils.get_credentails()
+    vault_creds = vault_utils.get_credentials()
     for client in vault_utils.get_clients(cacert=cacert):
         if client.hvac_client.is_sealed():
             client.hvac_client.unseal(vault_creds['keys'][0])
@@ -126,7 +124,7 @@ async def async_mojo_unseal_by_unit():
 async def async_unseal_by_unit(cacert=None):
     """Unseal any units reported as sealed using vault cacert."""
     cacert = cacert or get_cacert_file()
-    vault_creds = vault_utils.get_credentails()
+    vault_creds = vault_utils.get_credentials()
     for client in vault_utils.get_clients(cacert=cacert):
         if client.hvac_client.is_sealed():
             client.hvac_client.unseal(vault_creds['keys'][0])
@@ -222,16 +220,4 @@ def validate_ca(cacertificate, application="keystone", port=5000):
     :returns: None
     :rtype: None
     """
-    zaza.openstack.utilities.openstack.block_until_ca_exists(
-        application,
-        cacertificate.decode().strip())
-    vip = (zaza.model.get_application_config(application)
-           .get("vip").get("value"))
-    if vip:
-        ip = vip
-    else:
-        ip = zaza.model.get_app_ips(application)[0]
-    with tempfile.NamedTemporaryFile(mode='w') as fp:
-        fp.write(cacertificate.decode())
-        fp.flush()
-        requests.get('https://{}:{}'.format(ip, str(port)), verify=fp.name)
+    vault_utils.validate_ca(cacertificate, application, port)

zaza/openstack/charm_tests/vault/tests.py

@@ -21,9 +21,7 @@ import json
 import logging
 import unittest
 import uuid
-import tempfile
 
-import requests
 import tenacity
 from hvac.exceptions import InternalServerError
 
@@ -64,7 +62,7 @@ class BaseVaultTest(test_utils.OpenStackBaseTest):
         cls.vip_client = vault_utils.get_vip_client()
         if cls.vip_client:
             cls.clients.append(cls.vip_client)
-        cls.vault_creds = vault_utils.get_credentails()
+        cls.vault_creds = vault_utils.get_credentials()
         vault_utils.unseal_all(cls.clients, cls.vault_creds['keys'][0])
         vault_utils.auth_all(cls.clients, cls.vault_creds['root_token'])
         vault_utils.ensure_secret_backend(cls.clients[0])
@@ -180,26 +178,10 @@ class VaultTest(BaseVaultTest):
         except KeyError:
             # Already removed
             pass
-        zaza.openstack.utilities.openstack.block_until_ca_exists(
-            'keystone',
-            cacert.decode().strip())
         zaza.model.wait_for_application_states(
             states=test_config.get('target_deploy_status', {}))
-        ip = zaza.model.get_app_ips(
-            'keystone')[0]
 
-        with tempfile.NamedTemporaryFile(mode='w') as fp:
-            fp.write(cacert.decode())
-            fp.flush()
-            # Avoid race condition and retry
-            for attempt in tenacity.Retrying(
-                stop=tenacity.stop_after_attempt(3),
-                wait=tenacity.wait_exponential(
-                    multiplier=2, min=2, max=10)):
-                with attempt:
-                    logging.info(
-                        "Attempting to connect to https://{}:5000".format(ip))
-                    requests.get('https://{}:5000'.format(ip), verify=fp.name)
+        vault_utils.validate_ca(cacert)
 
     def test_all_clients_authenticated(self):
         """Check all vault clients are authenticated."""

zaza/openstack/charm_tests/vault/utils.py

@@ -18,6 +18,7 @@
 
 import base64
 import hvac
+import logging
 import requests
 import tempfile
 import urllib3
@@ -27,6 +28,7 @@ import tenacity
 import collections
 
 import zaza.model
+import zaza.openstack.utilities.openstack
 import zaza.utilities.networking as network_utils
 
 AUTH_FILE = "vault_tests.yaml"
@@ -70,10 +72,10 @@ class VaultFacade:
     def initialize(self):
         """Initialise vault and store resulting credentials."""
         if self.is_initialized:
-            self.vault_creds = get_credentails()
+            self.vault_creds = get_credentials()
         else:
             self.vault_creds = init_vault(self.unseal_client)
-            store_credentails(self.vault_creds)
+            store_credentials(self.vault_creds)
         self.initialized = is_initialized(self.unseal_client)
 
     def unseal(self):
@@ -294,7 +296,7 @@ def find_unit_with_creds():
     return unit
 
 
-def get_credentails():
+def get_credentials():
     """Retrieve vault token and keys from unit.
 
     Retrieve vault token and keys from unit. These are stored on a unit
@@ -315,7 +317,7 @@ def get_credentails():
     return creds
 
 
-def store_credentails(creds):
+def store_credentials(creds):
     """Store the supplied credentials.
 
     Store the supplied credentials on a vault unit. ONLY USE FOR FUNCTIONAL
@@ -334,7 +336,7 @@ def store_credentails(creds):
             '~/{}'.format(AUTH_FILE))
 
 
-def get_credentails_from_file(auth_file):
+def get_credentials_from_file(auth_file):
     """Read the vault credentials from the auth_file.
 
     :param auth_file: Path to file with credentials
@@ -347,7 +349,7 @@ def get_credentails_from_file(auth_file):
     return vault_creds
 
 
-def write_credentails(auth_file, vault_creds):
+def write_credentials(auth_file, vault_creds):
     """Write the vault credentials to the auth_file.
 
     :param auth_file: Path to file to write credentials
@@ -434,3 +436,37 @@ def run_upload_signed_csr(pem, root_ca, allowed_domains):
             'root-ca': base64.b64encode(root_ca).decode(),
             'allowed-domains=': allowed_domains,
             'ttl': '24h'})
+
+
+@tenacity.retry(
+    reraise=True,
+    wait=tenacity.wait_exponential(multiplier=2, min=2, max=10),
+    stop=tenacity.stop_after_attempt(3))
+def validate_ca(cacertificate, application="keystone", port=5000):
+    """Validate Certificate Authority against application.
+
+    :param cacertificate: PEM formatted CA certificate
+    :type cacertificate: str
+    :param application: Which application to validate against.
+    :type application: str
+    :param port: Port to validate against.
+    :type port: int
+    :returns: None
+    :rtype: None
+    """
+    zaza.openstack.utilities.openstack.block_until_ca_exists(
+        application,
+        cacertificate.decode().strip())
+    vip = (zaza.model.get_application_config(application)
+           .get("vip").get("value"))
+    if vip:
+        ip = vip
+    else:
+        ip = zaza.model.get_app_ips(application)[0]
+    with tempfile.NamedTemporaryFile(mode='w') as fp:
+        fp.write(cacertificate.decode())
+        fp.flush()
+        keystone_url = 'https://{}:{}'.format(ip, str(port))
+        logging.info(
+            'Attempting to connect to {}'.format(keystone_url))
+        requests.get(keystone_url, verify=fp.name)