a2d8e96d8e
The tests here run a series of commands on the vault client, without waiting in between. This is usually fine, but with the raft backend, there can be delays between initialising, unsealing, and the raft cluster settling. During this time, most vault commands will return errors.
352 lines
13 KiB
Python
352 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# Copyright 2018 Canonical Ltd.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Collection of tests for vault."""
|
|
|
|
import contextlib
|
|
import json
|
|
import logging
|
|
import unittest
|
|
import uuid
|
|
|
|
import tenacity
|
|
from hvac.exceptions import InternalServerError
|
|
|
|
import zaza.charm_lifecycle.utils as lifecycle_utils
|
|
import zaza.openstack.charm_tests.test_utils as test_utils
|
|
import zaza.openstack.charm_tests.vault.utils as vault_utils
|
|
import zaza.openstack.utilities.cert
|
|
import zaza.openstack.utilities.openstack
|
|
import zaza.model
|
|
import zaza.utilities.juju as juju_utils
|
|
|
|
|
|
@tenacity.retry(
|
|
retry=tenacity.retry_if_exception_type(InternalServerError),
|
|
retry_error_callback=lambda retry_state: False,
|
|
wait=tenacity.wait_fixed(2), # interval between retries
|
|
stop=tenacity.stop_after_attempt(10)) # retry 10 times
|
|
def retry_hvac_client_authenticated(client):
|
|
"""Check hvac client is authenticated with retry.
|
|
|
|
If is_authenticated() raise exception for all retries,
|
|
return False(which is done by `retry_error_callback`).
|
|
Otherwise, return whatever the returned value.
|
|
"""
|
|
return client.hvac_client.is_authenticated()
|
|
|
|
|
|
class BaseVaultTest(test_utils.OpenStackBaseTest):
|
|
"""Base class for vault tests."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
"""Run setup for Vault tests."""
|
|
cls.model_name = zaza.model.get_juju_model()
|
|
cls.lead_unit = zaza.model.get_lead_unit_name(
|
|
"vault", model_name=cls.model_name)
|
|
cls.clients = vault_utils.get_clients()
|
|
cls.vip_client = vault_utils.get_vip_client()
|
|
if cls.vip_client:
|
|
cls.clients.append(cls.vip_client)
|
|
cls.vault_creds = vault_utils.get_credentials()
|
|
|
|
# This little dance is to ensure a correct init and unseal sequence,
|
|
# for the case of vault with the raft backend.
|
|
# It will also work fine in other cases.
|
|
# The wait functions will raise AssertionErrors on timeouts.
|
|
init_client = vault_utils.wait_and_get_initialized_client(cls.clients)
|
|
vault_utils.unseal_all([init_client], cls.vault_creds['keys'][0])
|
|
vault_utils.wait_until_all_initialised(cls.clients)
|
|
vault_utils.unseal_all(cls.clients, cls.vault_creds['keys'][0])
|
|
|
|
vault_utils.auth_all(cls.clients, cls.vault_creds['root_token'])
|
|
vault_utils.wait_for_ha_settled(cls.clients)
|
|
vault_utils.ensure_secret_backend(cls.clients[0])
|
|
|
|
def tearDown(self):
|
|
"""Tun test cleanup for Vault tests."""
|
|
vault_utils.unseal_all(self.clients, self.vault_creds['keys'][0])
|
|
|
|
@contextlib.contextmanager
|
|
def pause_resume(self, services, pgrep_full=False):
|
|
"""Override pause_resume for Vault behavior."""
|
|
zaza.model.block_until_service_status(
|
|
self.lead_unit,
|
|
services,
|
|
'running',
|
|
model_name=self.model_name)
|
|
zaza.model.block_until_unit_wl_status(
|
|
self.lead_unit,
|
|
'active',
|
|
model_name=self.model_name)
|
|
zaza.model.block_until_all_units_idle(model_name=self.model_name)
|
|
zaza.model.run_action(
|
|
self.lead_unit,
|
|
'pause',
|
|
model_name=self.model_name)
|
|
zaza.model.block_until_service_status(
|
|
self.lead_unit,
|
|
services,
|
|
'blocked', # Service paused
|
|
model_name=self.model_name)
|
|
yield
|
|
zaza.model.run_action(
|
|
self.lead_unit,
|
|
'resume',
|
|
model_name=self.model_name)
|
|
zaza.model.block_until_service_status(
|
|
self.lead_unit,
|
|
services,
|
|
'blocked', # Service sealed
|
|
model_name=self.model_name)
|
|
|
|
|
|
class UnsealVault(BaseVaultTest):
|
|
"""Unseal Vault only.
|
|
|
|
Useful for bootstrapping Vault when it is present in test bundles for other
|
|
charms.
|
|
"""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
"""Run setup for UnsealVault class."""
|
|
super(UnsealVault, cls).setUpClass()
|
|
|
|
def test_unseal(self, test_config=None):
|
|
"""Unseal Vault.
|
|
|
|
:param test_config: (Optional) Zaza test config
|
|
:type test_config: charm_lifecycle.utils.get_charm_config()
|
|
"""
|
|
vault_utils.run_charm_authorize(self.vault_creds['root_token'])
|
|
if not test_config:
|
|
test_config = lifecycle_utils.get_charm_config()
|
|
try:
|
|
del test_config['target_deploy_status']['vault']
|
|
except KeyError:
|
|
# Already removed
|
|
pass
|
|
zaza.model.wait_for_application_states(
|
|
states=test_config.get('target_deploy_status', {}))
|
|
|
|
|
|
class VaultTest(BaseVaultTest):
|
|
"""Encapsulate vault tests."""
|
|
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
"""Run setup for Vault tests."""
|
|
super(VaultTest, cls).setUpClass()
|
|
|
|
def test_csr(self):
|
|
"""Test generating a csr and uploading a signed certificate."""
|
|
vault_actions = zaza.model.get_actions(
|
|
'vault')
|
|
if 'get-csr' not in vault_actions:
|
|
raise unittest.SkipTest('Action not defined')
|
|
try:
|
|
zaza.model.get_application(
|
|
'keystone')
|
|
except KeyError:
|
|
raise unittest.SkipTest('No client to test csr')
|
|
action = vault_utils.run_charm_authorize(
|
|
self.vault_creds['root_token'])
|
|
action = vault_utils.run_get_csr()
|
|
|
|
intermediate_csr = action.data['results']['output']
|
|
(cakey, cacert) = zaza.openstack.utilities.cert.generate_cert(
|
|
'DivineAuthority',
|
|
generate_ca=True)
|
|
intermediate_cert = zaza.openstack.utilities.cert.sign_csr(
|
|
intermediate_csr,
|
|
cakey.decode(),
|
|
cacert.decode(),
|
|
generate_ca=True)
|
|
action = vault_utils.run_upload_signed_csr(
|
|
pem=intermediate_cert,
|
|
root_ca=cacert,
|
|
allowed_domains='openstack.local')
|
|
|
|
test_config = lifecycle_utils.get_charm_config()
|
|
try:
|
|
del test_config['target_deploy_status']['vault']
|
|
except KeyError:
|
|
# Already removed
|
|
pass
|
|
zaza.model.wait_for_application_states(
|
|
states=test_config.get('target_deploy_status', {}))
|
|
|
|
vault_utils.validate_ca(cacert)
|
|
|
|
def test_all_clients_authenticated(self):
|
|
"""Check all vault clients are authenticated."""
|
|
for client in self.clients:
|
|
self.assertTrue(retry_hvac_client_authenticated(client))
|
|
|
|
def check_read(self, key, value):
|
|
"""Check reading the key from all vault units."""
|
|
for client in self.clients:
|
|
self.assertEqual(
|
|
client.hvac_client.read('secret/uuids')['data']['uuid'],
|
|
value)
|
|
|
|
def test_consistent_read_write(self):
|
|
"""Test reading and writing data to vault."""
|
|
key = 'secret/uuids'
|
|
for client in self.clients:
|
|
value = str(uuid.uuid1())
|
|
client.hvac_client.write(key, uuid=value, lease='1h')
|
|
# Now check all clients read the same value back
|
|
self.check_read(key, value)
|
|
|
|
@test_utils.skipIfNotHA('vault')
|
|
def test_vault_ha_statuses(self):
|
|
"""Check Vault charm HA status."""
|
|
leader = []
|
|
leader_address = []
|
|
leader_cluster_address = []
|
|
for client in self.clients:
|
|
self.assertTrue(client.hvac_client.ha_status['ha_enabled'])
|
|
leader_address.append(
|
|
client.hvac_client.ha_status['leader_address'])
|
|
leader_cluster_address.append(
|
|
client.hvac_client.ha_status['leader_cluster_address'])
|
|
if (client.hvac_client.ha_status['is_self'] and not
|
|
client.vip_client):
|
|
leader.append(client.addr)
|
|
# Check there is exactly one leader
|
|
self.assertEqual(len(leader), 1)
|
|
# Check both cluster addresses match accross the cluster
|
|
self.assertEqual(len(set(leader_address)), 1)
|
|
self.assertEqual(len(set(leader_cluster_address)), 1)
|
|
|
|
def test_check_vault_status(self):
|
|
"""Check Vault charm status."""
|
|
for client in self.clients:
|
|
self.assertFalse(client.hvac_client.seal_status['sealed'])
|
|
self.assertTrue(client.hvac_client.seal_status['cluster_name'])
|
|
|
|
def test_vault_authorize_charm_action(self):
|
|
"""Test the authorize_charm action."""
|
|
vault_actions = zaza.model.get_actions(
|
|
'vault')
|
|
if 'authorize-charm' not in vault_actions:
|
|
raise unittest.SkipTest('Action not defined')
|
|
action = vault_utils.run_charm_authorize(
|
|
self.vault_creds['root_token'])
|
|
self.assertEqual(action.status, 'completed')
|
|
client = self.clients[0]
|
|
self.assertIn(
|
|
'local-charm-policy',
|
|
client.hvac_client.list_policies())
|
|
|
|
def test_zzz_pause_resume(self):
|
|
"""Run pause and resume tests.
|
|
|
|
Pause service and check services are stopped, then resume and check
|
|
they are started.
|
|
"""
|
|
vault_actions = zaza.model.get_actions(
|
|
'vault')
|
|
if 'pause' not in vault_actions or 'resume' not in vault_actions:
|
|
raise unittest.SkipTest("The version of charm-vault tested does "
|
|
"not have pause/resume actions")
|
|
# this pauses and resumes the LEAD unit
|
|
with self.pause_resume(['vault']):
|
|
logging.info("Testing pause resume")
|
|
lead_client = vault_utils.extract_lead_unit_client(self.clients)
|
|
self.assertTrue(lead_client.hvac_client.seal_status['sealed'])
|
|
|
|
def test_vault_reload(self):
|
|
"""Run reload tests.
|
|
|
|
Reload service and check services were restarted
|
|
by doing simple change in the running config by API.
|
|
Then confirm that service is not sealed
|
|
"""
|
|
vault_actions = zaza.model.get_actions(
|
|
'vault')
|
|
if 'reload' not in vault_actions:
|
|
raise unittest.SkipTest("The version of charm-vault tested does "
|
|
"not have reload action")
|
|
|
|
container_results = zaza.model.run_on_leader(
|
|
"vault", "systemd-detect-virt --container"
|
|
)
|
|
container_rc = json.loads(container_results["Code"])
|
|
if container_rc == 0:
|
|
raise unittest.SkipTest(
|
|
"Vault unit is running in a container. Cannot use mlock."
|
|
)
|
|
|
|
lead_client = vault_utils.get_cluster_leader(self.clients)
|
|
running_config = vault_utils.get_running_config(lead_client)
|
|
value_to_set = not running_config['data']['disable_mlock']
|
|
|
|
logging.info("Setting disable-mlock to {}".format(str(value_to_set)))
|
|
zaza.model.set_application_config(
|
|
'vault',
|
|
{'disable-mlock': str(value_to_set)})
|
|
|
|
logging.info("Waiting for model to be idle ...")
|
|
zaza.model.block_until_all_units_idle(model_name=self.model_name)
|
|
|
|
logging.info("Testing action reload on {}".format(lead_client))
|
|
zaza.model.run_action(
|
|
juju_utils.get_unit_name_from_ip_address(
|
|
lead_client.addr, 'vault'),
|
|
'reload',
|
|
model_name=self.model_name)
|
|
|
|
logging.info("Getting new value ...")
|
|
new_value = vault_utils.get_running_config(lead_client)[
|
|
'data']['disable_mlock']
|
|
|
|
logging.info(
|
|
"Asserting new value {} is equal to set value {}"
|
|
.format(new_value, value_to_set))
|
|
self.assertEqual(
|
|
value_to_set,
|
|
new_value)
|
|
|
|
logging.info("Asserting not sealed")
|
|
self.assertFalse(lead_client.hvac_client.seal_status['sealed'])
|
|
|
|
def test_vault_restart(self):
|
|
"""Run pause and resume tests.
|
|
|
|
Restart service and check services are started.
|
|
"""
|
|
vault_actions = zaza.model.get_actions(
|
|
'vault')
|
|
if 'restart' not in vault_actions:
|
|
raise unittest.SkipTest("The version of charm-vault tested does "
|
|
"not have restart action")
|
|
logging.info("Testing restart")
|
|
zaza.model.run_action_on_leader(
|
|
'vault',
|
|
'restart',
|
|
action_params={})
|
|
|
|
lead_client = vault_utils.extract_lead_unit_client(self.clients)
|
|
self.assertTrue(lead_client.hvac_client.seal_status['sealed'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|