Merge pull request #187 from openstack-charmers/keystone-security-audit

Keystone security audit

zaza/charm_tests/keystone/tests.py

@@ -25,6 +25,7 @@ import zaza.utilities.exceptions as zaza_exceptions
 import zaza.utilities.juju as juju_utils
 import zaza.utilities.openstack as openstack_utils
 
+import zaza.charm_tests.test_utils as test_utils
 from zaza.charm_tests.keystone import (
     BaseKeystoneTest,
     DEMO_DOMAIN,
@@ -324,3 +325,40 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest):
                     openrc.update(
                         {'OS_AUTH_URL': 'http://{}:5000/v3'.format(ip)})
                     _validate_token_data(openrc)
+
+
+class SecurityTests(BaseKeystoneTest):
+    """Keystone security tests tests."""
+
+    @classmethod
+    def setUpClass(cls):
+        """Run class setup for running Keystone aa-tests."""
+        super(SecurityTests, cls).setUpClass()
+
+    def test_security_checklist(self):
+        """Verify expected state with security-checklist."""
+        # Changes fixing the below expected failures will be made following
+        # this initial work to get validation in. There will be bugs targeted
+        # to each one and resolved independently where possible.
+        expected_failures = [
+            'check-max-request-body-size',
+            'disable-admin-token',
+            'uses-sha256-for-hashing-tokens',
+            'validate-file-ownership',
+            'validate-file-permissions',
+        ]
+        expected_passes = [
+            'uses-fernet-token-after-default',
+            'insecure-debug-is-false',
+        ]
+
+        logging.info('Running `security-checklist` action'
+                     ' on Keystone leader unit')
+        test_utils.audit_assertions(
+            zaza.model.run_action_on_leader(
+                'keystone',
+                'security-checklist',
+                action_params={}),
+            expected_passes,
+            expected_failures,
+            expected_to_pass=False)

zaza/charm_tests/test_utils.py

@@ -38,6 +38,37 @@ def skipIfNotHA(service_name):
     return _skipIfNotHA_inner_1
 
 
+def audit_assertions(action,
+                     expected_passes,
+                     expected_failures=None,
+                     expected_to_pass=True):
+    """Check expected assertion failures in security-checklist actions.
+
+    :param action: Action object from running the security-checklist action
+    :type action: juju.action.Action
+    :param expected_passes: List of test names that are expected to pass
+    :type expected_passes: List[str]
+    :param expected_failures: List of test names that are expected to fail
+    :type expexted_failures: List[str]
+    :raises: AssertionError if the assertion fails.
+    """
+    if expected_failures is None:
+        expected_failures = []
+    if expected_to_pass:
+        assert action.data["status"] == "completed", \
+            "Security check is expected to pass by default"
+    else:
+        assert action.data["status"] == "failed", \
+            "Security check is not expected to pass by default"
+
+    results = action.data['results']
+    for key, value in results.items():
+        if key in expected_failures:
+            assert "FAIL" in value, "Unexpected test pass: {}".format(key)
+        if key in expected_passes:
+            assert value == "PASS", "Unexpected failure: {}".format(key)
+
+
 class OpenStackBaseTest(unittest.TestCase):
     """Generic helpers for testing OpenStack API charms."""