From e3bbd947c4c3477c68870d11a955870ac931332d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 24 Sep 2023 10:35:33 +0000 Subject: [PATCH 1/7] Add Keystone LDAP on K8s tests Add Keystone LDAP on K8s tests, this includes a minor refator of the existing machine keystone LDAP tests. --- zaza/openstack/charm_tests/keystone/tests.py | 31 +++-- .../charm_tests/keystone/tests_ldap_k8s.py | 108 ++++++++++++++++++ 2 files changed, 128 insertions(+), 11 deletions(-) create mode 100644 zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 2047d87..a2ee3c2 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -488,6 +488,10 @@ class LdapTests(BaseKeystoneTest): """Run class setup for running Keystone ldap-tests.""" super(LdapTests, cls).setUpClass() + def get_ldap_ips(self): + """Return the ip addresses for the ldap servers.""" + return zaza.model.get_app_ips("ldap-server") + def _get_ldap_config(self): """Generate ldap config for current model. @@ -495,7 +499,7 @@ class LdapTests(BaseKeystoneTest): for the keystone-ldap application. :rtype: Tuple[bool, Dict[str,str]] """ - ldap_ips = zaza.model.get_app_ips("ldap-server") + ldap_ips = self.get_ldap_ips() self.assertTrue(ldap_ips, "Should be at least one ldap server") return { 'ldap-server': "ldap://{}".format(ldap_ips[0]), @@ -703,7 +707,7 @@ class LdapExplicitCharmConfigTests(LdapTests): for the keystone-ldap application. :rtype: Tuple[bool, Dict[str,str]] """ - ldap_ips = zaza.model.get_app_ips("ldap-server") + ldap_ips = self.get_ldap_ips() self.assertTrue(ldap_ips, "Should be at least one ldap server") return { 'ldap-server': "ldap://{}".format(ldap_ips[0]), @@ -730,6 +734,15 @@ class LdapExplicitCharmConfigTests(LdapTests): ' group_tree_dn: "group_tree_dn_foobar"}', } + def get_domain_config(self): + """Return rendered domain config.""" + units = zaza.model.get_units("keystone-ldap", + model_name=self.model_name) + result = zaza.model.run_on_unit( + units[0].name, + "cat /etc/keystone/domains/keystone.userdomain.conf") + return result['stdout'] + def test_200_config_flags_precedence(self): """Validates precedence when the same config options are used.""" application_name = 'keystone-ldap' @@ -755,28 +768,24 @@ class LdapExplicitCharmConfigTests(LdapTests): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {}) ) - units = zaza.model.get_units("keystone-ldap", - model_name=self.model_name) - result = zaza.model.run_on_unit( - units[0].name, - "cat /etc/keystone/domains/keystone.userdomain.conf") + contents = self.get_domain_config() # not present in charm config, but present in config flags - self.assertIn("use_pool = True", result['stdout'], + self.assertIn("use_pool = True", contents, "use_pool value is expected to be present and " "set to True in the config file") # ldap-config-flags overriding empty charm config value self.assertIn("group_objectclass = posixGroup", - result['stdout'], + contents, "group_objectclass is expected to be present and" " set to posixGroup in the config file") # overridden by charm config, not written to file self.assertNotIn( "group_tree_dn_foobar", - result['stdout'], + contents, "user_tree_dn ldap-config-flags value needs to be " "overridden by ldap-user-tree-dn in config file") # complementing the above, value used is from charm setting - self.assertIn("group_tree_dn = ou=groups", result['stdout'], + self.assertIn("group_tree_dn = ou=groups", contents, "user_tree_dn value is expected to be present " "and set to dc=test,dc=com in the config file") diff --git a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py new file mode 100644 index 0000000..771f5cd --- /dev/null +++ b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py @@ -0,0 +1,108 @@ +# Copyright 2023 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Keystone LDAP tests on k8s.""" + +import json + +import contextlib +import zaza.openstack.charm_tests.keystone.tests as ks_tests +import zaza.openstack.charm_tests.tempest.tests as tempest_tests +import zaza.charm_lifecycle.utils as lifecycle_utils +import zaza.model +import subprocess + + +class LdapExplicitCharmConfigTestsK8S(ks_tests.LdapExplicitCharmConfigTests): + """Keystone LDAP tests for K8s deployment.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Keystone ldap-tests.""" + cls.model_name = zaza.model.get_juju_model() + cls.test_config = lifecycle_utils.get_charm_config(fatal=False) + cls.default_api_version = 3 + cls.api_v3 = 3 + cls.keystone_ips = cls.get_internal_ips("keystone", cls.model_name) + + @contextlib.contextmanager + def v3_keystone_preferred(self): + """Set the preferred keystone api to v3 within called context.""" + with contextlib.nullcontext(): + yield + + @staticmethod + def get_internal_ips(application, model_name): + """Return the internal ip addresses an application.""" + status = zaza.model.get_status(model_name=model_name) + units = status['applications'][application]["units"] + return [v.address for v in units.values()] + + def get_ldap_ips(self): + """Return the ip addresses for the ldap servers.""" + return self.get_internal_ips("ldap-server", self.model_name) + + def get_domain_config(self): + """Collect the rendered domain config file.""" + # libjuju does not support ssh to a payload container + cmd = [ + "juju", + "ssh", + "-m", + self.model_name, + "--container", + "keystone", + zaza.model.get_lead_unit("keystone").entity_id, + 'cat /etc/keystone/domains/keystone.userdomain.conf'] + out = subprocess.check_output(cmd) + return out.decode() + + def _get_ldap_config(self): + """Generate ldap config for current model. + + :return: tuple of whether ldap-server is running and if so, config + for the keystone-ldap application. + :rtype: Tuple[bool, Dict[str,str]] + """ + ldap_ips = self.get_ldap_ips() + self.assertTrue(ldap_ips, "Should be at least one ldap server") + config_flags = json.dumps({ + 'url': "ldap://{}".format(ldap_ips[0]), + 'user': 'cn=admin,dc=test,dc=com', + "use_pool": True, + 'password': 'crapper', + 'suffix': 'dc=test,dc=com', + 'query_scope': 'one', + 'user_objectclass': 'inetOrgPerson', + 'user_id_attribute': 'cn', + 'user_name_attribute': 'sn', + 'user_enabled_attribute': 'enabled', + 'user_enabled_invert': False, + 'user_enabled_mask': 0, + 'user_enabled_default': 'True', + 'group_tree_dn': 'ou=groups,dc=test,dc=com', + 'group_id_attribute': 'cn', + 'group_name_attribute': 'cn', + 'group_member_attribute': 'memberUid', + 'group_members_are_ids': True, + "group_objectclass": "posixGroup", + }) + return { + "ldap-config-flags": config_flags, + "domain-name": "userdomain"} + + +class KeystoneTempestTestK8S(tempest_tests.TempestTestScaleK8SBase): + """Test keystone k8s scale out and scale back.""" + + application_name = "keystone" From 48234651757e5803edabcc843c6a42741959311d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Oct 2023 11:59:35 +0000 Subject: [PATCH 2/7] Retry on 404s for keystone ldap --- .../charm_tests/keystone/tests_ldap_k8s.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py index 771f5cd..8cf1d81 100644 --- a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py +++ b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py @@ -14,8 +14,9 @@ """Keystone LDAP tests on k8s.""" import json - +import tenacity import contextlib +import keystoneauth1.exceptions.http.NotFound as http_NotFound import zaza.openstack.charm_tests.keystone.tests as ks_tests import zaza.openstack.charm_tests.tempest.tests as tempest_tests import zaza.charm_lifecycle.utils as lifecycle_utils @@ -101,6 +102,18 @@ class LdapExplicitCharmConfigTestsK8S(ks_tests.LdapExplicitCharmConfigTests): "ldap-config-flags": config_flags, "domain-name": "userdomain"} + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), + reraise=True, stop=tenacity.stop_after_attempt(5), + retry=tenacity.retry_if_exception_type(http_NotFound)) + def _find_keystone_v3_group(self, group, domain): + super()._find_keystone_v3_group(group, domain) + + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), + reraise=True, stop=tenacity.stop_after_attempt(5), + retry=tenacity.retry_if_exception_type(http_NotFound)) + def _find_keystone_v3_user(username, domain, group=None): + super()._find_keystone_v3_user(username, domain, group=group) + class KeystoneTempestTestK8S(tempest_tests.TempestTestScaleK8SBase): """Test keystone k8s scale out and scale back.""" From 590a17dfc0c5e4971d1e8a328ad33de1950413a2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Oct 2023 12:54:28 +0000 Subject: [PATCH 3/7] Fix http_NotFound import --- zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py index 8cf1d81..7470207 100644 --- a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py +++ b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py @@ -16,7 +16,7 @@ import json import tenacity import contextlib -import keystoneauth1.exceptions.http.NotFound as http_NotFound +from keystoneauth1.exceptions.http import NotFound as http_NotFound import zaza.openstack.charm_tests.keystone.tests as ks_tests import zaza.openstack.charm_tests.tempest.tests as tempest_tests import zaza.charm_lifecycle.utils as lifecycle_utils From dc3e9d399a97316fd91a47865a542ae923e19f61 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 24 Oct 2023 13:55:50 +0000 Subject: [PATCH 4/7] Fix call to super()._find_keystone_v3_group --- .../charm_tests/keystone/tests_ldap_k8s.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py index 7470207..dfdf9aa 100644 --- a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py +++ b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py @@ -17,6 +17,7 @@ import json import tenacity import contextlib from keystoneauth1.exceptions.http import NotFound as http_NotFound +import logging import zaza.openstack.charm_tests.keystone.tests as ks_tests import zaza.openstack.charm_tests.tempest.tests as tempest_tests import zaza.charm_lifecycle.utils as lifecycle_utils @@ -106,13 +107,21 @@ class LdapExplicitCharmConfigTestsK8S(ks_tests.LdapExplicitCharmConfigTests): reraise=True, stop=tenacity.stop_after_attempt(5), retry=tenacity.retry_if_exception_type(http_NotFound)) def _find_keystone_v3_group(self, group, domain): - super()._find_keystone_v3_group(group, domain) + logging.info('Looking for group: {}'.format(group)) + try: + return super()._find_keystone_v3_group(group, domain) + except AttributeError: + raise http_NotFound @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), reraise=True, stop=tenacity.stop_after_attempt(5), retry=tenacity.retry_if_exception_type(http_NotFound)) - def _find_keystone_v3_user(username, domain, group=None): - super()._find_keystone_v3_user(username, domain, group=group) + def _find_keystone_v3_user(self, username, domain, group=None): + logging.info('Looking for user: {}'.format(username)) + try: + return super()._find_keystone_v3_user(username, domain, group=group) + except AttributeError: + raise http_NotFound class KeystoneTempestTestK8S(tempest_tests.TempestTestScaleK8SBase): From baa02a30fcb6c1c681dc2da572e496ee93546790 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Oct 2023 11:47:09 +0000 Subject: [PATCH 5/7] Catch connection errors --- .../charm_tests/keystone/tests_ldap_k8s.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py index dfdf9aa..3fadc2a 100644 --- a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py +++ b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py @@ -18,12 +18,15 @@ import tenacity import contextlib from keystoneauth1.exceptions.http import NotFound as http_NotFound import logging +from requests.exceptions import ConnectionError import zaza.openstack.charm_tests.keystone.tests as ks_tests import zaza.openstack.charm_tests.tempest.tests as tempest_tests import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.model import subprocess +class KeystoneLookupError(Exception): + pass class LdapExplicitCharmConfigTestsK8S(ks_tests.LdapExplicitCharmConfigTests): """Keystone LDAP tests for K8s deployment.""" @@ -105,23 +108,23 @@ class LdapExplicitCharmConfigTestsK8S(ks_tests.LdapExplicitCharmConfigTests): @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), reraise=True, stop=tenacity.stop_after_attempt(5), - retry=tenacity.retry_if_exception_type(http_NotFound)) + retry=tenacity.retry_if_exception_type(KeystoneLookupError)) def _find_keystone_v3_group(self, group, domain): logging.info('Looking for group: {}'.format(group)) try: return super()._find_keystone_v3_group(group, domain) - except AttributeError: - raise http_NotFound + except (AttributeError, http_NotFound, ConnectionError) as error: + raise KeystoneLookupError @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), reraise=True, stop=tenacity.stop_after_attempt(5), - retry=tenacity.retry_if_exception_type(http_NotFound)) + retry=tenacity.retry_if_exception_type(KeystoneLookupError)) def _find_keystone_v3_user(self, username, domain, group=None): logging.info('Looking for user: {}'.format(username)) try: return super()._find_keystone_v3_user(username, domain, group=group) - except AttributeError: - raise http_NotFound + except (AttributeError, http_NotFound, ConnectionError) as error: + raise KeystoneLookupError class KeystoneTempestTestK8S(tempest_tests.TempestTestScaleK8SBase): From 8f9eff813bcb00b8cabb5e20bd76ea7686ba2f53 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Oct 2023 12:09:50 +0000 Subject: [PATCH 6/7] Fix lint --- .../charm_tests/keystone/tests_ldap_k8s.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py index 3fadc2a..b3abff4 100644 --- a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py +++ b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py @@ -25,9 +25,13 @@ import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.model import subprocess + class KeystoneLookupError(Exception): + """An error looking up data in keystone.""" + pass + class LdapExplicitCharmConfigTestsK8S(ks_tests.LdapExplicitCharmConfigTests): """Keystone LDAP tests for K8s deployment.""" @@ -108,22 +112,27 @@ class LdapExplicitCharmConfigTestsK8S(ks_tests.LdapExplicitCharmConfigTests): @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), reraise=True, stop=tenacity.stop_after_attempt(5), - retry=tenacity.retry_if_exception_type(KeystoneLookupError)) + retry=tenacity.retry_if_exception_type( + KeystoneLookupError)) def _find_keystone_v3_group(self, group, domain): logging.info('Looking for group: {}'.format(group)) try: return super()._find_keystone_v3_group(group, domain) - except (AttributeError, http_NotFound, ConnectionError) as error: + except (AttributeError, http_NotFound, ConnectionError): raise KeystoneLookupError @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), reraise=True, stop=tenacity.stop_after_attempt(5), - retry=tenacity.retry_if_exception_type(KeystoneLookupError)) + retry=tenacity.retry_if_exception_type( + KeystoneLookupError)) def _find_keystone_v3_user(self, username, domain, group=None): logging.info('Looking for user: {}'.format(username)) try: - return super()._find_keystone_v3_user(username, domain, group=group) - except (AttributeError, http_NotFound, ConnectionError) as error: + return super()._find_keystone_v3_user( + username, + domain, + group=group) + except (AttributeError, http_NotFound, ConnectionError): raise KeystoneLookupError From d98638fa4bca564f8ada84f45356e8c21c14dd6b Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 25 Oct 2023 12:24:55 +0000 Subject: [PATCH 7/7] Catch ConnectFailure --- zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py index b3abff4..2183d98 100644 --- a/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py +++ b/zaza/openstack/charm_tests/keystone/tests_ldap_k8s.py @@ -16,6 +16,7 @@ import json import tenacity import contextlib +from keystoneauth1.exceptions.connection import ConnectFailure from keystoneauth1.exceptions.http import NotFound as http_NotFound import logging from requests.exceptions import ConnectionError @@ -118,7 +119,8 @@ class LdapExplicitCharmConfigTestsK8S(ks_tests.LdapExplicitCharmConfigTests): logging.info('Looking for group: {}'.format(group)) try: return super()._find_keystone_v3_group(group, domain) - except (AttributeError, http_NotFound, ConnectionError): + except (AttributeError, http_NotFound, ConnectionError, + ConnectFailure): raise KeystoneLookupError @tenacity.retry(wait=tenacity.wait_exponential(multiplier=2, max=60), @@ -132,7 +134,8 @@ class LdapExplicitCharmConfigTestsK8S(ks_tests.LdapExplicitCharmConfigTests): username, domain, group=group) - except (AttributeError, http_NotFound, ConnectionError): + except (AttributeError, http_NotFound, ConnectionError, + ConnectFailure): raise KeystoneLookupError