From 5d457897265a42243034592c6db9cea72e6a0a6a Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Fri, 5 Apr 2024 18:05:40 -0300 Subject: [PATCH 01/15] Implement tests for ceph-mon's rotate-key action This PR implements tests for the ceph-mon charm's action 'rotate-key'. For now, the charm only supports key rotation for managers, but more entities will be added. This PR will be used to test those new entities as well. --- zaza/openstack/charm_tests/ceph/tests.py | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 46911c0..83d7ccd 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1748,3 +1748,39 @@ class CephMonJujuPersistent(test_utils.BaseCharmTest): ) data = json.loads(result['Stdout']) assert data['loglevel'] == 2 + + +class CephMonKeyRotationTests(test_utils.BaseCharmTest): + """Tests for the rotate-key action.""" + + def _get_key_for_mgr(self, unit, mgr_id): + cmd = 'sudo ceph auth ls' + result = zaza_model.run_on_unit(unit, cmd) + # Don't use json formatting, as it's buggy upstream. + data = result['Stdout'].split() + + for ix, line in enumerate(data): + # Structure: + # mgr.XXXX + # key: + # key contents + # That's why we need to move 2 positions ahead. + if line.startswith('mgr.') and line.endswith(mgr_id): + return data[ix + 2] + + def test_mgr_key_rotate(self): + """Test that rotating the manager key actually changes it.""" + unit = 'ceph-mon/0' + mgr_id = unit[-1] + old_key = self._get_key_for_mgr(unit, mgr_id) + self.assertIsNotNone(old_key) + + action_obj = zaza_model.run_action( + unit_name=unit, + action_name='rotate-key', + action_params={'entity': 'mgr'} + ) + zaza_utils.assertActionRanOk(action_obj) + new_key = self._get_key_for_mgr(unit, mgr_id) + self.assertIsNotNone(new_key) + self.assertNotEqual(old_key, new_key) From 3989d543f6fccbc7541cc960db5a59797390340a Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 8 Apr 2024 11:20:48 -0300 Subject: [PATCH 02/15] Get the keys for all managers to compare --- zaza/openstack/charm_tests/ceph/tests.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 83d7ccd..5c2b721 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1753,11 +1753,12 @@ class CephMonJujuPersistent(test_utils.BaseCharmTest): class CephMonKeyRotationTests(test_utils.BaseCharmTest): """Tests for the rotate-key action.""" - def _get_key_for_mgr(self, unit, mgr_id): + def _get_mgrs_keys(self, unit): cmd = 'sudo ceph auth ls' result = zaza_model.run_on_unit(unit, cmd) # Don't use json formatting, as it's buggy upstream. data = result['Stdout'].split() + ret = set() for ix, line in enumerate(data): # Structure: @@ -1765,22 +1766,20 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): # key: # key contents # That's why we need to move 2 positions ahead. - if line.startswith('mgr.') and line.endswith(mgr_id): - return data[ix + 2] + if line.startswith('mgr.'): + ret.add(data[ix + 2]) + return ret def test_mgr_key_rotate(self): """Test that rotating the manager key actually changes it.""" unit = 'ceph-mon/0' - mgr_id = unit[-1] - old_key = self._get_key_for_mgr(unit, mgr_id) - self.assertIsNotNone(old_key) + old_keys = self._get_mgrs_keys(unit) action_obj = zaza_model.run_action( unit_name=unit, action_name='rotate-key', action_params={'entity': 'mgr'} ) - zaza_utils.assertActionRanOk(action_obj) - new_key = self._get_key_for_mgr(unit, mgr_id) - self.assertIsNotNone(new_key) - self.assertNotEqual(old_key, new_key) + zaza_utils.assertActionRanOK(action_obj) + new_keys = self._get_mgrs_keys(unit) + self.assertNotEqual(old_keys, new_keys) From 99456d9229c26a526d83b20c2412dd9bc1ffc36a Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Tue, 9 Apr 2024 11:59:43 -0300 Subject: [PATCH 03/15] Adjust test so it can be used for more than managers --- zaza/openstack/charm_tests/ceph/tests.py | 42 +++++++++++++++++------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 5c2b721..049be07 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1753,7 +1753,7 @@ class CephMonJujuPersistent(test_utils.BaseCharmTest): class CephMonKeyRotationTests(test_utils.BaseCharmTest): """Tests for the rotate-key action.""" - def _get_mgrs_keys(self, unit): + def _get_all_keys(self, unit, entity_filter): cmd = 'sudo ceph auth ls' result = zaza_model.run_on_unit(unit, cmd) # Don't use json formatting, as it's buggy upstream. @@ -1762,24 +1762,44 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): for ix, line in enumerate(data): # Structure: - # mgr.XXXX + # $ENTITY # key: # key contents - # That's why we need to move 2 positions ahead. - if line.startswith('mgr.'): - ret.add(data[ix + 2]) + # That's why we need to move one position ahead. + if 'key:' in line and entity_filter(line[ix - 1]): + ret.add(data[ix + 1]) return ret - def test_mgr_key_rotate(self): - """Test that rotating the manager key actually changes it.""" - unit = 'ceph-mon/0' - old_keys = self._get_mgrs_keys(unit) + def _check_key_rotation(self, entity, unit): + def entity_filter(name): + return name.startswith(entity) + old_keys = self._get_all_keys(unit, entity_filter) action_obj = zaza_model.run_action( unit_name=unit, action_name='rotate-key', - action_params={'entity': 'mgr'} + action_params={'entity': entity} ) zaza_utils.assertActionRanOK(action_obj) - new_keys = self._get_mgrs_keys(unit) + zaza_model.wait_for_application_states() + new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) + self.assertEqual(new_keys - old_keys, 1) + + def _get_rgw_client(self, unit): + cmd = 'sudo ceph auth ls | grep client.rgw' + result = zaza_model.run_on_unit(unit, cmd) + return result['Stdout'].strip() + + def test_key_rotate(self): + """Test that rotating the keys actually changes them.""" + unit = 'ceph-mon/0' + self._check_key_rotation('mgr', unit) + + try: + zaza_model.get_application('ceph-radosgw') + rgw_client = self._get_rgw_client('ceph-radosgw/0') + if rgw_client: + self._check_key_rotation(rgw_client, unit) + except KeyError: + pass From b99a6c428a078e133678abd8b464925e77abb9d1 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Tue, 9 Apr 2024 13:47:25 -0300 Subject: [PATCH 04/15] Fix access to keys --- zaza/openstack/charm_tests/ceph/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 049be07..4e51d3c 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1766,7 +1766,7 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): # key: # key contents # That's why we need to move one position ahead. - if 'key:' in line and entity_filter(line[ix - 1]): + if 'key:' in line and entity_filter(data[ix - 1]): ret.add(data[ix + 1]) return ret @@ -1784,7 +1784,7 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): zaza_model.wait_for_application_states() new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) - self.assertEqual(new_keys - old_keys, 1) + self.assertEqual(len(new_keys - old_keys), 1) def _get_rgw_client(self, unit): cmd = 'sudo ceph auth ls | grep client.rgw' From 761913176c50c5e2b953e944341634c3f99fa0c8 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Tue, 9 Apr 2024 19:28:40 -0300 Subject: [PATCH 05/15] Make sure the result is the type of entity we're after --- zaza/openstack/charm_tests/ceph/tests.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 4e51d3c..c121884 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1767,7 +1767,7 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): # key contents # That's why we need to move one position ahead. if 'key:' in line and entity_filter(data[ix - 1]): - ret.add(data[ix + 1]) + ret.add((data[ix - 1], data[ix + 1])) return ret def _check_key_rotation(self, entity, unit): @@ -1784,7 +1784,13 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): zaza_model.wait_for_application_states() new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) - self.assertEqual(len(new_keys - old_keys), 1) + diff = new_keys - old_keys + self.assertEqual(len(diff), 1) + first = next(iter(diff)) + # Check that the entity matches. The 'entity_filter' + # callable will return a true-like value if it + # matches the type of entity we're after (i.e: 'mgr') + self.assertTrue(entity_filter(first[0])) def _get_rgw_client(self, unit): cmd = 'sudo ceph auth ls | grep client.rgw' @@ -1801,5 +1807,7 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): rgw_client = self._get_rgw_client('ceph-radosgw/0') if rgw_client: self._check_key_rotation(rgw_client, unit) + else: + logging.info('ceph-radosgw units present, but no RGW service') except KeyError: pass From 8fe9a110edbce62acdbddf4013b1ae8dec78659e Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Tue, 9 Apr 2024 19:47:24 -0300 Subject: [PATCH 06/15] Also test key rotation in OSD's --- zaza/openstack/charm_tests/ceph/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index c121884..06843b9 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1811,3 +1811,9 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): logging.info('ceph-radosgw units present, but no RGW service') except KeyError: pass + + try: + zaza_model.get_application('ceph-osd') + self._check_key_rotation('osd.0', unit) + except KeyError: + pass From f411c0b0b1adfd1a883b54e7da46613d85146ff3 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Wed, 10 Apr 2024 13:08:41 -0300 Subject: [PATCH 07/15] Use already existing functionality to fetch RGW client --- zaza/openstack/charm_tests/ceph/tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 06843b9..8bf4bf4 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1793,9 +1793,10 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): self.assertTrue(entity_filter(first[0])) def _get_rgw_client(self, unit): - cmd = 'sudo ceph auth ls | grep client.rgw' - result = zaza_model.run_on_unit(unit, cmd) - return result['Stdout'].strip() + ret = self._get_all_keys(unit, 'client.rgw') + if not ret: + return None + return next(iter(ret))[0] def test_key_rotate(self): """Test that rotating the keys actually changes them.""" From d10c243f8d1db7091730f813ea72a5b43c560e71 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Wed, 10 Apr 2024 13:41:40 -0300 Subject: [PATCH 08/15] Comment out OSD tests until PR is ready --- zaza/openstack/charm_tests/ceph/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 8bf4bf4..c394345 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1813,8 +1813,8 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): except KeyError: pass - try: - zaza_model.get_application('ceph-osd') - self._check_key_rotation('osd.0', unit) - except KeyError: - pass + #try: + # zaza_model.get_application('ceph-osd') + # self._check_key_rotation('osd.0', unit) + #except KeyError: + # pass From eee7611ccbbb8084e70f9d33c899fc61c11a4208 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Wed, 10 Apr 2024 13:45:12 -0300 Subject: [PATCH 09/15] PEP8 fix --- zaza/openstack/charm_tests/ceph/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index c394345..88f6ad6 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1813,8 +1813,8 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): except KeyError: pass - #try: - # zaza_model.get_application('ceph-osd') - # self._check_key_rotation('osd.0', unit) - #except KeyError: - # pass + # try: + # zaza_model.get_application('ceph-osd') + # self._check_key_rotation('osd.0', unit) + # except KeyError: + # pass From cbc117208e32a291d7478792a4a17d018607af5e Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Thu, 11 Apr 2024 13:14:45 -0300 Subject: [PATCH 10/15] Use the right unit to get the RGW client --- zaza/openstack/charm_tests/ceph/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index e000f42..d693276 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1811,7 +1811,7 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): try: zaza_model.get_application('ceph-radosgw') - rgw_client = self._get_rgw_client('ceph-radosgw/0') + rgw_client = self._get_rgw_client(unit) if rgw_client: self._check_key_rotation(rgw_client, unit) else: From 2f557c531d4565b16864aee7ddb0f90744f707db Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Thu, 11 Apr 2024 16:06:19 -0300 Subject: [PATCH 11/15] Use the correct callable --- zaza/openstack/charm_tests/ceph/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index d693276..f882c67 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1799,7 +1799,7 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): self.assertTrue(entity_filter(first[0])) def _get_rgw_client(self, unit): - ret = self._get_all_keys(unit, 'client.rgw') + ret = self._get_all_keys(unit, lambda x: x.startswith('client.rgw')) if not ret: return None return next(iter(ret))[0] From 06e876a7ad52ae457c407f81920ef0bfa685b7a1 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Fri, 12 Apr 2024 13:34:11 -0300 Subject: [PATCH 12/15] Uncomment ceph-osd tests --- zaza/openstack/charm_tests/ceph/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index f882c67..4acf217 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1819,8 +1819,8 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): except KeyError: pass - # try: - # zaza_model.get_application('ceph-osd') - # self._check_key_rotation('osd.0', unit) - # except KeyError: - # pass + try: + zaza_model.get_application('ceph-osd') + self._check_key_rotation('osd.0', unit) + except KeyError: + pass From e574fc00191cb6838815e551101360c58d9eef78 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 15 Apr 2024 11:18:08 -0300 Subject: [PATCH 13/15] Temporarily revert OSD testing to get RGW changes merged --- zaza/openstack/charm_tests/ceph/tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 4acf217..f882c67 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -1819,8 +1819,8 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): except KeyError: pass - try: - zaza_model.get_application('ceph-osd') - self._check_key_rotation('osd.0', unit) - except KeyError: - pass + # try: + # zaza_model.get_application('ceph-osd') + # self._check_key_rotation('osd.0', unit) + # except KeyError: + # pass From aa7d619a249b4ec66ffc335f9342a146a13e6197 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Mon, 15 Apr 2024 13:55:44 -0300 Subject: [PATCH 14/15] Add some time to get the unit to settle --- zaza/openstack/charm_tests/ceph/tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index f882c67..ca76984 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -23,6 +23,7 @@ from os import ( ) import requests import tempfile +import time import boto3 import botocore.exceptions import urllib3 @@ -1788,6 +1789,7 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): ) zaza_utils.assertActionRanOK(action_obj) zaza_model.wait_for_application_states() + time.sleep(5) # Give it time to finish. new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) diff = new_keys - old_keys From e083e56ee7f8d11cec4ba8f0173cd0c01c7b80e1 Mon Sep 17 00:00:00 2001 From: Luciano Lo Giudice Date: Fri, 19 Apr 2024 15:36:00 -0300 Subject: [PATCH 15/15] Remove OSD tests for initial release --- zaza/openstack/charm_tests/ceph/tests.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index ca76984..03c2e90 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -23,7 +23,6 @@ from os import ( ) import requests import tempfile -import time import boto3 import botocore.exceptions import urllib3 @@ -1789,7 +1788,6 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): ) zaza_utils.assertActionRanOK(action_obj) zaza_model.wait_for_application_states() - time.sleep(5) # Give it time to finish. new_keys = self._get_all_keys(unit, entity_filter) self.assertNotEqual(old_keys, new_keys) diff = new_keys - old_keys @@ -1820,9 +1818,3 @@ class CephMonKeyRotationTests(test_utils.BaseCharmTest): logging.info('ceph-radosgw units present, but no RGW service') except KeyError: pass - - # try: - # zaza_model.get_application('ceph-osd') - # self._check_key_rotation('osd.0', unit) - # except KeyError: - # pass