From 5579f2f110b8b08097c6a7b189669886a10cb378 Mon Sep 17 00:00:00 2001 From: Alvaro Uria Date: Mon, 20 Jul 2020 19:24:33 +0200 Subject: [PATCH 001/180] HaclusterTest: support scaleback test on bionic --- zaza/openstack/charm_tests/hacluster/tests.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 199d915..3ae13d1 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -21,6 +21,7 @@ import os import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.configure.hacluster +import zaza.utilities.juju as juju_utils class HaclusterTest(test_utils.OpenStackBaseTest): @@ -69,3 +70,71 @@ class HaclusterTest(test_utils.OpenStackBaseTest): self._toggle_maintenance_and_wait('true') self._toggle_maintenance_and_wait('false') + + def test_930_scaleback_bionic(self): + """Remove a unit, recalculate quorum and add a new one.""" + principle_app = 'keystone' + principle_units = zaza.model.get_status().applications[ + principle_app]['units'] + self.assertEqual(len(principle_units), 3) + doomed_principle = sorted(principle_units.keys())[0] + series = juju_utils.get_machine_series( + principle_units[doomed_principle].machine) + if series != 'bionic': + logging.debug("noop - only run test in bionic") + logging.info('SKIP') + return + + doomed_unit = juju_utils.get_subordinate_units( + [doomed_principle], charm_name='hac')[0] + + logging.info('Pausing unit {}'.format(doomed_unit)) + zaza.model.run_action( + doomed_unit, + 'pause', + raise_on_failure=True) + logging.info('OK') + + logging.info('Resuming unit {}'.format(doomed_unit)) + zaza.model.run_action( + doomed_unit, + 'resume', + raise_on_failure=True) + logging.info('OK') + + logging.info('Removing {}'.format(doomed_principle)) + zaza.model.destroy_unit( + principle_app, + doomed_principle, + wait_disappear=True) + logging.info('OK') + + logging.info('Updating corosync ring') + zaza.model.run_action_on_leader( + self.application_name, + 'update-ring', + action_params={'i-really-mean-it': True}, + raise_on_failure=True) + + _states = { + self.application_name: { + "workload-status": "blocked", + "workload-status-message": + "Insufficient peer units for ha cluster (require 3)" + }, + 'keystone': { + "workload-status": "blocked", + "workload-status-message": "Database not initialised", + }, + } + zaza.model.wait_for_application_states(states=_states) + zaza.model.block_until_all_units_idle() + logging.info('OK') + + logging.info('Adding a hacluster unit') + zaza.model.add_unit(principle_app, wait_appear=True) + _states = {self.application_name: { + "workload-status": "active", + "workload-status-message": "Unit is ready and clustered"}} + zaza.model.wait_for_application_states(states=_states) + logging.debug('OK') From 573acf9d1c0e2490589a4246b90083fc4113fdc9 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 27 Aug 2020 11:54:42 +0200 Subject: [PATCH 002/180] Move new scaleback test to new test class --- zaza/openstack/charm_tests/hacluster/tests.py | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 3ae13d1..7715a4a 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -24,15 +24,19 @@ import zaza.openstack.configure.hacluster import zaza.utilities.juju as juju_utils -class HaclusterTest(test_utils.OpenStackBaseTest): - """hacluster tests.""" +class HaclusterBaseTest(test_utils.OpenStackBaseTest): + """Base class for hacluster tests.""" @classmethod def setUpClass(cls): """Run class setup for running hacluster tests.""" - super(HaclusterTest, cls).setUpClass() + super(HaclusterBaseTest, cls).setUpClass() cls.vip = os.environ.get("TEST_VIP00") + +class HaclusterTest(HaclusterBaseTest): + """hacluster tests.""" + def test_900_action_cleanup(self): """The services can be cleaned up.""" zaza.model.run_action_on_leader( @@ -71,22 +75,20 @@ class HaclusterTest(test_utils.OpenStackBaseTest): self._toggle_maintenance_and_wait('true') self._toggle_maintenance_and_wait('false') - def test_930_scaleback_bionic(self): + +class HaclusterScalebackTest(HaclusterBaseTest): + """hacluster scaleback tests.""" + + _PRINCIPLE_APP = 'keystone' + + def test_930_scaleback(self): """Remove a unit, recalculate quorum and add a new one.""" - principle_app = 'keystone' principle_units = zaza.model.get_status().applications[ - principle_app]['units'] + self._PRINCIPLE_APP]['units'] self.assertEqual(len(principle_units), 3) doomed_principle = sorted(principle_units.keys())[0] - series = juju_utils.get_machine_series( - principle_units[doomed_principle].machine) - if series != 'bionic': - logging.debug("noop - only run test in bionic") - logging.info('SKIP') - return - doomed_unit = juju_utils.get_subordinate_units( - [doomed_principle], charm_name='hac')[0] + [doomed_principle], charm_name=self.application_name)[0] logging.info('Pausing unit {}'.format(doomed_unit)) zaza.model.run_action( @@ -104,7 +106,7 @@ class HaclusterTest(test_utils.OpenStackBaseTest): logging.info('Removing {}'.format(doomed_principle)) zaza.model.destroy_unit( - principle_app, + self._PRINCIPLE_APP, doomed_principle, wait_disappear=True) logging.info('OK') @@ -116,25 +118,25 @@ class HaclusterTest(test_utils.OpenStackBaseTest): action_params={'i-really-mean-it': True}, raise_on_failure=True) - _states = { + expected_states = { self.application_name: { "workload-status": "blocked", "workload-status-message": "Insufficient peer units for ha cluster (require 3)" }, - 'keystone': { + self._PRINCIPLE_APP: { "workload-status": "blocked", "workload-status-message": "Database not initialised", }, } - zaza.model.wait_for_application_states(states=_states) + zaza.model.wait_for_application_states(states=expected_states) zaza.model.block_until_all_units_idle() logging.info('OK') logging.info('Adding a hacluster unit') - zaza.model.add_unit(principle_app, wait_appear=True) - _states = {self.application_name: { + zaza.model.add_unit(self._PRINCIPLE_APP, wait_appear=True) + expected_states = {self.application_name: { "workload-status": "active", "workload-status-message": "Unit is ready and clustered"}} - zaza.model.wait_for_application_states(states=_states) + zaza.model.wait_for_application_states(states=expected_states) logging.debug('OK') From e48da475a2fe4a4b7d6b210def38aacfee05b83e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 27 Aug 2020 12:57:46 +0200 Subject: [PATCH 003/180] No need to resume hacluster unit before scaling down --- zaza/openstack/charm_tests/hacluster/tests.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 7715a4a..28f5248 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -97,13 +97,6 @@ class HaclusterScalebackTest(HaclusterBaseTest): raise_on_failure=True) logging.info('OK') - logging.info('Resuming unit {}'.format(doomed_unit)) - zaza.model.run_action( - doomed_unit, - 'resume', - raise_on_failure=True) - logging.info('OK') - logging.info('Removing {}'.format(doomed_principle)) zaza.model.destroy_unit( self._PRINCIPLE_APP, From b72855ca96d15a11c43e8b56a63a7f968e91d847 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 28 Aug 2020 13:15:31 +0200 Subject: [PATCH 004/180] Make HaclusterScalebackTest runnable against any charm --- zaza/openstack/charm_tests/hacluster/tests.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 28f5248..76eced8 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -79,16 +79,18 @@ class HaclusterTest(HaclusterBaseTest): class HaclusterScalebackTest(HaclusterBaseTest): """hacluster scaleback tests.""" - _PRINCIPLE_APP = 'keystone' + _PRINCIPLE_APP_NAME = 'keystone' + _HACLUSTER_APP_NAME = 'hacluster' + _HACLUSTER_CHARM_NAME = 'hacluster' def test_930_scaleback(self): """Remove a unit, recalculate quorum and add a new one.""" principle_units = zaza.model.get_status().applications[ - self._PRINCIPLE_APP]['units'] + self._PRINCIPLE_APP_NAME]['units'] self.assertEqual(len(principle_units), 3) doomed_principle = sorted(principle_units.keys())[0] doomed_unit = juju_utils.get_subordinate_units( - [doomed_principle], charm_name=self.application_name)[0] + [doomed_principle], charm_name=self._HACLUSTER_CHARM_NAME)[0] logging.info('Pausing unit {}'.format(doomed_unit)) zaza.model.run_action( @@ -99,25 +101,25 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('Removing {}'.format(doomed_principle)) zaza.model.destroy_unit( - self._PRINCIPLE_APP, + self._PRINCIPLE_APP_NAME, doomed_principle, wait_disappear=True) logging.info('OK') logging.info('Updating corosync ring') zaza.model.run_action_on_leader( - self.application_name, + self._HACLUSTER_APP_NAME, 'update-ring', action_params={'i-really-mean-it': True}, raise_on_failure=True) expected_states = { - self.application_name: { + self._HACLUSTER_APP_NAME: { "workload-status": "blocked", "workload-status-message": "Insufficient peer units for ha cluster (require 3)" }, - self._PRINCIPLE_APP: { + self._PRINCIPLE_APP_NAME: { "workload-status": "blocked", "workload-status-message": "Database not initialised", }, @@ -127,8 +129,8 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('OK') logging.info('Adding a hacluster unit') - zaza.model.add_unit(self._PRINCIPLE_APP, wait_appear=True) - expected_states = {self.application_name: { + zaza.model.add_unit(self._PRINCIPLE_APP_NAME, wait_appear=True) + expected_states = {self._HACLUSTER_APP_NAME: { "workload-status": "active", "workload-status-message": "Unit is ready and clustered"}} zaza.model.wait_for_application_states(states=expected_states) From 14f7e3848cb5989912769e3a7c47345cd1f56624 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 8 Sep 2020 12:51:53 +0200 Subject: [PATCH 005/180] Remove hacluster test option hacluster_app_name --- zaza/openstack/charm_tests/hacluster/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 2728a68..e02832b 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -85,7 +85,6 @@ class HaclusterScalebackTest(HaclusterBaseTest): super(HaclusterScalebackTest, cls).setUpClass() test_config = cls.test_config['tests_options']['hacluster'] cls._principle_app_name = test_config['principle-app-name'] - cls._hacluster_app_name = test_config['hacluster-app-name'] cls._hacluster_charm_name = test_config['hacluster-charm-name'] def test_930_scaleback(self): @@ -121,8 +120,9 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('OK') logging.info('Updating corosync ring') + hacluster_app_name = other_hacluster_unit.application zaza.model.run_action_on_leader( - self._hacluster_app_name, + hacluster_app_name, 'update-ring', action_params={'i-really-mean-it': True}, raise_on_failure=True) @@ -132,7 +132,7 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('OK') logging.info('Waiting for model to settle') - expected_states = {self._hacluster_app_name: { + expected_states = {hacluster_app_name: { "workload-status": "active", "workload-status-message": "Unit is ready and clustered"}} zaza.model.wait_for_application_states(states=expected_states) From 11f3060f6b1564d6497cbfc2b2f80251bcc27ca3 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 9 Sep 2020 10:08:37 +0200 Subject: [PATCH 006/180] Fix unit-name -> app-name conversion --- zaza/openstack/charm_tests/hacluster/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index e02832b..29751e1 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -120,7 +120,8 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('OK') logging.info('Updating corosync ring') - hacluster_app_name = other_hacluster_unit.application + hacluster_app_name = zaza.model.get_unit_from_name( + other_hacluster_unit).application zaza.model.run_action_on_leader( hacluster_app_name, 'update-ring', From 09789767f301e1b7133d76b441e4d15ca7c5251e Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 25 Sep 2020 11:19:32 +0200 Subject: [PATCH 007/180] Enhance common launch instance helper Move retry logic to the launch single guest method. Add optional arguments that is passed on to the downstream launch guest helper. --- zaza/openstack/charm_tests/test_utils.py | 35 ++++++++++++++---------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index a3614f0..1f24aa9 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -526,7 +526,8 @@ class OpenStackBaseTest(BaseCharmTest): # Test did not define self.RESOURCE_PREFIX, ignore. pass - def launch_guest(self, guest_name, userdata=None): + def launch_guest(self, guest_name, userdata=None, use_boot_volume=False, + instance_key=None): """Launch two guests to use in tests. Note that it is up to the caller to have set the RESOURCE_PREFIX class @@ -539,9 +540,14 @@ class OpenStackBaseTest(BaseCharmTest): :type guest_name: str :param userdata: Userdata to attach to instance :type userdata: Optional[str] + :param use_boot_volume: Whether to boot guest from a shared volume. + :type use_boot_volume: boolean + :param instance_key: Key to collect associated config data with. + :type instance_key: Optional[str] :returns: Nova instance objects :rtype: Server """ + instance_key = instance_key or glance_setup.LTS_IMAGE_NAME instance_name = '{}-{}'.format(self.RESOURCE_PREFIX, guest_name) instance = self.retrieve_guest(instance_name) @@ -554,10 +560,16 @@ class OpenStackBaseTest(BaseCharmTest): instance.id, msg="server") - return configure_guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - vm_name=instance_name, - userdata=userdata) + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)): + with attempt: + return configure_guest.launch_instance( + instance_key, + vm_name=instance_name, + use_boot_volume=use_boot_volume, + userdata=userdata) def launch_guests(self, userdata=None): """Launch two guests to use in tests. @@ -572,15 +584,10 @@ class OpenStackBaseTest(BaseCharmTest): """ launched_instances = [] for guest_number in range(1, 2+1): - for attempt in tenacity.Retrying( - stop=tenacity.stop_after_attempt(3), - wait=tenacity.wait_exponential( - multiplier=1, min=2, max=10)): - with attempt: - launched_instances.append( - self.launch_guest( - guest_name='ins-{}'.format(guest_number), - userdata=userdata)) + launched_instances.append( + self.launch_guest( + guest_name='ins-{}'.format(guest_number), + userdata=userdata)) return launched_instances def retrieve_guest(self, guest_name): From 786b40fc53bf3a13d2df1faafd2c15c21b17aca5 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 25 Sep 2020 11:21:15 +0200 Subject: [PATCH 008/180] nova: Use common launch guest helper The common helper adds automatic retry handling and resource cleanup. --- zaza/openstack/charm_tests/nova/tests.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index c4dd5b4..16c9337 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -38,32 +38,35 @@ class BaseGuestCreateTest(unittest.TestCase): zaza.openstack.configure.guest.launch_instance(instance_key) -class CirrosGuestCreateTest(BaseGuestCreateTest): +class CirrosGuestCreateTest(test_utils.OpenStackBaseTest): """Tests to launch a cirros image.""" def test_launch_small_instance(self): """Launch a cirros instance and test connectivity.""" - zaza.openstack.configure.guest.launch_instance( - glance_setup.CIRROS_IMAGE_NAME) + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'cirros', instance_key=glance_setup.CIRROS_IMAGE_NAME) -class LTSGuestCreateTest(BaseGuestCreateTest): +class LTSGuestCreateTest(test_utils.OpenStackBaseTest): """Tests to launch a LTS image.""" def test_launch_small_instance(self): """Launch a Bionic instance and test connectivity.""" - zaza.openstack.configure.guest.launch_instance( - glance_setup.LTS_IMAGE_NAME) + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME) -class LTSGuestCreateVolumeBackedTest(BaseGuestCreateTest): +class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): """Tests to launch a LTS image.""" def test_launch_small_instance(self): """Launch a Bionic instance and test connectivity.""" - zaza.openstack.configure.guest.launch_instance( - glance_setup.LTS_IMAGE_NAME, - use_boot_volume=True) + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'volume-backed-ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME, + use_boot_volmue=True) class NovaCompute(test_utils.OpenStackBaseTest): From 49268243ba11b8cb8c1510ffb2eea28943f5cb3b Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 22 Sep 2020 16:18:09 +0200 Subject: [PATCH 009/180] Improve logging --- zaza/openstack/charm_tests/cinder_backup/tests.py | 8 ++++---- zaza/openstack/utilities/openstack.py | 13 ++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 7d3442e..1cdd887 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -63,7 +63,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): self.cinder_client.volumes, vol_new.id, expected_status="available", - msg="Volume status wait") + msg="Extended volume") def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self): """Create, backup, delete, restore a ceph-backed cinder volume. @@ -99,7 +99,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): wait_iteration_max_time=180, stop_after_attempt=30, expected_status='available', - msg='Volume status wait') + msg='ceph-backed cinder volume') # Backup the volume vol_backup = self.cinder_client.backups.create( @@ -111,7 +111,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): wait_iteration_max_time=180, stop_after_attempt=30, expected_status='available', - msg='Volume status wait') + msg='Backup volume') # Delete the volume openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) # Restore the volume @@ -122,7 +122,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): wait_iteration_max_time=180, stop_after_attempt=15, expected_status='available', - msg='Backup status wait') + msg='Restored backup volume') # Delete the backup openstack_utils.delete_volume_backup( self.cinder_client, diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 35972c6..35e188d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1896,14 +1896,13 @@ def _resource_reaches_status(resource, resource_id, :param expected_status: status to expect resource to reach :type expected_status: str :param msg: text to identify purpose in logging - :type msy: str + :type msg: str :raises: AssertionError """ resource_status = resource.get(resource_id).status - logging.info(resource_status) - assert resource_status == expected_status, ( - "Resource in {} state, waiting for {}" .format(resource_status, - expected_status,)) + logging.info("{}: resource {} in {} state, waiting for {}".format( + msg, resource_id, resource_status, expected_status)) + assert resource_status == expected_status def resource_reaches_status(resource, @@ -1965,8 +1964,8 @@ def _resource_removed(resource, resource_id, msg="resource"): :raises: AssertionError """ matching = [r for r in resource.list() if r.id == resource_id] - logging.debug("Resource {} still present".format(resource_id)) - assert len(matching) == 0, "Resource {} still present".format(resource_id) + logging.debug("{}: resource {} still present".format(msg, resource_id)) + assert len(matching) == 0 def resource_removed(resource, From e70be304e5d1cc45ab07b8251dfad393260e486d Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Fri, 25 Sep 2020 18:22:09 +1200 Subject: [PATCH 010/180] Add tests for OVN charm NRPE checks The ovn-* charms have added NRPE service checks, this ensures that the service check files have been added to the units as expected. --- zaza/openstack/charm_tests/ovn/tests.py | 63 ++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index d12911e..30abddd 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -15,11 +15,14 @@ """Encapsulate OVN testing.""" import logging -import tenacity import juju +import tenacity + import zaza + +import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -36,10 +39,19 @@ class BaseCharmOperationTest(test_utils.BaseCharmTest): """Run class setup for OVN charm operation tests.""" super(BaseCharmOperationTest, cls).setUpClass() cls.services = ['NotImplemented'] # This must be overridden + cls.nrpe_checks = ['NotImplemented'] # This must be overridden cls.current_release = openstack_utils.get_os_release( openstack_utils.get_current_os_release_pair( cls.release_application or cls.application_name)) + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + # sleep for 2mins to allow 1min cron job to run... + wait=tenacity.wait_fixed(120), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_commands_on_units(self, cmds, units): + return generic_utils.check_commands_on_units(cmds, units) + def test_pause_resume(self): """Run pause and resume tests. @@ -50,6 +62,19 @@ class BaseCharmOperationTest(test_utils.BaseCharmTest): logging.info('Testing pause resume (services="{}")' .format(self.services)) + def test_nrpe_configured(self): + """Confirm that the NRPE service check files are created.""" + units = zaza.model.get_units(self.application_name) + cmds = [] + for check_name in self.nrpe_checks: + cmds.append( + 'egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_{}.cfg'.format(check_name) + ) + ret = self._retry_check_commands_on_units(cmds, units) + logging.info(ret) + self.assertIsNone(ret, msg=ret) + class CentralCharmOperationTest(BaseCharmOperationTest): """OVN Central Charm operation tests.""" @@ -62,6 +87,22 @@ class CentralCharmOperationTest(BaseCharmOperationTest): 'ovn-northd', 'ovsdb-server', ] + source = zaza.model.get_application_config( + cls.application_name)['source']['value'] + logging.info(source) + if 'train' in source: + cls.nrpe_checks = [ + 'ovn-northd', + 'ovn-nb-ovsdb', + 'ovn-sb-ovsdb', + ] + else: + # Ussuri or later (distro or cloudarchive) + cls.nrpe_checks = [ + 'ovn-northd', + 'ovn-ovsdb-server-sb', + 'ovn-ovsdb-server-nb', + ] class ChassisCharmOperationTest(BaseCharmOperationTest): @@ -76,6 +117,26 @@ class ChassisCharmOperationTest(BaseCharmOperationTest): cls.services = [ 'ovn-controller', ] + if cls.application_name == 'ovn-chassis': + principal_app_name = 'magpie' + else: + principal_app_name = cls.application_name + source = zaza.model.get_application_config( + principal_app_name)['source']['value'] + logging.info(source) + if 'train' in source: + cls.nrpe_checks = [ + 'ovn-host', + 'ovs-vswitchd', + 'ovsdb-server', + ] + else: + # Ussuri or later (distro or cloudarchive) + cls.nrpe_checks = [ + 'ovn-controller', + 'ovsdb-server', + 'ovs-vswitchd', + ] class OVSOVNMigrationTest(test_utils.BaseCharmTest): From a71a0f38ff4a8491c1c8fcd94f52eec2cedfa230 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 25 Sep 2020 14:57:18 +0200 Subject: [PATCH 011/180] Add some tenacity around cinder backup creation --- .../charm_tests/cinder_backup/tests.py | 63 +++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 1cdd887..1acbf9e 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -89,29 +89,43 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): self.assertEqual(pool_name, expected_pool) - # Create ceph-backed cinder volume - cinder_vol = self.cinder_client.volumes.create( - name='{}-410-vol'.format(self.RESOURCE_PREFIX), - size=1) - openstack_utils.resource_reaches_status( - self.cinder_client.volumes, - cinder_vol.id, - wait_iteration_max_time=180, - stop_after_attempt=30, - expected_status='available', - msg='ceph-backed cinder volume') + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3)): + with attempt: + # Create ceph-backed cinder volume + cinder_vol_name = '{}-410-vol-{}'.format( + self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) + cinder_vol = self.cinder_client.volumes.create( + name=cinder_vol_name, size=1) + openstack_utils.resource_reaches_status( + self.cinder_client.volumes, + cinder_vol.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='ceph-backed cinder volume') + + # Back up the volume + # NOTE(lourot): sometimes, especially on Mitaka, the backup + # remains stuck forever in 'creating' state and the volume in + # 'backing-up' state. See lp:1877076 + # Attempting to create another volume and another backup + # usually then succeeds. Release notes and bug trackers show + # that many things have been fixed and are still left to be + # fixed in this area. + # When the backup creation succeeds, it usually does within + # 12 minutes. + vol_backup_name = cinder_vol_name + '-backup' + vol_backup = self.cinder_client.backups.create( + cinder_vol.id, vol_backup_name) + openstack_utils.resource_reaches_status( + self.cinder_client.backups, + vol_backup.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='Backup volume') - # Backup the volume - vol_backup = self.cinder_client.backups.create( - cinder_vol.id, - name='{}-410-backup-vol'.format(self.RESOURCE_PREFIX)) - openstack_utils.resource_reaches_status( - self.cinder_client.backups, - vol_backup.id, - wait_iteration_max_time=180, - stop_after_attempt=30, - expected_status='available', - msg='Backup volume') # Delete the volume openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) # Restore the volume @@ -143,12 +157,11 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): obj_count_samples.append(obj_count) pool_size_samples.append(kb_used) - name = '{}-410-vol'.format(self.RESOURCE_PREFIX) vols = self.cinder_client.volumes.list() try: - cinder_vols = [v for v in vols if v.name == name] + cinder_vols = [v for v in vols if v.name == cinder_vol_name] except AttributeError: - cinder_vols = [v for v in vols if v.display_name == name] + cinder_vols = [v for v in vols if v.display_name == cinder_vol_name] if not cinder_vols: # NOTE(hopem): it appears that at some point cinder-backup stopped # restoring volume metadata properly so revert to default name if From f9d28f16b421699a478554f74b6b3a5895864610 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 28 Sep 2020 11:03:33 +0200 Subject: [PATCH 012/180] Make linter happy --- zaza/openstack/charm_tests/cinder_backup/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 1acbf9e..2281cc1 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -161,7 +161,8 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): try: cinder_vols = [v for v in vols if v.name == cinder_vol_name] except AttributeError: - cinder_vols = [v for v in vols if v.display_name == cinder_vol_name] + cinder_vols = [v for v in vols if + v.display_name == cinder_vol_name] if not cinder_vols: # NOTE(hopem): it appears that at some point cinder-backup stopped # restoring volume metadata properly so revert to default name if From 7a67d98b4585756398e47f745a445e91677062fb Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Fri, 2 Oct 2020 11:00:51 +1300 Subject: [PATCH 013/180] Only log OVN NRPE failure when it fails --- zaza/openstack/charm_tests/ovn/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 30abddd..c5eae5e 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -72,7 +72,8 @@ class BaseCharmOperationTest(test_utils.BaseCharmTest): 'check_{}.cfg'.format(check_name) ) ret = self._retry_check_commands_on_units(cmds, units) - logging.info(ret) + if ret: + logging.info(ret) self.assertIsNone(ret, msg=ret) From efcb7a7e0653a1b5a475064ced25608f1d94ee2f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Fri, 2 Oct 2020 10:50:10 +0200 Subject: [PATCH 014/180] Fix typo --- zaza/openstack/charm_tests/nova/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 16c9337..79e0e70 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -66,7 +66,7 @@ class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): self.RESOURCE_PREFIX = 'zaza-nova' self.launch_guest( 'volume-backed-ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME, - use_boot_volmue=True) + use_boot_volume=True) class NovaCompute(test_utils.OpenStackBaseTest): From 7c2f5cdf2493d39505b28d1b3243c6ab76b59e01 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Sat, 3 Oct 2020 00:23:05 +0000 Subject: [PATCH 015/180] Add Ironic tests --- requirements.txt | 1 + .../test_zaza_utilities_openstack.py | 14 +- zaza/openstack/charm_tests/glance/setup.py | 49 ++++- zaza/openstack/charm_tests/ironic/__init__.py | 15 ++ zaza/openstack/charm_tests/ironic/setup.py | 167 ++++++++++++++++++ zaza/openstack/charm_tests/ironic/tests.py | 83 +++++++++ zaza/openstack/utilities/openstack.py | 13 +- 7 files changed, 329 insertions(+), 13 deletions(-) create mode 100644 zaza/openstack/charm_tests/ironic/__init__.py create mode 100644 zaza/openstack/charm_tests/ironic/setup.py create mode 100644 zaza/openstack/charm_tests/ironic/tests.py diff --git a/requirements.txt b/requirements.txt index 8b15edc..cec0286 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ python-ceilometerclient python-cinderclient python-glanceclient python-heatclient +python-ironicclient python-keystoneclient python-manilaclient python-neutronclient diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 02bd578..5ad0b9b 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -504,7 +504,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): glance_mock.images.upload.assert_called_once_with( '9d1125af', f(), - ) + backend=None) self.resource_reaches_status.assert_called_once_with( glance_mock.images, '9d1125af', @@ -529,7 +529,11 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.upload_image_to_glance.assert_called_once_with( glance_mock, 'wibbly/c.img', - 'bob') + 'bob', + backend=None, + disk_format='qcow2', + visibility='public', + container_format='bare') def test_create_image_pass_directory(self): glance_mock = mock.MagicMock() @@ -549,7 +553,11 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.upload_image_to_glance.assert_called_once_with( glance_mock, 'tests/c.img', - 'bob') + 'bob', + backend=None, + disk_format='qcow2', + visibility='public', + container_format='bare') self.gettempdir.assert_not_called() def test_create_ssh_key(self): diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index ab0a3b3..6686d70 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -32,8 +32,37 @@ def basic_setup(): """ +def _get_default_glance_client(): + """Create default Glance client using overcloud credentials.""" + keystone_session = openstack_utils.get_overcloud_keystone_session() + glance_client = openstack_utils.get_glance_session_client(keystone_session) + return glance_client + + +def get_stores_info(glance_client=None): + """Retrieve glance backing store info. + + :param glance_client: Authenticated glanceclient + :type glance_client: glanceclient.Client + """ + glance_client = glance_client or _get_default_glance_client() + stores = glance_client.images.get_stores_info().get("stores", []) + return stores + + +def get_store_ids(glance_client=None): + """Retrieve glance backing store ids. + + :param glance_client: Authenticated glanceclient + :type glance_client: glanceclient.Client + """ + stores = get_stores_info(glance_client) + return [store["id"] for store in stores] + + def add_image(image_url, glance_client=None, image_name=None, tags=[], - properties=None): + properties=None, backend=None, disk_format='qcow2', + visibility='public', container_format='bare'): """Retrieve image from ``image_url`` and add it to glance. :param image_url: Retrievable URL with image data @@ -47,10 +76,14 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[], :param properties: Properties to add to image :type properties: dict """ - if not glance_client: - keystone_session = openstack_utils.get_overcloud_keystone_session() - glance_client = openstack_utils.get_glance_session_client( - keystone_session) + glance_client = glance_client or _get_default_glance_client() + if backend is not None: + stores = get_store_ids(glance_client) + if backend not in stores: + raise ValueError("Invalid backend: %(backend)s " + "(available: %(available)s)" % { + "backend": backend, + "available": ", ".join(stores)}) if image_name: image = openstack_utils.get_images_by_name( glance_client, image_name) @@ -65,7 +98,11 @@ def add_image(image_url, glance_client=None, image_name=None, tags=[], image_url, image_name, tags=tags, - properties=properties) + properties=properties, + backend=backend, + disk_format=disk_format, + visibility=visibility, + container_format=container_format) def add_cirros_image(glance_client=None, image_name=None): diff --git a/zaza/openstack/charm_tests/ironic/__init__.py b/zaza/openstack/charm_tests/ironic/__init__.py new file mode 100644 index 0000000..aedad05 --- /dev/null +++ b/zaza/openstack/charm_tests/ironic/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ironic.""" diff --git a/zaza/openstack/charm_tests/ironic/setup.py b/zaza/openstack/charm_tests/ironic/setup.py new file mode 100644 index 0000000..b03c8cb --- /dev/null +++ b/zaza/openstack/charm_tests/ironic/setup.py @@ -0,0 +1,167 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for configuring ironic.""" + +import copy +import os + +import zaza.openstack.charm_tests.glance.setup as glance_setup +import zaza.openstack.utilities.openstack as openstack_utils +from zaza.openstack.utilities import ( + cli as cli_utils, +) +import zaza.model as zaza_model + + +FLAVORS = { + 'bm1.small': { + 'flavorid': 2, + 'ram': 2048, + 'disk': 20, + 'vcpus': 1, + 'properties': { + "resources:CUSTOM_BAREMETAL1_SMALL": 1, + }, + }, + 'bm1.medium': { + 'flavorid': 3, + 'ram': 4096, + 'disk': 40, + 'vcpus': 2, + 'properties': { + "resources:CUSTOM_BAREMETAL1_MEDIUM": 1, + }, + }, + 'bm1.large': { + 'flavorid': 4, + 'ram': 8192, + 'disk': 40, + 'vcpus': 4, + 'properties': { + "resources:CUSTOM_BAREMETAL1_LARGE": 1, + }, + }, + 'bm1.tempest': { + 'flavorid': 6, + 'ram': 256, + 'disk': 1, + 'vcpus': 1, + 'properties': { + "resources:CUSTOM_BAREMETAL1_TEMPEST": 1, + }, + }, + 'bm2.tempest': { + 'flavorid': 7, + 'ram': 512, + 'disk': 1, + 'vcpus': 1, + 'properties': { + "resources:CUSTOM_BAREMETAL2_TEMPEST": 1, + }, + }, +} + + +def add_ironic_deployment_image(initrd_url=None, kernel_url=None): + """Add Ironic deploy images to glance. + + :param initrd_url: URL where the ari image resides + :type initrd_url: str + :param kernel_url: URL where the aki image resides + :type kernel_url: str + """ + base_name = 'ironic-deploy' + initrd_name = "{}-initrd".format(base_name) + vmlinuz_name = "{}-vmlinuz".format(base_name) + if not initrd_url: + initrd_url = os.environ.get('TEST_IRONIC_DEPLOY_INITRD', None) + if not kernel_url: + kernel_url = os.environ.get('TEST_IRONIC_DEPLOY_VMLINUZ', None) + if not all([initrd_url, kernel_url]): + raise ValueError("Missing required deployment image URLs") + glance_setup.add_image( + initrd_url, + image_name=initrd_name, + backend="swift", + disk_format="ari", + container_format="ari") + glance_setup.add_image( + kernel_url, + image_name=vmlinuz_name, + backend="swift", + disk_format="aki", + container_format="aki") + + +def add_ironic_os_image(image_url=None): + """Upload the operating system images built for bare metal deployments. + + :param image_url: URL where the image resides + :type image_url: str + """ + image_url = image_url or os.environ.get( + 'TEST_IRONIC_RAW_BM_IMAGE', None) + image_name = "baremetal-ubuntu-image" + if image_url is None: + raise ValueError("Missing image_url") + + glance_setup.add_image( + image_url, + image_name=image_name, + backend="swift", + disk_format="raw", + container_format="bare") + + +def set_temp_url_secret(): + """Run the set-temp-url-secret on the ironic-conductor leader. + + This is needed if direct boot method is enabled. + """ + zaza_model.run_action_on_leader( + 'ironic-conductor', + 'set-temp-url-secret', + action_params={}) + + +def create_bm_flavors(nova_client=None): + """Create baremetal flavors. + + :param nova_client: Authenticated nova client + :type nova_client: novaclient.v2.client.Client + """ + if not nova_client: + keystone_session = openstack_utils.get_overcloud_keystone_session() + nova_client = openstack_utils.get_nova_session_client( + keystone_session) + cli_utils.setup_logging() + names = [flavor.name for flavor in nova_client.flavors.list()] + # Disable scheduling based on standard flavor properties + default_properties = { + "resources:VCPU": 0, + "resources:MEMORY_MB": 0, + "resources:DISK_GB": 0, + } + for flavor in FLAVORS.keys(): + if flavor not in names: + properties = copy.deepcopy(default_properties) + properties.update(FLAVORS[flavor]["properties"]) + bm_flavor = nova_client.flavors.create( + name=flavor, + ram=FLAVORS[flavor]['ram'], + vcpus=FLAVORS[flavor]['vcpus'], + disk=FLAVORS[flavor]['disk'], + flavorid=FLAVORS[flavor]['flavorid']) + bm_flavor.set_keys(properties) diff --git a/zaza/openstack/charm_tests/ironic/tests.py b/zaza/openstack/charm_tests/ironic/tests.py new file mode 100644 index 0000000..9cfb85a --- /dev/null +++ b/zaza/openstack/charm_tests/ironic/tests.py @@ -0,0 +1,83 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate ironic testing.""" + +import logging + +import ironicclient.client as ironic_client +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +def _get_ironic_client(ironic_api_version="1.58"): + keystone_session = openstack_utils.get_overcloud_keystone_session() + ironic = ironic_client.Client(1, session=keystone_session, + os_ironic_api_version=ironic_api_version) + return ironic + + +class IronicTest(test_utils.OpenStackBaseTest): + """Run Ironic specific tests.""" + + _SERVICES = ['ironic-api'] + + def test_110_catalog_endpoints(self): + """Verify that the endpoints are present in the catalog.""" + overcloud_auth = openstack_utils.get_overcloud_auth() + keystone_client = openstack_utils.get_keystone_client( + overcloud_auth) + actual_endpoints = keystone_client.service_catalog.get_endpoints() + actual_interfaces = [endpoint['interface'] for endpoint in + actual_endpoints["baremetal"]] + for expected_interface in ('internal', 'admin', 'public'): + assert(expected_interface in actual_interfaces) + + def test_400_api_connection(self): + """Simple api calls to check service is up and responding.""" + ironic = _get_ironic_client() + + logging.info('listing conductors') + conductors = ironic.conductor.list() + assert(len(conductors) > 0) + + # By default, only IPMI HW type is enabled. iDrac and Redfish + # can optionally be enabled + drivers = ironic.driver.list() + driver_names = [drv.name for drv in drivers] + + expected = ['intel-ipmi', 'ipmi'] + for exp in expected: + assert(exp in driver_names) + assert(len(driver_names) == 2) + + def test_900_restart_on_config_change(self): + """Checking restart happens on config change. + + Change debug mode and assert that change propagates to the correct + file and that services are restarted as a result + """ + self.restart_on_changed_debug_oslo_config_file( + '/etc/ironic/ironic.conf', self._SERVICES) + + def test_910_pause_resume(self): + """Run pause and resume tests. + + Pause service and check services are stopped then resume and check + they are started + """ + logging.info('Skipping pause resume test LP: #1886202...') + return + with self.pause_resume(self._SERVICES): + logging.info("Testing pause resume") diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 35972c6..b62885e 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2060,7 +2060,8 @@ def delete_volume_backup(cinder, vol_backup_id): def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', - visibility='public', container_format='bare'): + visibility='public', container_format='bare', + backend=None): """Upload the given image to glance and apply the given label. :param glance: Authenticated glanceclient @@ -2086,7 +2087,7 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', disk_format=disk_format, visibility=visibility, container_format=container_format) - glance.images.upload(image.id, open(local_path, 'rb')) + glance.images.upload(image.id, open(local_path, 'rb'), backend=backend) resource_reaches_status( glance.images, @@ -2098,7 +2099,8 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], - properties=None): + properties=None, backend=None, disk_format='qcow2', + visibility='public', container_format='bare'): """Download the image and upload it to glance. Download an image from image_url and upload it to glance labelling @@ -2132,7 +2134,10 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], if not os.path.exists(local_path): download_image(image_url, local_path) - image = upload_image_to_glance(glance, local_path, image_name) + image = upload_image_to_glance( + glance, local_path, image_name, backend=backend, + disk_format=disk_format, visibility=visibility, + container_format=container_format) for tag in tags: result = glance.image_tags.update(image.id, tag) logging.debug( From 181bb98d2060f1edbacb8beb0555918f3e58c5e1 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 9 Oct 2020 09:39:00 +0200 Subject: [PATCH 016/180] Clean up newly created instances if faulty (#439) --- zaza/openstack/charm_tests/test_utils.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 1f24aa9..9157bd3 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -550,21 +550,23 @@ class OpenStackBaseTest(BaseCharmTest): instance_key = instance_key or glance_setup.LTS_IMAGE_NAME instance_name = '{}-{}'.format(self.RESOURCE_PREFIX, guest_name) - instance = self.retrieve_guest(instance_name) - if instance: - logging.info('Removing already existing instance ({}) with ' - 'requested name ({})' - .format(instance.id, instance_name)) - openstack_utils.delete_resource( - self.nova_client.servers, - instance.id, - msg="server") - for attempt in tenacity.Retrying( stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential( multiplier=1, min=2, max=10)): with attempt: + old_instance_with_same_name = self.retrieve_guest( + instance_name) + if old_instance_with_same_name: + logging.info( + 'Removing already existing instance ({}) with ' + 'requested name ({})' + .format(old_instance_with_same_name.id, instance_name)) + openstack_utils.delete_resource( + self.nova_client.servers, + old_instance_with_same_name.id, + msg="server") + return configure_guest.launch_instance( instance_key, vm_name=instance_name, From f27ee7c1f6972dbaa6b3e22b0259ac3dda8140a2 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 9 Oct 2020 11:44:03 +0200 Subject: [PATCH 017/180] Make determination of app under test more robust --- zaza/openstack/charm_tests/test_utils.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 9157bd3..74b76b2 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -127,10 +127,28 @@ class BaseCharmTest(unittest.TestCase): else: cls.model_name = model.get_juju_model() cls.test_config = lifecycle_utils.get_charm_config(fatal=False) + charm_under_test_name = cls.test_config['charm_name'] + if application_name: cls.application_name = application_name else: - cls.application_name = cls.test_config['charm_name'] + deployed_app_names = model.sync_deployed(model_name=cls.model_name) + if charm_under_test_name in deployed_app_names: + # There is an application named like the charm under test. + # Let's consider it the application under test: + cls.application_name = charm_under_test_name + else: + # Let's search for any application whose name starts with the + # name of the charm under test and assume it's the application + # under test: + for app_name in deployed_app_names: + if app_name.startswith(charm_under_test_name): + cls.application_name = app_name + break + else: + logging.warning('Could not find application under test') + return + cls.lead_unit = model.get_lead_unit_name( cls.application_name, model_name=cls.model_name) From 299fae65dbc561ad766cef3da2a3b2c2e0f84ba4 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 12 Oct 2020 12:03:44 +0000 Subject: [PATCH 018/180] Fix test runs with no charm_name Fix z-o-t so that tests that specify an application name can be run irrespective of whether the charm_name has been set in tests.yaml. Closes issue #442 --- zaza/openstack/charm_tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 74b76b2..493a868 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -127,11 +127,11 @@ class BaseCharmTest(unittest.TestCase): else: cls.model_name = model.get_juju_model() cls.test_config = lifecycle_utils.get_charm_config(fatal=False) - charm_under_test_name = cls.test_config['charm_name'] if application_name: cls.application_name = application_name else: + charm_under_test_name = cls.test_config['charm_name'] deployed_app_names = model.sync_deployed(model_name=cls.model_name) if charm_under_test_name in deployed_app_names: # There is an application named like the charm under test. From 1889d0db530ecc03f61976815d0704040b62e0e1 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Mon, 12 Oct 2020 17:51:04 +0200 Subject: [PATCH 019/180] Add func-tests for ceph-osd 'service' action --- zaza/openstack/charm_tests/ceph/osd/tests.py | 177 +++++++++++++++++++ 1 file changed, 177 insertions(+) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index e3f7be1..b8080c0 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -16,6 +16,10 @@ import logging import unittest +import re + +from copy import deepcopy +from typing import List import zaza.openstack.charm_tests.test_utils as test_utils import zaza.model as zaza_model @@ -47,3 +51,176 @@ class SecurityTest(unittest.TestCase): expected_passes, expected_failures, expected_to_pass=True) + + +class OsdService: + """Simple representation of ceph-osd systemd service.""" + + def __init__(self, id_: int): + """ + Init service using its ID. + + e.g.: id_=1 -> ceph.-osd@1.service + """ + self.id = id_ + self.name = 'ceph-osd@{}'.format(id_) + + +class ServiceTest(unittest.TestCase): + """ceph-osd systemd service tests.""" + + TESTED_UNIT = 'ceph-osd/0' + SERVICE_PATTERN = re.compile(r'ceph-osd@(?P\d+)\.service') + + def __init__(self, methodName='runTest'): + """Initialize Test Case.""" + super(ServiceTest, self).__init__(methodName) + self._available_services = None + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph service tests.""" + super(ServiceTest, cls).setUpClass() + + def setUp(self): + """Run test setup.""" + # Note: This counter reset is needed because ceph-osd service is + # limited to 3 restarts per 30 mins which is insufficient + # when running functional tests for 'service' action. This + # limitation is defined in /lib/systemd/system/ceph-osd@.service + # in section [Service] with options 'StartLimitInterval' and + # 'StartLimitBurst' + reset_counter = 'systemctl reset-failed' + zaza_model.run_on_unit(self.TESTED_UNIT, reset_counter) + + def tearDown(self): + """Start ceph-osd services after each test. + + This ensures that the environment is ready for the next tests. + """ + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': 'all'}, + raise_on_failure=True) + + @property + def available_services(self) -> List[OsdService]: + """Return list of all ceph-osd services present on the TESTED_UNIT.""" + if self._available_services is None: + self._available_services = self._fetch_osd_services() + return self._available_services + + def _fetch_osd_services(self) -> List[OsdService]: + """Fetch all ceph-osd services present on the TESTED_UNIT.""" + service_list = [] + service_list_cmd = 'systemctl list-units --full --all ' \ + '--no-pager -t service' + result = zaza_model.run_on_unit(self.TESTED_UNIT, service_list_cmd) + for line in result['Stdout'].split('\n'): + service_name = self.SERVICE_PATTERN.search(line) + if service_name: + service_id = int(service_name.group('service_id')) + service_list.append(OsdService(service_id)) + return service_list + + def test_start_stop_all_by_keyword(self): + """Start and Stop all ceph-osd services using keyword 'all'.""" + service_list = [service.name for service in self.available_services] + + logging.info("Running 'service stop=all' action on {} " + "unit".format(self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'stop': 'all'}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') + + logging.info("Running 'service start=all' action on {} " + "unit".format(self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': 'all'}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running', + pgrep_full=True) + + def test_start_stop_all_by_list(self): + """Start and Stop all ceph-osd services using explicit list.""" + service_list = [service.name for service in self.available_services] + service_ids = [str(service.id) for service in self.available_services] + action_params = ','.join(service_ids) + + logging.info("Running 'service stop={}' action on {} " + "unit".format(action_params, self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'stop': action_params}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') + + logging.info("Running 'service start={}' action on {} " + "unit".format(action_params, self.TESTED_UNIT)) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': action_params}) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running', + pgrep_full=True) + + def test_stop_specific(self): + """Stop only specified ceph-osd service.""" + if len(self.available_services) < 2: + raise unittest.SkipTest('This test can be performed only if ' + 'there\'s more than one ceph-osd service ' + 'present on the tested unit') + + should_run = deepcopy(self.available_services) + to_stop = should_run.pop() + should_run = [service.name for service in should_run] + + logging.info("Running 'service stop={} on {} " + "unit".format(to_stop.id, self.TESTED_UNIT)) + + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'stop': to_stop.id}) + + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=[to_stop.name, ], + target_status='stopped') + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=should_run, + target_status='running', + pgrep_full=True) + + def test_start_specific(self): + """Start only specified ceph-osd service.""" + if len(self.available_services) < 2: + raise unittest.SkipTest('This test can be performed only if ' + 'there\'s more than one ceph-osd service ' + 'present on the tested unit') + + service_names = [service.name for service in self.available_services] + should_stop = deepcopy(self.available_services) + to_start = should_stop.pop() + should_stop = [service.name for service in should_stop] + + logging.info("Stopping all running ceph-osd services") + service_stop_cmd = 'systemctl stop ceph-osd.target' + zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd) + + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=service_names, + target_status='stopped') + + logging.info("Running 'service start={} on {} " + "unit".format(to_start.id, self.TESTED_UNIT)) + + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + action_params={'start': to_start.id}) + + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=[to_start.name, ], + target_status='running', + pgrep_full=True) + zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, + services=should_stop, + target_status='stopped') From d7893798cafbc2a6346b9b924ea6fad5cf5f3a1b Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Mon, 31 Aug 2020 12:08:43 +0530 Subject: [PATCH 020/180] Add new keystone-ldap charm options in LDAP tests New charm options for keystone-ldap are getting introdcued as part of bug [1]. Modify the keystone ldap tests to modify ldap options as per the new charm options. Add ldap options to verify ldap-config-flags takes precendece over explicit config flags. [1] https://bugs.launchpad.net/charm-keystone-ldap/+bug/1832765 --- zaza/openstack/charm_tests/keystone/tests.py | 35 ++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index ed5f42a..9abe78c 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -455,3 +455,38 @@ class LdapTests(BaseKeystoneTest): self.assertIsNotNone(johndoe, "user 'john doe' was unknown") janedoe = self._find_keystone_v3_user('jane doe', 'userdomain') self.assertIsNotNone(janedoe, "user 'jane doe' was unknown") + + +class LdapExplicitCharmConfigTests(LdapTests): + """Keystone ldap tests tests.""" + + def _get_ldap_config(self): + """Generate ldap config for current model. + + :return: tuple of whether ldap-server is running and if so, config + for the keystone-ldap application. + :rtype: Tuple[bool, Dict[str,str]] + """ + ldap_ips = zaza.model.get_app_ips("ldap-server") + self.assertTrue(ldap_ips, "Should be at least one ldap server") + return { + 'ldap-server': "ldap://{}".format(ldap_ips[0]), + 'ldap-user': 'cn=admin,dc=test,dc=com', + 'ldap-password': 'crapper', + 'ldap-suffix': 'dc=test,dc=com', + 'domain-name': 'userdomain', + 'ldap-query-scope': 'one', + 'ldap-user-objectclass': 'inetOrgPerson', + 'ldap-user-id-attribute': 'cn', + 'ldap-user-name-attribute': 'sn', + 'ldap-user-enabled-attribute': 'enabled', + 'ldap-user-enabled-invert': False, + 'ldap-user-enabled-mask': 0, + 'ldap-user-enabled-default': 'True', + 'ldap-group-tree-dn': 'ou=groups', + 'ldap-group-objectclass': 'groupOfNames', + 'ldap-group-id-attribute': 'cn', + 'ldap-group-member-attribute': 'memberUid', + 'ldap-group-members-are-ids': True, + 'ldap-config-flags': '{group_objectclass: "posixGroup"}', + } From b44e556d0aa08abd0c252f64d5d201db46c117a2 Mon Sep 17 00:00:00 2001 From: Xav Paice Date: Tue, 13 Oct 2020 20:21:46 +1300 Subject: [PATCH 021/180] Add tests for NRPE checks in aodh Adds a test to ensure that Aodh units related to nrpe get service check files created. --- zaza/openstack/charm_tests/aodh/tests.py | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/zaza/openstack/charm_tests/aodh/tests.py b/zaza/openstack/charm_tests/aodh/tests.py index e0178df..3b966e5 100644 --- a/zaza/openstack/charm_tests/aodh/tests.py +++ b/zaza/openstack/charm_tests/aodh/tests.py @@ -25,6 +25,7 @@ import zaza.model import zaza.openstack.configure.guest import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.generic as generic_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.configure.telemetry as telemetry_utils @@ -70,6 +71,13 @@ class AodhTest(test_utils.OpenStackBaseTest): """Check that aodh api is responding.""" self.aodh_client.alarm.list() + @tenacity.retry( + retry=tenacity.retry_if_result(lambda ret: ret is not None), + wait=tenacity.wait_fixed(120), + stop=tenacity.stop_after_attempt(2)) + def _retry_check_commands_on_units(self, cmds, units): + return generic_utils.check_commands_on_units(cmds, units) + @property def services(self): """Return a list of the service that should be running.""" @@ -139,6 +147,26 @@ class AodhTest(test_utils.OpenStackBaseTest): logging.info("Testing pause resume") self.query_aodh_api() + def test_902_nrpe_service_checks(self): + """Confirm that the NRPE service check files are created.""" + units = zaza.model.get_units('aodh') + cmds = [] + if self.release >= self.xenial_ocata: + services = ['aodh-evaluator', 'aodh-notifier', + 'aodh-listener', 'apache2'] + else: + services = ['aodh-api', 'aodh-evaluator', + 'aodh-notifier', 'aodh-listener'] + for check_name in services: + cmds.append( + 'egrep -oh /usr/local.* /etc/nagios/nrpe.d/' + 'check_{}.cfg'.format(check_name) + ) + ret = self._retry_check_commands_on_units(cmds, units) + if ret: + logging.info(ret) + self.assertIsNone(ret, msg=ret) + class AodhServerAlarmTest(test_utils.OpenStackBaseTest): """Test server events trigger Aodh alarms.""" From 24755c23820167dbcc45b09eb0ec922da9f74a7c Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Wed, 14 Oct 2020 13:47:20 +0200 Subject: [PATCH 022/180] Add function that waits on desired service status using `systemctl` --- zaza/openstack/charm_tests/ceph/osd/tests.py | 108 +++++++++++++------ 1 file changed, 77 insertions(+), 31 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index b8080c0..58bc5bc 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -66,6 +66,55 @@ class OsdService: self.name = 'ceph-osd@{}'.format(id_) +async def async_wait_for_service_status(unit_name, services, target_status, + model_name=None, timeout=2700): + """Wait for all services on the unit are in the desired state. + + Note:This function emulates the + `zaza.model.async_block_until_service_status` function, but it's using + `systemctl is-active` command istead of `pidof/pgrep` of the original + function. + + :param unit_name: Name of unit to run action on + :type unit_name: str + :param services: List of services to check + :type services: [] + :param target_status: State services must be in (stopped or running) + :type target_status: str + :param model_name: Name of model to query. + :type model_name: str + :param timeout: Time to wait for status to be achieved + :type timeout: int + """ + async def _check_service(): + for service in services: + command = r"systemctl is-active '{}'".format(service) + out = await zaza_model.async_run_on_unit( + unit_name, + command, + model_name=model_name, + timeout=timeout) + response = out['Stdout'].strip() + + if target_status == "running" and response == 'active': + return True + elif target_status == "stopped" and response == 'inactive': + return True + else: + return False + logging.info("Unsing fancy method") + accepted_states = ('stopped', 'running') + if target_status not in accepted_states: + raise RuntimeError('Invalid target state "{}". Accepted states: ' + '{}'.format(target_status, accepted_states)) + + async with zaza_model.run_in_model(model_name): + await zaza_model.async_block_until(_check_service, timeout=timeout) + + +wait_for_service = zaza_model.sync_wrapper(async_wait_for_service_status) + + class ServiceTest(unittest.TestCase): """ceph-osd systemd service tests.""" @@ -130,18 +179,17 @@ class ServiceTest(unittest.TestCase): "unit".format(self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'stop': 'all'}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') logging.info("Running 'service start=all' action on {} " "unit".format(self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'start': 'all'}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='running', - pgrep_full=True) + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running') def test_start_stop_all_by_list(self): """Start and Stop all ceph-osd services using explicit list.""" @@ -153,18 +201,17 @@ class ServiceTest(unittest.TestCase): "unit".format(action_params, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'stop': action_params}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='stopped') logging.info("Running 'service start={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'start': action_params}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_list, - target_status='running', - pgrep_full=True) + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_list, + target_status='running') def test_stop_specific(self): """Stop only specified ceph-osd service.""" @@ -183,13 +230,12 @@ class ServiceTest(unittest.TestCase): zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'stop': to_stop.id}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=[to_stop.name, ], - target_status='stopped') - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=should_run, - target_status='running', - pgrep_full=True) + wait_for_service(unit_name=self.TESTED_UNIT, + services=[to_stop.name, ], + target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=should_run, + target_status='running') def test_start_specific(self): """Start only specified ceph-osd service.""" @@ -207,9 +253,9 @@ class ServiceTest(unittest.TestCase): service_stop_cmd = 'systemctl stop ceph-osd.target' zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=service_names, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=service_names, + target_status='stopped') logging.info("Running 'service start={} on {} " "unit".format(to_start.id, self.TESTED_UNIT)) @@ -217,10 +263,10 @@ class ServiceTest(unittest.TestCase): zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', action_params={'start': to_start.id}) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=[to_start.name, ], - target_status='running', - pgrep_full=True) - zaza_model.block_until_service_status(unit_name=self.TESTED_UNIT, - services=should_stop, - target_status='stopped') + wait_for_service(unit_name=self.TESTED_UNIT, + services=[to_start.name, ], + target_status='running') + + wait_for_service(unit_name=self.TESTED_UNIT, + services=should_stop, + target_status='stopped') From 12db27016eb348394cc8fd64d98924985da4b0a2 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Wed, 14 Oct 2020 13:55:23 +0200 Subject: [PATCH 023/180] Fix typos --- zaza/openstack/charm_tests/ceph/osd/tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 58bc5bc..35b6de4 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -68,11 +68,11 @@ class OsdService: async def async_wait_for_service_status(unit_name, services, target_status, model_name=None, timeout=2700): - """Wait for all services on the unit are in the desired state. + """Wait for all services on the unit to be in the desired state. - Note:This function emulates the + Note: This function emulates the `zaza.model.async_block_until_service_status` function, but it's using - `systemctl is-active` command istead of `pidof/pgrep` of the original + `systemctl is-active` command instead of `pidof/pgrep` of the original function. :param unit_name: Name of unit to run action on @@ -102,7 +102,7 @@ async def async_wait_for_service_status(unit_name, services, target_status, return True else: return False - logging.info("Unsing fancy method") + accepted_states = ('stopped', 'running') if target_status not in accepted_states: raise RuntimeError('Invalid target state "{}". Accepted states: ' From 440ee4b274f6ea71c312dad332cd1f08daf42267 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sun, 18 Oct 2020 22:28:32 +0200 Subject: [PATCH 024/180] octavia: Disable check for member operational status Temporarily disable this check until we figure out why operational_status sometimes does not become 'ONLINE' while the member does indeed work and the subsequent retrieval of payload through loadbalancer is successful ref LP: #1896729. --- zaza/openstack/charm_tests/octavia/tests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index eb0b217..e95a33f 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -280,7 +280,13 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): lambda x: octavia_client.member_show( pool_id=pool_id, member_id=x), member_id, - operating_status='ONLINE' if monitor else '') + operating_status='') + # Temporarily disable this check until we figure out why + # operational_status sometimes does not become 'ONLINE' + # while the member does indeed work and the subsequent + # retrieval of payload through loadbalancer is successful + # ref LP: #1896729. + # operating_status='ONLINE' if monitor else '') logging.info(resp) return lb From 4f83645b1c51857eb9bf073f564d00ff773df4c4 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Mon, 19 Oct 2020 08:40:23 +0000 Subject: [PATCH 025/180] Add python-ironicclient to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index f4e31f8..b18cd9a 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ install_require = [ 'python-barbicanclient>=4.0.1,<5.0.0', 'python-designateclient>=1.5,<3.0.0', 'python-heatclient<2.0.0', + 'python-ironicclient', 'python-glanceclient<3.0.0', 'python-keystoneclient<3.22.0', 'python-manilaclient<2.0.0', From a75d49d9595f9d34c1a96e2371517f140306ffa2 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Mon, 19 Oct 2020 11:44:13 +0000 Subject: [PATCH 026/180] Add tenacity to image uploads --- zaza/openstack/charm_tests/ironic/setup.py | 29 +++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/ironic/setup.py b/zaza/openstack/charm_tests/ironic/setup.py index b03c8cb..fe7d034 100644 --- a/zaza/openstack/charm_tests/ironic/setup.py +++ b/zaza/openstack/charm_tests/ironic/setup.py @@ -16,6 +16,7 @@ import copy import os +import tenacity import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.utilities.openstack as openstack_utils @@ -74,6 +75,20 @@ FLAVORS = { } +def _add_image(url, image_name, backend="swift", + disk_format="raw", container_format="bare"): + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + reraise=True): + with attempt: + glance_setup.add_image( + url, + image_name=image_name, + backend=backend, + disk_format=disk_format, + container_format=container_format) + + def add_ironic_deployment_image(initrd_url=None, kernel_url=None): """Add Ironic deploy images to glance. @@ -91,15 +106,17 @@ def add_ironic_deployment_image(initrd_url=None, kernel_url=None): kernel_url = os.environ.get('TEST_IRONIC_DEPLOY_VMLINUZ', None) if not all([initrd_url, kernel_url]): raise ValueError("Missing required deployment image URLs") - glance_setup.add_image( + + _add_image( initrd_url, - image_name=initrd_name, + initrd_name, backend="swift", disk_format="ari", container_format="ari") - glance_setup.add_image( + + _add_image( kernel_url, - image_name=vmlinuz_name, + vmlinuz_name, backend="swift", disk_format="aki", container_format="aki") @@ -117,9 +134,9 @@ def add_ironic_os_image(image_url=None): if image_url is None: raise ValueError("Missing image_url") - glance_setup.add_image( + _add_image( image_url, - image_name=image_name, + image_name, backend="swift", disk_format="raw", container_format="bare") From 67aaa10ad6f15d9be513a5205b7871a18f90ebf2 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 20 Oct 2020 14:43:47 +0200 Subject: [PATCH 027/180] Retry Nova flavor- and keypair-creation Fixes #452 --- zaza/openstack/charm_tests/nova/setup.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/nova/setup.py b/zaza/openstack/charm_tests/nova/setup.py index c556c97..7f8ee53 100644 --- a/zaza/openstack/charm_tests/nova/setup.py +++ b/zaza/openstack/charm_tests/nova/setup.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Code for configureing nova.""" +"""Code for configuring nova.""" + +import tenacity import zaza.openstack.utilities.openstack as openstack_utils from zaza.openstack.utilities import ( @@ -21,6 +23,9 @@ from zaza.openstack.utilities import ( import zaza.openstack.charm_tests.nova.utils as nova_utils +@tenacity.retry(stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)) def create_flavors(nova_client=None): """Create basic flavors. @@ -43,6 +48,9 @@ def create_flavors(nova_client=None): flavorid=nova_utils.FLAVORS[flavor]['flavorid']) +@tenacity.retry(stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10)) def manage_ssh_key(nova_client=None): """Create basic flavors. From 79e43c7f684f18839a5ff59457cfc8e9116b3b59 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 22 Oct 2020 15:28:16 +0100 Subject: [PATCH 028/180] bionic-train -> ussuri needs to upgrade placement For the openstack upgrade of train to ussuri, the placement charm needs to be upgraded. Add it to the UPGRADE_SERVICES. Related launchpad bug: LP:1724174 [1] [1]: https://bugs.launchpad.net/openstack-mojo-specs/+bug/1724174 --- zaza/openstack/utilities/openstack.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index b62885e..8d25433 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -123,6 +123,10 @@ CHARM_TYPES = { 'pkg': 'ceph-common', 'origin_setting': 'source' }, + 'placement': { + 'pkg': 'placement-common', + 'origin_setting': 'openstack-origin' + }, } # Older tests use the order the services appear in the list to imply @@ -143,6 +147,7 @@ UPGRADE_SERVICES = [ 'type': CHARM_TYPES['openstack-dashboard']}, {'name': 'ovn-central', 'type': CHARM_TYPES['ovn-central']}, {'name': 'ceph-mon', 'type': CHARM_TYPES['ceph-mon']}, + {'name': 'placement', 'type': CHARM_TYPES['placement']}, ] From 990be121717396802a496de1fd8453de2ef67029 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Wed, 28 Oct 2020 13:58:49 +0100 Subject: [PATCH 029/180] Fix async_wait_for_service_status() checking only first service in the list --- zaza/openstack/charm_tests/ceph/osd/tests.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 35b6de4..6f1bfac 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -60,7 +60,7 @@ class OsdService: """ Init service using its ID. - e.g.: id_=1 -> ceph.-osd@1.service + e.g.: id_=1 -> ceph-osd@1 """ self.id = id_ self.name = 'ceph-osd@{}'.format(id_) @@ -87,6 +87,7 @@ async def async_wait_for_service_status(unit_name, services, target_status, :type timeout: int """ async def _check_service(): + services_ok = True for service in services: command = r"systemctl is-active '{}'".format(service) out = await zaza_model.async_run_on_unit( @@ -97,11 +98,14 @@ async def async_wait_for_service_status(unit_name, services, target_status, response = out['Stdout'].strip() if target_status == "running" and response == 'active': - return True + continue elif target_status == "stopped" and response == 'inactive': - return True + continue else: - return False + services_ok = False + break + + return services_ok accepted_states = ('stopped', 'running') if target_status not in accepted_states: From 991d7ea7a41fe86ed5c5abe36966214514c02cce Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Tue, 3 Nov 2020 15:47:18 +0100 Subject: [PATCH 030/180] Removed `typing` usage --- zaza/openstack/charm_tests/ceph/osd/tests.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 6f1bfac..03e212f 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -19,7 +19,6 @@ import unittest import re from copy import deepcopy -from typing import List import zaza.openstack.charm_tests.test_utils as test_utils import zaza.model as zaza_model @@ -56,7 +55,7 @@ class SecurityTest(unittest.TestCase): class OsdService: """Simple representation of ceph-osd systemd service.""" - def __init__(self, id_: int): + def __init__(self, id_): """ Init service using its ID. @@ -78,7 +77,7 @@ async def async_wait_for_service_status(unit_name, services, target_status, :param unit_name: Name of unit to run action on :type unit_name: str :param services: List of services to check - :type services: [] + :type services: List[str] :param target_status: State services must be in (stopped or running) :type target_status: str :param model_name: Name of model to query. @@ -122,7 +121,7 @@ wait_for_service = zaza_model.sync_wrapper(async_wait_for_service_status) class ServiceTest(unittest.TestCase): """ceph-osd systemd service tests.""" - TESTED_UNIT = 'ceph-osd/0' + TESTED_UNIT = 'ceph-osd/0' # This can be any ceph-osd unit in the model SERVICE_PATTERN = re.compile(r'ceph-osd@(?P\d+)\.service') def __init__(self, methodName='runTest'): @@ -156,13 +155,13 @@ class ServiceTest(unittest.TestCase): raise_on_failure=True) @property - def available_services(self) -> List[OsdService]: + def available_services(self): """Return list of all ceph-osd services present on the TESTED_UNIT.""" if self._available_services is None: self._available_services = self._fetch_osd_services() return self._available_services - def _fetch_osd_services(self) -> List[OsdService]: + def _fetch_osd_services(self): """Fetch all ceph-osd services present on the TESTED_UNIT.""" service_list = [] service_list_cmd = 'systemctl list-units --full --all ' \ @@ -181,7 +180,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop=all' action on {} " "unit".format(self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', + zaza_model.run_action_on_units([self.TESTED_UNIT], 'service', action_params={'stop': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, From a91ae76975d40dd94fc211368af964b9084ee0fa Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 6 Nov 2020 14:12:59 +0000 Subject: [PATCH 031/180] Stop assuming there are two iscsi gateways The tests assume there are two iscsi gateways which is not a safe assumption. --- .../openstack/charm_tests/ceph/iscsi/tests.py | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/iscsi/tests.py b/zaza/openstack/charm_tests/ceph/iscsi/tests.py index a986833..0766e4a 100644 --- a/zaza/openstack/charm_tests/ceph/iscsi/tests.py +++ b/zaza/openstack/charm_tests/ceph/iscsi/tests.py @@ -64,26 +64,25 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): :rtype: Dict """ gw_units = zaza.model.get_units('ceph-iscsi') - primary_gw = gw_units[0] - secondary_gw = gw_units[1] host_names = generic_utils.get_unit_hostnames(gw_units, fqdn=True) client_entity_ids = [ u.entity_id for u in zaza.model.get_units('ubuntu')] ctxt = { 'client_entity_ids': sorted(client_entity_ids), 'gw_iqn': self.GW_IQN, - 'gw1_ip': primary_gw.public_address, - 'gw1_hostname': host_names[primary_gw.entity_id], - 'gw1_entity_id': primary_gw.entity_id, - 'gw2_ip': secondary_gw.public_address, - 'gw2_hostname': host_names[secondary_gw.entity_id], - 'gw2_entity_id': secondary_gw.entity_id, 'chap_creds': 'username={chap_username} password={chap_password}', 'gwcli_gw_dir': '/iscsi-targets/{gw_iqn}/gateways', 'gwcli_hosts_dir': '/iscsi-targets/{gw_iqn}/hosts', 'gwcli_disk_dir': '/disks', 'gwcli_client_dir': '{gwcli_hosts_dir}/{client_initiatorname}', } + ctxt['gateway_units'] = [ + { + 'entity_id': u.entity_id, + 'ip': u.public_address, + 'hostname': host_names[u.entity_id]} + for u in zaza.model.get_units('ceph-iscsi')] + ctxt['gw_ip'] = sorted([g['ip'] for g in ctxt['gateway_units']])[0] return ctxt def run_commands(self, unit_name, commands, ctxt): @@ -116,9 +115,8 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): 'ceph-iscsi', 'create-target', action_params={ - 'gateway-units': '{} {}'.format( - ctxt['gw1_entity_id'], - ctxt['gw2_entity_id']), + 'gateway-units': ' '.join([g['entity_id'] + for g in ctxt['gateway_units']]), 'iqn': self.GW_IQN, 'rbd-pool-name': ctxt.get('pool_name', ''), 'ec-rbd-metadata-pool': ctxt.get('ec_meta_pool_name', ''), @@ -139,7 +137,7 @@ class CephISCSIGatewayTest(test_utils.BaseCharmTest): base_op_cmd = ('iscsiadm --mode node --targetname {gw_iqn} ' '--op=update ').format(**ctxt) setup_cmds = [ - 'iscsiadm -m discovery -t st -p {gw1_ip}', + 'iscsiadm -m discovery -t st -p {gw_ip}', base_op_cmd + '-n node.session.auth.authmethod -v CHAP', base_op_cmd + '-n node.session.auth.username -v {chap_username}', base_op_cmd + '-n node.session.auth.password -v {chap_password}', From c91ebb5ec42d12acfbd12af73fc124378dc5805e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 9 Nov 2020 10:38:23 +0100 Subject: [PATCH 032/180] Remove unused security tests (#441) --- .../charm_tests/security/__init__.py | 15 ---- zaza/openstack/charm_tests/security/tests.py | 81 ------------------- 2 files changed, 96 deletions(-) delete mode 100644 zaza/openstack/charm_tests/security/__init__.py delete mode 100644 zaza/openstack/charm_tests/security/tests.py diff --git a/zaza/openstack/charm_tests/security/__init__.py b/zaza/openstack/charm_tests/security/__init__.py deleted file mode 100644 index ec47696..0000000 --- a/zaza/openstack/charm_tests/security/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2018 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test security checklist.""" diff --git a/zaza/openstack/charm_tests/security/tests.py b/zaza/openstack/charm_tests/security/tests.py deleted file mode 100644 index 741b53b..0000000 --- a/zaza/openstack/charm_tests/security/tests.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2018 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Encapsulate general security testing.""" - -import unittest - -import zaza.model as model -import zaza.charm_lifecycle.utils as utils -from zaza.openstack.utilities.file_assertions import ( - assert_path_glob, - assert_single_file, -) - - -def _make_test_function(application, file_details, paths=None): - """Generate a test function given the specified inputs. - - :param application: Application name to assert file ownership on - :type application: str - :param file_details: Dictionary of file details to test - :type file_details: dict - :param paths: List of paths to test in this application - :type paths: Optional[list(str)] - :returns: Test function - :rtype: unittest.TestCase - """ - def test(self): - for unit in model.get_units(application): - unit = unit.entity_id - if '*' in file_details['path']: - assert_path_glob(self, unit, file_details, paths) - else: - assert_single_file(self, unit, file_details) - return test - - -def _add_tests(): - """Add tests to the unittest.TestCase.""" - def class_decorator(cls): - """Add tests based on input yaml to `cls`.""" - files = utils.get_charm_config('./file-assertions.yaml') - deployed_applications = model.sync_deployed() - for name, attributes in files.items(): - # Lets make sure to only add tests for deployed applications - if name in deployed_applications: - paths = [ - file['path'] for - file in attributes['files'] - if "*" not in file["path"] - ] - for file in attributes['files']: - test_func = _make_test_function(name, file, paths=paths) - setattr( - cls, - 'test_{}_{}'.format(name, file['path']), - test_func) - return cls - return class_decorator - - -class FileOwnershipTest(unittest.TestCase): - """Encapsulate File ownership tests.""" - - pass - - -FileOwnershipTest = _add_tests()(FileOwnershipTest) From 82c1a5321b0cf6498b161059f4d1f8d5b8e719fe Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 10 Nov 2020 15:34:32 +0100 Subject: [PATCH 033/180] Fix volume and backup name --- zaza/openstack/charm_tests/cinder_backup/tests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 2281cc1..41da8ab 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -93,7 +93,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): stop=tenacity.stop_after_attempt(3)): with attempt: # Create ceph-backed cinder volume - cinder_vol_name = '{}-410-vol-{}'.format( + cinder_vol_name = '{}-410{}-vol'.format( self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) cinder_vol = self.cinder_client.volumes.create( name=cinder_vol_name, size=1) @@ -115,7 +115,12 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): # fixed in this area. # When the backup creation succeeds, it usually does within # 12 minutes. - vol_backup_name = cinder_vol_name + '-backup' + # NOTE(lourot): it seems like we have to stick to the + # `-backup-vol` naming convention otherwise + # cinder-backup fails with a ceph/rados error about not being + # able to find the pool. See lp:1897587 + vol_backup_name = '{}-410{}-backup-vol'.format( + self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) vol_backup = self.cinder_client.backups.create( cinder_vol.id, vol_backup_name) openstack_utils.resource_reaches_status( From ca40adc07878543896ef369f28b5eafe8a3a6f83 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 11 Nov 2020 08:44:51 +0100 Subject: [PATCH 034/180] Fix comment --- zaza/openstack/charm_tests/cinder_backup/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 41da8ab..c16e207 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -116,7 +116,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): # When the backup creation succeeds, it usually does within # 12 minutes. # NOTE(lourot): it seems like we have to stick to the - # `-backup-vol` naming convention otherwise + # `--backup-vol` naming convention otherwise # cinder-backup fails with a ceph/rados error about not being # able to find the pool. See lp:1897587 vol_backup_name = '{}-410{}-backup-vol'.format( From 64fd8026975837282f335790ea1f8af5ac7882df Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 11 Nov 2020 14:55:09 +0100 Subject: [PATCH 035/180] Fix call to cinder_client.backups.create() --- zaza/openstack/charm_tests/cinder_backup/tests.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index c16e207..97b3658 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -93,7 +93,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): stop=tenacity.stop_after_attempt(3)): with attempt: # Create ceph-backed cinder volume - cinder_vol_name = '{}-410{}-vol'.format( + cinder_vol_name = '{}-410-{}-vol'.format( self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) cinder_vol = self.cinder_client.volumes.create( name=cinder_vol_name, size=1) @@ -115,14 +115,10 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): # fixed in this area. # When the backup creation succeeds, it usually does within # 12 minutes. - # NOTE(lourot): it seems like we have to stick to the - # `--backup-vol` naming convention otherwise - # cinder-backup fails with a ceph/rados error about not being - # able to find the pool. See lp:1897587 - vol_backup_name = '{}-410{}-backup-vol'.format( + vol_backup_name = '{}-410-{}-backup-vol'.format( self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) vol_backup = self.cinder_client.backups.create( - cinder_vol.id, vol_backup_name) + cinder_vol.id, name=vol_backup_name) openstack_utils.resource_reaches_status( self.cinder_client.backups, vol_backup.id, From e8e16a353cc1011c7e315e798e2569544b409bb2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 12 Nov 2020 12:14:42 +0000 Subject: [PATCH 036/180] Add magpie tests --- zaza/openstack/charm_tests/magpie/__init__.py | 15 ++++ zaza/openstack/charm_tests/magpie/tests.py | 82 +++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 zaza/openstack/charm_tests/magpie/__init__.py create mode 100644 zaza/openstack/charm_tests/magpie/tests.py diff --git a/zaza/openstack/charm_tests/magpie/__init__.py b/zaza/openstack/charm_tests/magpie/__init__.py new file mode 100644 index 0000000..7fd0805 --- /dev/null +++ b/zaza/openstack/charm_tests/magpie/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing Magpie.""" diff --git a/zaza/openstack/charm_tests/magpie/tests.py b/zaza/openstack/charm_tests/magpie/tests.py new file mode 100644 index 0000000..0ac25a6 --- /dev/null +++ b/zaza/openstack/charm_tests/magpie/tests.py @@ -0,0 +1,82 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Magpie testing.""" + +import logging + +import zaza + +import zaza.model +import zaza.openstack.charm_tests.test_utils as test_utils + + +class MagpieTest(test_utils.BaseCharmTest): + """Base Magpie tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for Magpie charm operation tests.""" + super(MagpieTest, cls).setUpClass() + unit_names = sorted( + [i.entity_id + for i in zaza.model.get_units('magpie')]) + cls.test_unit_0 = unit_names[0] + cls.test_unit_1 = unit_names[1] + + def test_break_dns_single(self): + """Check DNS failure is reflected in workload status.""" + zaza.model.run_on_unit( + self.test_unit_0, + 'mv /etc/resolv.conf /etc/resolv.conf.bak') + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + zaza.model.block_until_unit_wl_message_match( + self.test_unit_0, + '.*rev dns failed.*') + logging.info('Restoring /etc/resolv.conf') + zaza.model.run_on_unit( + self.test_unit_0, + 'mv /etc/resolv.conf.bak /etc/resolv.conf') + logging.info('Updating status') + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + + def test_break_ping_single(self): + """Check ping failure is reflected in workload status.""" + icmp = "iptables {} INPUT -p icmp --icmp-type echo-request -j REJECT" + logging.info('Blocking ping on {}'.format(self.test_unit_1)) + zaza.model.run_on_unit( + self.test_unit_1, + icmp.format('--append')) + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + logging.info('Checking status on {}'.format(self.test_unit_0)) + zaza.model.block_until_unit_wl_message_match( + self.test_unit_0, + '.*icmp failed.*') + logging.info('Allowing ping on {}'.format(self.test_unit_1)) + zaza.model.run_on_unit( + self.test_unit_1, + icmp.format('--delete')) + zaza.model.run_on_unit( + self.test_unit_0, + './hooks/update-status') + logging.info('Checking status on {}'.format(self.test_unit_0)) + zaza.model.block_until_unit_wl_message_match( + self.test_unit_0, + '.*icmp ok.*') From b23d9e285b68315ba80bf85656a9661ba4d3f06c Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 18 Nov 2020 09:36:44 +0000 Subject: [PATCH 037/180] Support nova client micro versions Support micro versions when requesting a nova client. --- unit_tests/utilities/test_zaza_utilities_openstack.py | 9 +++++++++ zaza/openstack/utilities/openstack.py | 7 +++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 5ad0b9b..a047d9d 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -289,6 +289,15 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): openstack_utils.get_undercloud_keystone_session() self.get_keystone_session.assert_called_once_with(_auth, verify=None) + def test_get_nova_session_client(self): + session_mock = mock.MagicMock() + self.patch_object(openstack_utils.novaclient_client, "Client") + openstack_utils.get_nova_session_client(session_mock) + self.Client.assert_called_once_with(2, session=session_mock) + self.Client.reset_mock() + openstack_utils.get_nova_session_client(session_mock, version=2.56) + self.Client.assert_called_once_with(2.56, session=session_mock) + def test_get_urllib_opener(self): self.patch_object(openstack_utils.urllib.request, "ProxyHandler") self.patch_object(openstack_utils.urllib.request, "HTTPHandler") diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 8d25433..5d2dcca 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -253,15 +253,18 @@ def get_designate_session_client(**kwargs): **kwargs) -def get_nova_session_client(session): +def get_nova_session_client(session, version=None): """Return novaclient authenticated by keystone session. :param session: Keystone session object :type session: keystoneauth1.session.Session object + :param version: Version of client to request. + :type version: float :returns: Authenticated novaclient :rtype: novaclient.Client object """ - return novaclient_client.Client(2, session=session) + version = version or 2 + return novaclient_client.Client(version, session=session) def get_neutron_session_client(session): From dfecc58f5af22b726d7b799b9b332e8a00428afe Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 18 Nov 2020 12:13:59 +0000 Subject: [PATCH 038/180] Extend resource_reaches_status The resource_reaches_status assumes that the attribute to be checked is called 'status' but this can vary between resource types so allow the name of the attribute to specified. --- .../utilities/test_zaza_utilities_openstack.py | 9 ++++++--- zaza/openstack/utilities/openstack.py | 13 ++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 5ad0b9b..75509aa 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -369,12 +369,15 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'e01df65a') def test__resource_reaches_status_bespoke(self): + client_mock = mock.MagicMock() resource_mock = mock.MagicMock() - resource_mock.get.return_value = mock.MagicMock(status='readyish') + resource_mock.special_status = 'readyish' + client_mock.get.return_value = resource_mock openstack_utils._resource_reaches_status( - resource_mock, + client_mock, 'e01df65a', - 'readyish') + 'readyish', + resource_attribute='special_status') def test__resource_reaches_status_bespoke_fail(self): resource_mock = mock.MagicMock() diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index c309116..0b25093 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1886,7 +1886,8 @@ def download_image(image_url, target_file): def _resource_reaches_status(resource, resource_id, expected_status='available', - msg='resource'): + msg='resource', + resource_attribute='status'): """Wait for an openstack resources status to reach an expected status. Wait for an openstack resources status to reach an expected status @@ -1902,9 +1903,11 @@ def _resource_reaches_status(resource, resource_id, :type expected_status: str :param msg: text to identify purpose in logging :type msg: str + :param resource_attribute: Resource attribute to check against + :type resource_attribute: str :raises: AssertionError """ - resource_status = resource.get(resource_id).status + resource_status = getattr(resource.get(resource_id), resource_attribute) logging.info("{}: resource {} in {} state, waiting for {}".format( msg, resource_id, resource_status, expected_status)) assert resource_status == expected_status @@ -1914,6 +1917,7 @@ def resource_reaches_status(resource, resource_id, expected_status='available', msg='resource', + resource_attribute='status', wait_exponential_multiplier=1, wait_iteration_max_time=60, stop_after_attempt=8, @@ -1933,6 +1937,8 @@ def resource_reaches_status(resource, :type expected_status: str :param msg: text to identify purpose in logging :type msg: str + :param resource_attribute: Resource attribute to check against + :type resource_attribute: str :param wait_exponential_multiplier: Wait 2^x * wait_exponential_multiplier seconds between each retry :type wait_exponential_multiplier: int @@ -1954,7 +1960,8 @@ def resource_reaches_status(resource, resource, resource_id, expected_status, - msg) + msg, + resource_attribute) def _resource_removed(resource, resource_id, msg="resource"): From c5c11194a1a5032a954308889d0f76872ffe8d6f Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 17 Nov 2020 16:02:29 +0100 Subject: [PATCH 039/180] Support 'major.minor' when determining OS release of OVN packages --- zaza/openstack/utilities/openstack.py | 22 ++++++++++++++++++++-- zaza/openstack/utilities/os_versions.py | 14 +++++++++----- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index c309116..67f5f02 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -19,6 +19,7 @@ This module contains a number of functions for interacting with OpenStack. from .os_versions import ( OPENSTACK_CODENAMES, SWIFT_CODENAMES, + OVN_CODENAMES, PACKAGE_CODENAMES, OPENSTACK_RELEASES_PAIRS, ) @@ -1479,8 +1480,23 @@ def get_swift_codename(version): :returns: Codename for swift :rtype: string """ - codenames = [k for k, v in six.iteritems(SWIFT_CODENAMES) if version in v] - return codenames[0] + return _get_special_codename(version, SWIFT_CODENAMES) + + +def get_ovn_codename(version): + """Determine OpenStack codename that corresponds to OVN version. + + :param version: Version of OVN + :type version: string + :returns: Codename for OVN + :rtype: string + """ + return _get_special_codename(version, OVN_CODENAMES) + + +def _get_special_codename(version, codenames): + found = [k for k, v in six.iteritems(codenames) if version in v] + return found[0] def get_os_code_info(package, pkg_version): @@ -1517,6 +1533,8 @@ def get_os_code_info(package, pkg_version): # < Liberty co-ordinated project versions if 'swift' in package: return get_swift_codename(vers) + elif 'ovn' in package: + return get_ovn_codename(vers) else: return OPENSTACK_CODENAMES[vers] diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 75dc723..33e2fac 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -71,7 +71,6 @@ OPENSTACK_RELEASES_PAIRS = [ 'eoan_train', 'bionic_ussuri', 'focal_ussuri', 'focal_victoria', 'groovy_victoria'] -# The ugly duckling - must list releases oldest to newest SWIFT_CODENAMES = OrderedDict([ ('diablo', ['1.4.3']), @@ -113,6 +112,15 @@ SWIFT_CODENAMES = OrderedDict([ ['2.25.0']), ]) +OVN_CODENAMES = OrderedDict([ + ('train', + ['2.12']), + ('ussuri', + ['20.03']), + ('victoria', + ['20.06']), +]) + # >= Liberty version->codename mapping PACKAGE_CODENAMES = { 'nova-common': OrderedDict([ @@ -245,10 +253,6 @@ PACKAGE_CODENAMES = { ('10', 'ussuri'), ('11', 'victoria'), ]), - 'ovn-common': OrderedDict([ - ('2', 'train'), - ('20', 'ussuri'), - ]), 'ceph-common': OrderedDict([ ('10', 'mitaka'), # jewel ('12', 'queens'), # luminous From 817473675d8eb06673b5bb64fb7699cad9cc51c0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 18 Nov 2020 14:12:13 +0000 Subject: [PATCH 040/180] Numbers are imutable so safe to have as default argument --- zaza/openstack/utilities/openstack.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 5d2dcca..112c572 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -253,7 +253,7 @@ def get_designate_session_client(**kwargs): **kwargs) -def get_nova_session_client(session, version=None): +def get_nova_session_client(session, version=2): """Return novaclient authenticated by keystone session. :param session: Keystone session object @@ -263,7 +263,6 @@ def get_nova_session_client(session, version=None): :returns: Authenticated novaclient :rtype: novaclient.Client object """ - version = version or 2 return novaclient_client.Client(version, session=session) From b07afac21ac1e95a62fb656b19928c7d61ad4ede Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 18 Nov 2020 14:33:37 +0000 Subject: [PATCH 041/180] Add placement-common versions for train->ussuri upgrade --- zaza/openstack/utilities/openstack.py | 1 - zaza/openstack/utilities/os_versions.py | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index c309116..ae4d8c4 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1493,7 +1493,6 @@ def get_os_code_info(package, pkg_version): :returns: Codename for package :rtype: string """ - # {'code_num': entry, 'code_name': OPENSTACK_CODENAMES[entry]} # Remove epoch if it exists if ':' in pkg_version: pkg_version = pkg_version.split(':')[1:][0] diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 75dc723..2c806d5 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -256,4 +256,9 @@ PACKAGE_CODENAMES = { ('14', 'train'), # nautilus ('15', 'ussuri'), # octopus ]), + 'placement-common': OrderedDict([ + ('2', 'train'), + ('3', 'ussuri'), + ('4', 'victoria'), + ]), } From 7404b0cb9b1a4e757842e836b35d2d34bfd4403a Mon Sep 17 00:00:00 2001 From: Rodrigo Barbieri Date: Thu, 12 Nov 2020 14:42:50 -0300 Subject: [PATCH 042/180] Update Keystone LDAP tests - Reversed ldap-config-flags vs charm configs precedence for https://review.opendev.org/748364 - Added new functional test to validate the precedence - Added zaza helper function to assist handling charm configs that do not accept "" (like booleans and ints), and reverting to unset values through config_change() - Fixed zaza reset helper function that apparently never worked ({}.keys() returns dict_keys which is not a sequence) --- unit_tests/charm_tests/test_utils.py | 62 +++++++++- zaza/openstack/charm_tests/keystone/tests.py | 114 +++++++++++++++---- zaza/openstack/charm_tests/test_utils.py | 42 ++++++- 3 files changed, 194 insertions(+), 24 deletions(-) diff --git a/unit_tests/charm_tests/test_utils.py b/unit_tests/charm_tests/test_utils.py index ffb7f83..c1e8166 100644 --- a/unit_tests/charm_tests/test_utils.py +++ b/unit_tests/charm_tests/test_utils.py @@ -97,7 +97,7 @@ class TestBaseCharmTest(ut_utils.BaseTestCase): self.set_application_config.reset_mock() self.assertFalse(self.set_application_config.called) self.reset_application_config.assert_called_once_with( - 'anApp', alterna_config.keys(), model_name='aModel') + 'anApp', list(alterna_config.keys()), model_name='aModel') self.wait_for_application_states.assert_has_calls([ mock.call(model_name='aModel', states={}), mock.call(model_name='aModel', states={}), @@ -125,6 +125,66 @@ class TestBaseCharmTest(ut_utils.BaseTestCase): self.assertFalse(self.wait_for_agent_status.called) self.assertFalse(self.wait_for_application_states.called) + def test_separate_non_string_config(self): + intended_cfg_keys = ['foo2', 'foo3', 'foo4', 'foo5'] + current_config_mock = { + 'foo2': None, + 'foo3': 'old_bar3', + 'foo4': None, + 'foo5': 'old_bar5', + } + self.patch_target('config_current') + self.config_current.return_value = current_config_mock + non_string_type_keys = ['foo2', 'foo3', 'foo4'] + expected_result_filtered = { + 'foo3': 'old_bar3', + 'foo5': 'old_bar5', + } + expected_result_special = { + 'foo2': None, + 'foo4': None, + } + current, non_string = ( + self.target.config_current_separate_non_string_type_keys( + non_string_type_keys, intended_cfg_keys, 'application_name') + ) + + self.assertEqual(expected_result_filtered, current) + self.assertEqual(expected_result_special, non_string) + + self.config_current.assert_called_once_with( + 'application_name', intended_cfg_keys) + + def test_separate_special_config_None_params(self): + current_config_mock = { + 'foo1': 'old_bar1', + 'foo2': None, + 'foo3': 'old_bar3', + 'foo4': None, + 'foo5': 'old_bar5', + } + self.patch_target('config_current') + self.config_current.return_value = current_config_mock + non_string_type_keys = ['foo2', 'foo3', 'foo4'] + expected_result_filtered = { + 'foo1': 'old_bar1', + 'foo3': 'old_bar3', + 'foo5': 'old_bar5', + } + expected_result_special = { + 'foo2': None, + 'foo4': None, + } + current, non_string = ( + self.target.config_current_separate_non_string_type_keys( + non_string_type_keys) + ) + + self.assertEqual(expected_result_filtered, current) + self.assertEqual(expected_result_special, non_string) + + self.config_current.assert_called_once_with(None, None) + class TestOpenStackBaseTest(ut_utils.BaseTestCase): diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index 9abe78c..fb0c2f3 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -380,7 +380,12 @@ class SecurityTests(BaseKeystoneTest): class LdapTests(BaseKeystoneTest): - """Keystone ldap tests tests.""" + """Keystone ldap tests.""" + + non_string_type_keys = ('ldap-user-enabled-mask', + 'ldap-user-enabled-invert', + 'ldap-group-members-are-ids', + 'ldap-use-pool') @classmethod def setUpClass(cls): @@ -434,31 +439,44 @@ class LdapTests(BaseKeystoneTest): def test_100_keystone_ldap_users(self): """Validate basic functionality of keystone API with ldap.""" application_name = 'keystone-ldap' - config = self._get_ldap_config() + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) with self.config_change( - self.config_current(application_name, config.keys()), - config, - application_name=application_name): - logging.info( - 'Waiting for users to become available in keystone...' - ) - test_config = lifecycle_utils.get_charm_config(fatal=False) - zaza.model.wait_for_application_states( - states=test_config.get("target_deploy_status", {}) - ) + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Waiting for users to become available in keystone...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) - with self.v3_keystone_preferred(): - # NOTE(jamespage): Test fixture should have johndoe and janedoe - # accounts - johndoe = self._find_keystone_v3_user('john doe', 'userdomain') - self.assertIsNotNone(johndoe, "user 'john doe' was unknown") - janedoe = self._find_keystone_v3_user('jane doe', 'userdomain') - self.assertIsNotNone(janedoe, "user 'jane doe' was unknown") + with self.v3_keystone_preferred(): + # NOTE(jamespage): Test fixture should have + # johndoe and janedoe accounts + johndoe = self._find_keystone_v3_user( + 'john doe', 'userdomain') + self.assertIsNotNone( + johndoe, "user 'john doe' was unknown") + janedoe = self._find_keystone_v3_user( + 'jane doe', 'userdomain') + self.assertIsNotNone( + janedoe, "user 'jane doe' was unknown") class LdapExplicitCharmConfigTests(LdapTests): - """Keystone ldap tests tests.""" + """Keystone ldap tests.""" def _get_ldap_config(self): """Generate ldap config for current model. @@ -484,9 +502,61 @@ class LdapExplicitCharmConfigTests(LdapTests): 'ldap-user-enabled-mask': 0, 'ldap-user-enabled-default': 'True', 'ldap-group-tree-dn': 'ou=groups', - 'ldap-group-objectclass': 'groupOfNames', + 'ldap-group-objectclass': '', 'ldap-group-id-attribute': 'cn', 'ldap-group-member-attribute': 'memberUid', 'ldap-group-members-are-ids': True, - 'ldap-config-flags': '{group_objectclass: "posixGroup"}', + 'ldap-config-flags': '{group_objectclass: "posixGroup",' + ' use_pool: True,' + ' group_tree_dn: "group_tree_dn_foobar"}', } + + def test_200_config_flags_precedence(self): + """Validates precedence when the same config options are used.""" + application_name = 'keystone-ldap' + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) + + with self.config_change( + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Performing LDAP settings validation in keystone.conf...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) + units = zaza.model.get_units("keystone-ldap", + model_name=self.model_name) + result = zaza.model.run_on_unit( + units[0].name, + "cat /etc/keystone/domains/keystone.userdomain.conf") + # not present in charm config, but present in config flags + self.assertIn("use_pool = True", result['stdout'], + "use_pool value is expected to be present and " + "set to True in the config file") + # ldap-config-flags overriding empty charm config value + self.assertIn("group_objectclass = posixGroup", + result['stdout'], + "group_objectclass is expected to be present and" + " set to posixGroup in the config file") + # overridden by charm config, not written to file + self.assertNotIn( + "group_tree_dn_foobar", + result['stdout'], + "user_tree_dn ldap-config-flags value needs to be " + "overridden by ldap-user-tree-dn in config file") + # complementing the above, value used is from charm setting + self.assertIn("group_tree_dn = ou=groups", result['stdout'], + "user_tree_dn value is expected to be present " + "and set to dc=test,dc=com in the config file") diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 493a868..14ee0f3 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -154,6 +154,46 @@ class BaseCharmTest(unittest.TestCase): model_name=cls.model_name) logging.debug('Leader unit is {}'.format(cls.lead_unit)) + def config_current_separate_non_string_type_keys( + self, non_string_type_keys, config_keys=None, + application_name=None): + """Obtain current config and the non-string type config separately. + + If the charm config option is not string, it will not accept being + reverted back in "config_change()" method if the current value is None. + Therefore, obtain the current config and separate those out, so they + can be used for a separate invocation of "config_change()" with + reset_to_charm_default set to True. + + :param config_keys: iterable of strs to index into the current config. + If None, return all keys from the config + :type config_keys: Optional[Iterable[str]] + :param non_string_type_keys: list of non-string type keys to be + separated out only if their current value + is None + :type non_string_type_keys: list + :param application_name: String application name for use when called + by a charm under test other than the object's + application. + :type application_name: Optional[str] + :return: Dictionary of current charm configs without the + non-string type keys provided, and dictionary of the + non-string keys found in the supplied config_keys list. + :rtype: Dict[str, Any], Dict[str, None] + """ + current_config = self.config_current(application_name, config_keys) + non_string_type_config = {} + if config_keys is None: + config_keys = list(current_config.keys()) + for key in config_keys: + # We only care if the current value is None, otherwise it will + # not face issues being reverted by "config_change()" + if key in non_string_type_keys and current_config[key] is None: + non_string_type_config[key] = None + current_config.pop(key) + + return current_config, non_string_type_config + def config_current(self, application_name=None, keys=None): """Get Current Config of an application normalized into key-values. @@ -275,7 +315,7 @@ class BaseCharmTest(unittest.TestCase): 'charm default: "{}"' .format(alternate_config.keys())) model.reset_application_config(application_name, - alternate_config.keys(), + list(alternate_config.keys()), model_name=self.model_name) elif default_config == alternate_config: logging.debug('default_config == alternate_config, not attempting ' From 90aca8be5eead4e2af4b24a56bdfdfb2668776cc Mon Sep 17 00:00:00 2001 From: Arif Ali Date: Wed, 7 Oct 2020 17:39:35 +0100 Subject: [PATCH 043/180] Add ldap group/membership tests Adds a test to check for groups that are coming from LDAP. Adds a test to ensure that openstack is able to check the membership of a user in the group. Signed-off-by: Arif Ali --- zaza/openstack/charm_tests/keystone/tests.py | 130 ++++++++++++++++++- 1 file changed, 125 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index fb0c2f3..b2830ef 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -407,13 +407,22 @@ class LdapTests(BaseKeystoneTest): 'ldap-password': 'crapper', 'ldap-suffix': 'dc=test,dc=com', 'domain-name': 'userdomain', + 'ldap-config-flags': + { + 'group_tree_dn': 'ou=groups,dc=test,dc=com', + 'group_objectclass': 'posixGroup', + 'group_name_attribute': 'cn', + 'group_member_attribute': 'memberUid', + 'group_members_are_ids': 'true', + } } - def _find_keystone_v3_user(self, username, domain): + def _find_keystone_v3_user(self, username, domain, group=None): """Find a user within a specified keystone v3 domain. :param str username: Username to search for in keystone :param str domain: username selected from which domain + :param str group: group to search for in keystone for group membership :return: return username if found :rtype: Optional[str] """ @@ -423,9 +432,15 @@ class LdapTests(BaseKeystoneTest): openstack_utils.get_overcloud_auth(address=ip)) client = openstack_utils.get_keystone_session_client(session) - domain_users = client.users.list( - domain=client.domains.find(name=domain).id - ) + if group is None: + domain_users = client.users.list( + domain=client.domains.find(name=domain).id, + ) + else: + domain_users = client.users.list( + domain=client.domains.find(name=domain).id, + group=self._find_keystone_v3_group(group, domain).id, + ) usernames = [u.name.lower() for u in domain_users] if username.lower() in usernames: @@ -436,6 +451,33 @@ class LdapTests(BaseKeystoneTest): ) return None + def _find_keystone_v3_group(self, group, domain): + """Find a group within a specified keystone v3 domain. + + :param str group: Group to search for in keystone + :param str domain: group selected from which domain + :return: return group if found + :rtype: Optional[str] + """ + for ip in self.keystone_ips: + logging.info('Keystone IP {}'.format(ip)) + session = openstack_utils.get_keystone_session( + openstack_utils.get_overcloud_auth(address=ip)) + client = openstack_utils.get_keystone_session_client(session) + + domain_groups = client.groups.list( + domain=client.domains.find(name=domain).id + ) + + for searched_group in domain_groups: + if searched_group.name.lower() == group.lower(): + return searched_group + + logging.debug( + "Group {} was not found. Returning None.".format(group) + ) + return None + def test_100_keystone_ldap_users(self): """Validate basic functionality of keystone API with ldap.""" application_name = 'keystone-ldap' @@ -474,6 +516,83 @@ class LdapTests(BaseKeystoneTest): self.assertIsNotNone( janedoe, "user 'jane doe' was unknown") + def test_101_keystone_ldap_groups(self): + """Validate basic functionality of keystone API with ldap.""" + application_name = 'keystone-ldap' + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) + + with self.config_change( + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Waiting for groups to become available in keystone...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) + + with self.v3_keystone_preferred(): + # NOTE(arif-ali): Test fixture should have openstack and + # admin groups + openstack_group = self._find_keystone_v3_group( + 'openstack', 'userdomain') + self.assertIsNotNone( + openstack_group.name, "group 'openstack' was unknown") + admin_group = self._find_keystone_v3_group( + 'admin', 'userdomain') + self.assertIsNotNone( + admin_group.name, "group 'admin' was unknown") + + def test_102_keystone_ldap_group_membership(self): + """Validate basic functionality of keystone API with ldap.""" + application_name = 'keystone-ldap' + intended_cfg = self._get_ldap_config() + current_cfg, non_string_cfg = ( + self.config_current_separate_non_string_type_keys( + self.non_string_type_keys, intended_cfg, application_name) + ) + + with self.config_change( + {}, + non_string_cfg, + application_name=application_name, + reset_to_charm_default=True): + with self.config_change( + current_cfg, + intended_cfg, + application_name=application_name): + logging.info( + 'Waiting for groups to become available in keystone...' + ) + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {}) + ) + + with self.v3_keystone_preferred(): + # NOTE(arif-ali): Test fixture should have openstack and + # admin groups + openstack_group = self._find_keystone_v3_user( + 'john doe', 'userdomain', group='openstack') + self.assertIsNotNone( + openstack_group, + "john doe was not in group 'openstack'") + admin_group = self._find_keystone_v3_user( + 'john doe', 'userdomain', group='admin') + self.assertIsNotNone( + admin_group, "'john doe' was not in group 'admin'") + class LdapExplicitCharmConfigTests(LdapTests): """Keystone ldap tests.""" @@ -501,9 +620,10 @@ class LdapExplicitCharmConfigTests(LdapTests): 'ldap-user-enabled-invert': False, 'ldap-user-enabled-mask': 0, 'ldap-user-enabled-default': 'True', - 'ldap-group-tree-dn': 'ou=groups', + 'ldap-group-tree-dn': 'ou=groups,dc=test,dc=com', 'ldap-group-objectclass': '', 'ldap-group-id-attribute': 'cn', + 'ldap-group-name-attribute': 'cn', 'ldap-group-member-attribute': 'memberUid', 'ldap-group-members-are-ids': True, 'ldap-config-flags': '{group_objectclass: "posixGroup",' From d6f719e3313659c3a290e239412887a747f9598c Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Thu, 26 Nov 2020 14:42:12 +0100 Subject: [PATCH 044/180] Skip 'service' action tests on systems without systemd --- zaza/openstack/charm_tests/ceph/osd/tests.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 03e212f..63f9c4e 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -136,8 +136,14 @@ class ServiceTest(unittest.TestCase): def setUp(self): """Run test setup.""" - # Note: This counter reset is needed because ceph-osd service is - # limited to 3 restarts per 30 mins which is insufficient + # Skip 'service' action tests on systems without systemd + result = zaza_model.run_on_unit(self.TESTED_UNIT, 'which systemctl') + if not result['Stdout']: + raise unittest.SkipTest("'service' action is not supported on " + "systems without 'systemd'. Skipping " + "tests.") + # Note(mkalcok): This counter reset is needed because ceph-osd service + # is limited to 3 restarts per 30 mins which is insufficient # when running functional tests for 'service' action. This # limitation is defined in /lib/systemd/system/ceph-osd@.service # in section [Service] with options 'StartLimitInterval' and From a8ca4720a3d9199949a24e61db3129b21b3ea269 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 27 Nov 2020 09:53:12 +0100 Subject: [PATCH 045/180] Fix BlueStoreCompressionCharmOperation on Victoria (#468) Before this fix, the test tried to determine the OpenStack release based on the ceph-mon charm. Unfortunately Ceph has the same version on Ussuri and Victoria. As a consequence the test would wrongly conclude that it's testing against "groovy_ussuri", which isn't a valid Ubuntu/OpenStack pair. With this fix, the test now determines the OpenStack release based on the keystone charm, with which we are able to tell Ussuri and Victoria apart. This test class is being run against the following charm functional tests at the moment, which all have a keystone charm in their test bundles: nova-compute, cinder-ceph, glance, ceph-fs, ceph-radosgw and gnocchi. --- zaza/openstack/charm_tests/ceph/tests.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 19ce1ec..349d514 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -881,9 +881,7 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest): def setUpClass(cls): """Perform class one time initialization.""" super(BlueStoreCompressionCharmOperation, cls).setUpClass() - cls.current_release = zaza_openstack.get_os_release( - zaza_openstack.get_current_os_release_pair( - application='ceph-mon')) + cls.current_release = zaza_openstack.get_os_release() cls.bionic_rocky = zaza_openstack.get_os_release('bionic_rocky') def setUp(self): From 69b61c63f4ca647e597c17a98baa33d1fe9ee0a7 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 27 Nov 2020 12:04:43 +0100 Subject: [PATCH 046/180] Add HaclusterScaleBackAndForthTest --- zaza/openstack/charm_tests/hacluster/tests.py | 62 ++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 29751e1..99272ed 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -77,7 +77,67 @@ class HaclusterTest(HaclusterBaseTest): class HaclusterScalebackTest(HaclusterBaseTest): - """hacluster scaleback tests.""" + """hacluster scaleback tests. + + Use for testing older releases where lp:1400481 wasn't fixed yet. + Superseded by HaclusterScaleBackAndForthTest. + """ + + @classmethod + def setUpClass(cls): + """Run class setup for running hacluster scaleback tests.""" + super(HaclusterScalebackTest, cls).setUpClass() + test_config = cls.test_config['tests_options']['hacluster'] + cls._principle_app_name = test_config['principle-app-name'] + cls._hacluster_charm_name = test_config['hacluster-charm-name'] + + def test_930_scaleback(self): + """Remove a unit and add a new one.""" + principle_units = sorted(zaza.model.get_status().applications[ + self._principle_app_name]['units'].keys()) + self.assertEqual(len(principle_units), 3) + doomed_principle_unit = principle_units[0] + other_principle_unit = principle_units[1] + doomed_hacluster_unit = juju_utils.get_subordinate_units( + [doomed_principle_unit], charm_name=self._hacluster_charm_name)[0] + other_hacluster_unit = juju_utils.get_subordinate_units( + [other_principle_unit], charm_name=self._hacluster_charm_name)[0] + + logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) + zaza.model.run_action( + doomed_hacluster_unit, + 'pause', + raise_on_failure=True) + logging.info('OK') + + logging.info('Removing {}'.format(doomed_principle_unit)) + zaza.model.destroy_unit( + self._principle_app_name, + doomed_principle_unit, + wait_disappear=True) + logging.info('OK') + + logging.info('Waiting for model to settle') + zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') + zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + zaza.model.block_until_all_units_idle() + logging.info('OK') + + logging.info('Adding an hacluster unit') + zaza.model.add_unit(self._principle_app_name, wait_appear=True) + logging.info('OK') + + logging.info('Waiting for model to settle') + zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active') + # NOTE(lourot): the principle application remains blocked after scaling + # back up until lp:1400481 is solved. + zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + zaza.model.block_until_all_units_idle() + logging.debug('OK') + + +class HaclusterScaleBackAndForthTest(HaclusterBaseTest): + """hacluster tests scaling back and forth.""" @classmethod def setUpClass(cls): From a3248185f147d04774c3e8f2e32c84d386be385c Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 27 Nov 2020 13:52:44 +0100 Subject: [PATCH 047/180] Fix copy-paste mistake --- zaza/openstack/charm_tests/hacluster/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 99272ed..517cde2 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -141,8 +141,8 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): @classmethod def setUpClass(cls): - """Run class setup for running hacluster scaleback tests.""" - super(HaclusterScalebackTest, cls).setUpClass() + """Run class setup for running hacluster tests.""" + super(HaclusterScaleBackAndForthTest, cls).setUpClass() test_config = cls.test_config['tests_options']['hacluster'] cls._principle_app_name = test_config['principle-app-name'] cls._hacluster_charm_name = test_config['hacluster-charm-name'] From 7f45d461e76159c929ce333fc7a9badd8f11d9d7 Mon Sep 17 00:00:00 2001 From: coreycb Date: Mon, 30 Nov 2020 10:42:35 -0500 Subject: [PATCH 048/180] Log stderr when manila-ganesha verify() fails (#461) --- zaza/openstack/charm_tests/manila_ganesha/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 27009bf..02ea825 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -16,6 +16,8 @@ """Encapsulate Manila Ganesha testing.""" +import logging + from tenacity import Retrying, stop_after_attempt, wait_exponential from manilaclient import client as manilaclient @@ -87,6 +89,8 @@ packages: # Write a file on instance_1 def verify_setup(stdin, stdout, stderr): status = stdout.channel.recv_exit_status() + if status: + logging.info("{}".format(stderr.readlines()[0].strip())) self.assertEqual(status, 0) mount_path = share.export_locations[0] @@ -119,6 +123,8 @@ packages: def verify(stdin, stdout, stderr): status = stdout.channel.recv_exit_status() + if status: + logging.info("{}".format(stderr.readlines()[0].strip())) self.assertEqual(status, 0) out = "" for line in iter(stdout.readline, ""): From 7ced54b382adaa2181c1134e17126ac1d485028b Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Wed, 2 Dec 2020 10:22:00 +0000 Subject: [PATCH 049/180] Concurrent series upgrade updates (#466) * Updates to concurrent series upgrade Updates to make it run more in parallel and spend less time waiting on the whole model when updating machines. * Make the concurrent series upgrade tests work This is a number of changes to get the concurrent (here called 'parallel' historically) series upgrade tests to work. A number of changes were required which included limiting the number of concurrent async co-routines (futures) that could be run as with large models it hits the limits of the Py3 runtime. * Fix the tests and change pause order in maybe_pause_things Due to an additional model helper call, an additional model AsyncMock is required. Also the pause order had changed, and this is restored to ensure the original design is retained (for pause order). Clean up some commented out code and sort out a few PEP8 errors. * Update comment to reflect code (3 -> 4) * Fix tests that fail on bionic but pass on focal Essentially, asyncio.gather has different behaviour on bionic that focal. Although this doesn't affect testing, it does affect the unit tests. These changes are simply to normalise the behaviour of unit tests on focal and bionic. --- ..._zaza_utilities_parallel_series_upgrade.py | 26 +++- .../charm_tests/charm_upgrade/tests.py | 1 + zaza/openstack/charm_tests/mysql/utils.py | 18 ++- .../series_upgrade/parallel_tests.py | 41 ++++-- zaza/openstack/charm_tests/test_utils.py | 26 +++- zaza/openstack/charm_tests/vault/setup.py | 37 +++++ .../utilities/parallel_series_upgrade.py | 130 ++++++++++++------ zaza/openstack/utilities/series_upgrade.py | 7 +- 8 files changed, 220 insertions(+), 66 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 457ce1b..0231dbf 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -182,6 +182,8 @@ class TestParallelSeriesUpgrade(AioTestCase): self.model.async_wait_for_unit_idle = mock.AsyncMock() self.async_run_on_machine = mock.AsyncMock() self.model.async_run_on_machine = self.async_run_on_machine + self.model.async_block_until_units_on_machine_are_idle = \ + mock.AsyncMock() @mock.patch.object(upgrade_utils.cl_utils, 'get_class') async def test_run_post_application_upgrade_functions( @@ -492,7 +494,13 @@ class TestParallelSeriesUpgrade(AioTestCase): mock_remove_confdef_file.assert_called_once_with('1') mock_add_confdef_file.assert_called_once_with('1') - async def test_maybe_pause_things_primary(self): + @mock.patch("asyncio.gather") + async def test_maybe_pause_things_primary(self, mock_gather): + async def _gather(*args): + for f in args: + await f + + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], @@ -503,7 +511,13 @@ class TestParallelSeriesUpgrade(AioTestCase): mock.call('app/2', "pause", action_params={}), ]) - async def test_maybe_pause_things_subordinates(self): + @mock.patch("asyncio.gather") + async def test_maybe_pause_things_subordinates(self, mock_gather): + async def _gather(*args): + for f in args: + await f + + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], @@ -514,7 +528,13 @@ class TestParallelSeriesUpgrade(AioTestCase): mock.call('app-hacluster/2', "pause", action_params={}), ]) - async def test_maybe_pause_things_all(self): + @mock.patch("asyncio.gather") + async def test_maybe_pause_things_all(self, mock_gather): + async def _gather(*args): + for f in args: + await f + + mock_gather.side_effect = _gather await upgrade_utils.maybe_pause_things( FAKE_STATUS, ['app/1', 'app/2'], diff --git a/zaza/openstack/charm_tests/charm_upgrade/tests.py b/zaza/openstack/charm_tests/charm_upgrade/tests.py index f5f301a..f55caf0 100644 --- a/zaza/openstack/charm_tests/charm_upgrade/tests.py +++ b/zaza/openstack/charm_tests/charm_upgrade/tests.py @@ -35,6 +35,7 @@ class FullCloudCharmUpgradeTest(unittest.TestCase): """Run setup for Charm Upgrades.""" cli_utils.setup_logging() cls.lts = LTSGuestCreateTest() + cls.lts.setUpClass() cls.target_charm_namespace = '~openstack-charmers-next' def get_upgrade_url(self, charm_url): diff --git a/zaza/openstack/charm_tests/mysql/utils.py b/zaza/openstack/charm_tests/mysql/utils.py index 1fe5114..75da5d9 100644 --- a/zaza/openstack/charm_tests/mysql/utils.py +++ b/zaza/openstack/charm_tests/mysql/utils.py @@ -19,8 +19,16 @@ import zaza.model as model async def complete_cluster_series_upgrade(): """Run the complete-cluster-series-upgrade action on the lead unit.""" - # TODO: Make this work across either mysql or percona-cluster names - await model.async_run_action_on_leader( - 'mysql', - 'complete-cluster-series-upgrade', - action_params={}) + # Note that some models use mysql as the application name, and other's use + # percona-cluster. Try mysql first, and if it doesn't exist, then try + # percona-cluster instead. + try: + await model.async_run_action_on_leader( + 'mysql', + 'complete-cluster-series-upgrade', + action_params={}) + except KeyError: + await model.async_run_action_on_leader( + 'percona-cluster', + 'complete-cluster-series-upgrade', + action_params={}) diff --git a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py index a49eae6..e524fa2 100644 --- a/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py +++ b/zaza/openstack/charm_tests/series_upgrade/parallel_tests.py @@ -21,6 +21,7 @@ import logging import os import sys import unittest +import juju from zaza import model from zaza.openstack.utilities import ( @@ -55,6 +56,12 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): @classmethod def setUpClass(cls): """Run setup for Series Upgrades.""" + # NOTE(ajkavanagh): Set the jujulib Connection frame size to 4GB to + # cope with all the outputs from series upgrade; long term, don't send + # that output back, which will require that the upgrade function in the + # charm doesn't capture the output of the upgrade in the action, but + # instead puts it somewhere that can by "juju scp"ed. + juju.client.connection.Connection.MAX_FRAME_SIZE = 2**32 cli_utils.setup_logging() cls.from_series = None cls.to_series = None @@ -89,19 +96,25 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): upgrade_function = \ parallel_series_upgrade.parallel_series_upgrade + # allow up to 4 parallel upgrades at a time. This is to limit the + # amount of data/calls that asyncio is handling as it's gets + # unstable if all the applications are done at the same time. + sem = asyncio.Semaphore(4) for charm_name in apps: charm = applications[charm_name]['charm'] name = upgrade_utils.extract_charm_name_from_url(charm) upgrade_config = parallel_series_upgrade.app_config(name) upgrade_functions.append( - upgrade_function( - charm_name, - **upgrade_config, - from_series=from_series, - to_series=to_series, - completed_machines=completed_machines, - workaround_script=workaround_script, - files=files)) + wrap_coroutine_with_sem( + sem, + upgrade_function( + charm_name, + **upgrade_config, + from_series=from_series, + to_series=to_series, + completed_machines=completed_machines, + workaround_script=workaround_script, + files=files))) asyncio.get_event_loop().run_until_complete( asyncio.gather(*upgrade_functions)) model.block_until_all_units_idle() @@ -109,6 +122,18 @@ class ParallelSeriesUpgradeTest(unittest.TestCase): logging.info("Done!") +async def wrap_coroutine_with_sem(sem, coroutine): + """Wrap a coroutine with a semaphore to limit concurrency. + + :param sem: The semaphore to limit concurrency + :type sem: asyncio.Semaphore + :param coroutine: the corouting to limit concurrency + :type coroutine: types.CoroutineType + """ + async with sem: + await coroutine + + class OpenStackParallelSeriesUpgrade(ParallelSeriesUpgradeTest): """OpenStack Series Upgrade. diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 14ee0f3..5a3e6c1 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -120,7 +120,23 @@ class BaseCharmTest(unittest.TestCase): @classmethod def setUpClass(cls, application_name=None, model_alias=None): - """Run setup for test class to create common resources.""" + """Run setup for test class to create common resources. + + Note: the derived class may not use the application_name; if it's set + to None then this setUpClass() method will attempt to extract the + application name from the charm_config (essentially the test.yaml) + using the key 'charm_name' in the test_config. If that isn't present, + then there will be no application_name set, and this is considered a + generic scenario of a whole model rather than a particular charm under + test. + + :param application_name: the name of the applications that the derived + class is testing. If None, then it's a generic test not connected + to any single charm. + :type application_name: Optional[str] + :param model_alias: the alias to use if needed. + :type model_alias: Optional[str] + """ cls.model_aliases = model.get_juju_model_aliases() if model_alias: cls.model_name = cls.model_aliases[model_alias] @@ -131,7 +147,13 @@ class BaseCharmTest(unittest.TestCase): if application_name: cls.application_name = application_name else: - charm_under_test_name = cls.test_config['charm_name'] + try: + charm_under_test_name = cls.test_config['charm_name'] + except KeyError: + logging.warning("No application_name and no charm config so " + "not setting the application_name. Likely a " + "scenario test.") + return deployed_app_names = model.sync_deployed(model_name=cls.model_name) if charm_under_test_name in deployed_app_names: # There is an application named like the charm under test. diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 4db90b5..adfb92d 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -26,6 +26,7 @@ import zaza.model import zaza.openstack.utilities.cert import zaza.openstack.utilities.openstack import zaza.openstack.utilities.generic +import zaza.openstack.utilities.exceptions as zaza_exceptions import zaza.utilities.juju as juju_utils @@ -73,9 +74,27 @@ def basic_setup_and_unseal(cacert=None): zaza.model.run_on_unit(unit.name, './hooks/update-status') +async def mojo_or_default_unseal_by_unit(): + """Unseal any units reported as sealed using a cacert. + + The mojo cacert is tried first, and if that doesn't exist, then the default + zaza located cacert is used. + """ + try: + await mojo_unseal_by_unit() + except zaza_exceptions.CACERTNotFound: + await unseal_by_unit() + + def mojo_unseal_by_unit(): """Unseal any units reported as sealed using mojo cacert.""" cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path() + unseal_by_unit(cacert) + + +def unseal_by_unit(cacert=None): + """Unseal any units reported as sealed using mojo cacert.""" + cacert = cacert or get_cacert_file() vault_creds = vault_utils.get_credentails() for client in vault_utils.get_clients(cacert=cacert): if client.hvac_client.is_sealed(): @@ -86,9 +105,27 @@ def mojo_unseal_by_unit(): zaza.model.run_on_unit(unit_name, './hooks/update-status') +async def async_mojo_or_default_unseal_by_unit(): + """Unseal any units reported as sealed using a cacert. + + The mojo cacert is tried first, and if that doesn't exist, then the default + zaza located cacert is used. + """ + try: + await async_mojo_unseal_by_unit() + except zaza_exceptions.CACERTNotFound: + await async_unseal_by_unit() + + async def async_mojo_unseal_by_unit(): """Unseal any units reported as sealed using mojo cacert.""" cacert = zaza.openstack.utilities.generic.get_mojo_cacert_path() + await async_unseal_by_unit(cacert) + + +async def async_unseal_by_unit(cacert=None): + """Unseal any units reported as sealed using vault cacert.""" + cacert = cacert or get_cacert_file() vault_creds = vault_utils.get_credentails() for client in vault_utils.get_clients(cacert=cacert): if client.hvac_client.is_sealed(): diff --git a/zaza/openstack/utilities/parallel_series_upgrade.py b/zaza/openstack/utilities/parallel_series_upgrade.py index aa6ab0e..610496d 100755 --- a/zaza/openstack/utilities/parallel_series_upgrade.py +++ b/zaza/openstack/utilities/parallel_series_upgrade.py @@ -58,7 +58,9 @@ def app_config(charm_name): } exceptions = { 'rabbitmq-server': { - 'origin': 'source', + # NOTE: AJK disable config-changed on rabbitmq-server due to bug: + # #1896520 + 'origin': None, 'pause_non_leader_subordinate': False, 'post_application_upgrade_functions': [ ('zaza.openstack.charm_tests.rabbitmq_server.utils.' @@ -94,7 +96,7 @@ def app_config(charm_name): 'pause_non_leader_subordinate': True, 'post_upgrade_functions': [ ('zaza.openstack.charm_tests.vault.setup.' - 'async_mojo_unseal_by_unit')] + 'async_mojo_or_default_unseal_by_unit')] }, 'mongodb': { 'origin': None, @@ -191,49 +193,64 @@ async def parallel_series_upgrade( status = (await model.async_get_status()).applications[application] logging.info( "Configuring leader / non leaders for {}".format(application)) - leader, non_leaders = get_leader_and_non_leaders(status) - for leader_name, leader_unit in leader.items(): + leaders, non_leaders = get_leader_and_non_leaders(status) + for leader_unit in leaders.values(): leader_machine = leader_unit["machine"] - leader = leader_name - machines = [ - unit["machine"] for name, unit - in non_leaders.items() - if unit['machine'] not in completed_machines] + machines = [unit["machine"] for name, unit in non_leaders.items() + if unit['machine'] not in completed_machines] await maybe_pause_things( status, non_leaders, pause_non_leader_subordinate, pause_non_leader_primary) - await series_upgrade_utils.async_set_series( - application, to_series=to_series) - app_idle = [ + # wait for the entire application set to be idle before starting upgrades + await asyncio.gather(*[ model.async_wait_for_unit_idle(unit, include_subordinates=True) - for unit in status["units"] - ] - await asyncio.gather(*app_idle) + for unit in status["units"]]) await prepare_series_upgrade(leader_machine, to_series=to_series) - prepare_group = [ - prepare_series_upgrade(machine, to_series=to_series) - for machine in machines] - await asyncio.gather(*prepare_group) + await asyncio.gather(*[ + wait_for_idle_then_prepare_series_upgrade( + machine, to_series=to_series) + for machine in machines]) if leader_machine not in completed_machines: machines.append(leader_machine) - upgrade_group = [ + await asyncio.gather(*[ series_upgrade_machine( machine, origin=origin, application=application, files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) - for machine in machines - ] - await asyncio.gather(*upgrade_group) + for machine in machines]) completed_machines.extend(machines) + await series_upgrade_utils.async_set_series( + application, to_series=to_series) await run_post_application_upgrade_functions( post_application_upgrade_functions) +async def wait_for_idle_then_prepare_series_upgrade( + machine, to_series, model_name=None): + """Wait for the units to idle the do prepare_series_upgrade. + + We need to be sure that all the units are idle prior to actually calling + prepare_series_upgrade() as otherwise the call will fail. It has to be + checked because when the leader is paused it may kick off relation hooks in + the other units in an HA group. + + :param machine: the machine that is going to be prepared + :type machine: str + :param to_series: The series to which to upgrade + :type to_series: str + :param model_name: Name of model to query. + :type model_name: str + """ + await model.async_block_until_units_on_machine_are_idle( + machine, model_name=model_name) + await prepare_series_upgrade(machine, to_series=to_series) + + async def serial_series_upgrade( application, from_series='xenial', @@ -307,8 +324,10 @@ async def serial_series_upgrade( non_leaders, pause_non_leader_subordinate, pause_non_leader_primary) + logging.info("Finishing pausing application: {}".format(application)) await series_upgrade_utils.async_set_series( application, to_series=to_series) + logging.info("Finished set series for application: {}".format(application)) if not follower_first and leader_machine not in completed_machines: await model.async_wait_for_unit_idle(leader, include_subordinates=True) await prepare_series_upgrade(leader_machine, to_series=to_series) @@ -321,6 +340,8 @@ async def serial_series_upgrade( files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) completed_machines.append(leader_machine) + logging.info("Finished upgrading of leader for application: {}" + .format(application)) # for machine in machines: for unit_name, unit in non_leaders.items(): @@ -339,6 +360,8 @@ async def serial_series_upgrade( files=files, workaround_script=workaround_script, post_upgrade_functions=post_upgrade_functions) completed_machines.append(machine) + logging.info("Finished upgrading non leaders for application: {}" + .format(application)) if follower_first and leader_machine not in completed_machines: await model.async_wait_for_unit_idle(leader, include_subordinates=True) @@ -354,6 +377,7 @@ async def serial_series_upgrade( completed_machines.append(leader_machine) await run_post_application_upgrade_functions( post_application_upgrade_functions) + logging.info("Done series upgrade for: {}".format(application)) async def series_upgrade_machine( @@ -381,17 +405,16 @@ async def series_upgrade_machine( :returns: None :rtype: None """ - logging.info( - "About to series-upgrade ({})".format(machine)) + logging.info("About to series-upgrade ({})".format(machine)) await run_pre_upgrade_functions(machine, pre_upgrade_functions) await add_confdef_file(machine) await async_dist_upgrade(machine) await async_do_release_upgrade(machine) await remove_confdef_file(machine) await reboot(machine) + await series_upgrade_utils.async_complete_series_upgrade(machine) if origin: await os_utils.async_set_origin(application, origin) - await series_upgrade_utils.async_complete_series_upgrade(machine) await run_post_upgrade_functions(post_upgrade_functions) @@ -484,8 +507,7 @@ async def maybe_pause_things( :returns: Nothing :trype: None """ - subordinate_pauses = [] - leader_pauses = [] + unit_pauses = [] for unit in units: if pause_non_leader_subordinate: if status["units"][unit].get("subordinates"): @@ -495,15 +517,19 @@ async def maybe_pause_things( logging.info("Skipping pausing {} - blacklisted" .format(subordinate)) else: - logging.info("Pausing {}".format(subordinate)) - subordinate_pauses.append(model.async_run_action( - subordinate, "pause", action_params={})) + unit_pauses.append( + _pause_helper("subordinate", subordinate)) if pause_non_leader_primary: - logging.info("Pausing {}".format(unit)) - leader_pauses.append( - model.async_run_action(unit, "pause", action_params={})) - await asyncio.gather(*leader_pauses) - await asyncio.gather(*subordinate_pauses) + unit_pauses.append(_pause_helper("leader", unit)) + if unit_pauses: + await asyncio.gather(*unit_pauses) + + +async def _pause_helper(_type, unit): + """Pause helper to ensure that the log happens nearer to the action.""" + logging.info("Pausing ({}) {}".format(_type, unit)) + await model.async_run_action(unit, "pause", action_params={}) + logging.info("Finished Pausing ({}) {}".format(_type, unit)) def get_leader_and_non_leaders(status): @@ -541,14 +567,14 @@ async def prepare_series_upgrade(machine, to_series): NOTE: This is a new feature in juju behind a feature flag and not yet in libjuju. export JUJU_DEV_FEATURE_FLAGS=upgrade-series - :param machine_num: Machine number - :type machine_num: str + :param machine: Machine number + :type machine: str :param to_series: The series to which to upgrade :type to_series: str :returns: None :rtype: None """ - logging.info("Preparing series upgrade for: {}".format(machine)) + logging.info("Preparing series upgrade for: %s", machine) await series_upgrade_utils.async_prepare_series_upgrade( machine, to_series=to_series) @@ -564,9 +590,8 @@ async def reboot(machine): try: await model.async_run_on_machine(machine, 'sudo init 6 & exit') # await run_on_machine(unit, "sudo reboot && exit") - except subprocess.CalledProcessError as e: - logging.warn("Error doing reboot: {}".format(e)) - pass + except subprocess.CalledProcessError as error: + logging.warning("Error doing reboot: %s", error) async def async_dist_upgrade(machine): @@ -577,16 +602,31 @@ async def async_dist_upgrade(machine): :returns: None :rtype: None """ - logging.info('Updating package db ' + machine) + logging.info('Updating package db %s', machine) update_cmd = 'sudo apt-get update' await model.async_run_on_machine(machine, update_cmd) - logging.info('Updating existing packages ' + machine) + logging.info('Updating existing packages %s', machine) dist_upgrade_cmd = ( """yes | sudo DEBIAN_FRONTEND=noninteractive apt-get --assume-yes """ """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") await model.async_run_on_machine(machine, dist_upgrade_cmd) + rdict = await model.async_run_on_machine( + machine, + "cat /var/run/reboot-required || true") + if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): + logging.info("dist-upgrade required reboot machine: %s", machine) + await reboot(machine) + logging.info("Waiting for machine to come back afer reboot: %s", + machine) + await model.async_block_until_file_missing_on_machine( + machine, "/var/run/reboot-required") + logging.info("Waiting for machine idleness on %s", machine) + await asyncio.sleep(5.0) + await model.async_block_until_units_on_machine_are_idle(machine) + # TODO: change this to wait on units on the machine + # await model.async_block_until_all_units_idle() async def async_do_release_upgrade(machine): @@ -597,7 +637,7 @@ async def async_do_release_upgrade(machine): :returns: None :rtype: None """ - logging.info('Upgrading ' + machine) + logging.info('Upgrading %s', machine) do_release_upgrade_cmd = ( 'yes | sudo DEBIAN_FRONTEND=noninteractive ' 'do-release-upgrade -f DistUpgradeViewNonInteractive') diff --git a/zaza/openstack/utilities/series_upgrade.py b/zaza/openstack/utilities/series_upgrade.py index 97ba153..42683f6 100644 --- a/zaza/openstack/utilities/series_upgrade.py +++ b/zaza/openstack/utilities/series_upgrade.py @@ -884,7 +884,8 @@ def dist_upgrade(unit_name): """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") model.run_on_unit(unit_name, dist_upgrade_cmd) - rdict = model.run_on_unit(unit_name, "cat /var/run/reboot-required") + rdict = model.run_on_unit(unit_name, + "cat /var/run/reboot-required || true") if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): logging.info("dist-upgrade required reboot {}".format(unit_name)) os_utils.reboot(unit_name) @@ -919,8 +920,8 @@ async def async_dist_upgrade(unit_name): """-o "Dpkg::Options::=--force-confdef" """ """-o "Dpkg::Options::=--force-confold" dist-upgrade""") await model.async_run_on_unit(unit_name, dist_upgrade_cmd) - rdict = await model.async_run_on_unit(unit_name, - "cat /var/run/reboot-required") + rdict = await model.async_run_on_unit( + unit_name, "cat /var/run/reboot-required || true") if "Stdout" in rdict and "restart" in rdict["Stdout"].lower(): logging.info("dist-upgrade required reboot {}".format(unit_name)) await os_utils.async_reboot(unit_name) From 10756f7a663d611f6dae0f69d57fe07c97ff9243 Mon Sep 17 00:00:00 2001 From: Linda Guo Date: Fri, 11 Dec 2020 10:13:41 +1100 Subject: [PATCH 050/180] Add NovaComputeActionTest test class for virsh_audit action --- zaza/openstack/charm_tests/nova/tests.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 79e0e70..b8e7552 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -155,6 +155,29 @@ class NovaCompute(test_utils.OpenStackBaseTest): self.assertFalse(int(run['Code']) == 0) +class NovaComputeActionTest(test_utils.OpenStackBaseTest): + """Run nova-compute specific tests. + + Add this test class for new nova-compute action + to avoid breaking older version + """ + + def test_virsh_audit_action(self): + """Test virsh-audit action.""" + for unit in zaza.model.get_units('nova-compute', + model_name=self.model_name): + logging.info('Running `virsh-audit` action' + ' on unit {}'.format(unit.entity_id)) + action = zaza.model.run_action( + unit.entity_id, + 'virsh-audit', + model_name=self.model_name, + action_params={}) + if "failed" in action.data["status"]: + raise Exception( + "The action failed: {}".format(action.data["message"])) + + class NovaCloudController(test_utils.OpenStackBaseTest): """Run nova-cloud-controller specific tests.""" From cfdabe273a36cf82323619ab929e38b5773ae2ed Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Fri, 11 Dec 2020 09:49:33 +0100 Subject: [PATCH 051/180] Adjust tests after rework of the 'service' action into `start` and `stop` actions --- zaza/openstack/charm_tests/ceph/osd/tests.py | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 63f9c4e..c4e6e36 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -156,8 +156,8 @@ class ServiceTest(unittest.TestCase): This ensures that the environment is ready for the next tests. """ - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'start': 'all'}, + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'disks': 'all'}, raise_on_failure=True) @property @@ -186,16 +186,16 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop=all' action on {} " "unit".format(self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT], 'service', - action_params={'stop': 'all'}) + zaza_model.run_action_on_units([self.TESTED_UNIT], 'stop', + action_params={'disks': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='stopped') logging.info("Running 'service start=all' action on {} " "unit".format(self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'start': 'all'}) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'disks': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='running') @@ -208,16 +208,16 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'stop': action_params}) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop', + action_params={'disks': action_params}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='stopped') logging.info("Running 'service start={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'start': action_params}) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'disks': action_params}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='running') @@ -236,8 +236,8 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop={} on {} " "unit".format(to_stop.id, self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'stop': to_stop.id}) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop', + action_params={'disks': to_stop.id}) wait_for_service(unit_name=self.TESTED_UNIT, services=[to_stop.name, ], @@ -269,8 +269,8 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service start={} on {} " "unit".format(to_start.id, self.TESTED_UNIT)) - zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'service', - action_params={'start': to_start.id}) + zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', + action_params={'disks': to_start.id}) wait_for_service(unit_name=self.TESTED_UNIT, services=[to_start.name, ], From 96592c6a85d90eb2594b24798d0d564a0e17c009 Mon Sep 17 00:00:00 2001 From: Martin Kalcok Date: Fri, 11 Dec 2020 16:16:15 +0100 Subject: [PATCH 052/180] Rename parameter `disks` to `osds` --- zaza/openstack/charm_tests/ceph/osd/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index c4e6e36..63fb128 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -157,7 +157,7 @@ class ServiceTest(unittest.TestCase): This ensures that the environment is ready for the next tests. """ zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', - action_params={'disks': 'all'}, + action_params={'osds': 'all'}, raise_on_failure=True) @property @@ -187,7 +187,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop=all' action on {} " "unit".format(self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT], 'stop', - action_params={'disks': 'all'}) + action_params={'osds': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='stopped') @@ -195,7 +195,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service start=all' action on {} " "unit".format(self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', - action_params={'disks': 'all'}) + action_params={'osds': 'all'}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='running') @@ -209,7 +209,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service stop={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop', - action_params={'disks': action_params}) + action_params={'osds': action_params}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='stopped') @@ -217,7 +217,7 @@ class ServiceTest(unittest.TestCase): logging.info("Running 'service start={}' action on {} " "unit".format(action_params, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', - action_params={'disks': action_params}) + action_params={'osds': action_params}) wait_for_service(unit_name=self.TESTED_UNIT, services=service_list, target_status='running') @@ -237,7 +237,7 @@ class ServiceTest(unittest.TestCase): "unit".format(to_stop.id, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'stop', - action_params={'disks': to_stop.id}) + action_params={'osds': to_stop.id}) wait_for_service(unit_name=self.TESTED_UNIT, services=[to_stop.name, ], @@ -270,7 +270,7 @@ class ServiceTest(unittest.TestCase): "unit".format(to_start.id, self.TESTED_UNIT)) zaza_model.run_action_on_units([self.TESTED_UNIT, ], 'start', - action_params={'disks': to_start.id}) + action_params={'osds': to_start.id}) wait_for_service(unit_name=self.TESTED_UNIT, services=[to_start.name, ], From a35ba0917edf4a22bec4d579926bfbd0157d1a08 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 6 Jan 2021 11:26:01 +0000 Subject: [PATCH 053/180] Allow an app to be set when calling get_os_release (#477) Ceph deployments may not contain a keystone service which causes calls to get_os_release to fail as it calls get_current_os_release_pair without sepecifying an application (keystone is the default). --- unit_tests/utilities/test_zaza_utilities_openstack.py | 8 ++++++++ zaza/openstack/utilities/openstack.py | 8 ++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 6d91d7d..48408e0 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -908,6 +908,14 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): release_comp = xenial_queens > xenial_mitaka self.assertTrue(release_comp) + # Check specifying an application + self._get_os_rel_pair.reset_mock() + self._get_os_rel_pair.return_value = 'xenial_mitaka' + expected = 4 + result = openstack_utils.get_os_release(application='myapp') + self.assertEqual(expected, result) + self._get_os_rel_pair.assert_called_once_with(application='myapp') + def test_get_keystone_api_version(self): self.patch_object(openstack_utils, "get_current_os_versions") self.patch_object(openstack_utils, "get_application_config_option") diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 06aa14c..a93e61d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1602,15 +1602,19 @@ def get_current_os_release_pair(application='keystone'): return '{}_{}'.format(series, os_version) -def get_os_release(release_pair=None): +def get_os_release(release_pair=None, application=None): """Return index of release in OPENSTACK_RELEASES_PAIRS. + :param release_pair: OpenStack release pair eg 'focal_ussuri' + :type release_pair: string + :param application: Name of application to derive release pair from. + :type application: string :returns: Index of the release :rtype: int :raises: exceptions.ReleasePairNotFound """ if release_pair is None: - release_pair = get_current_os_release_pair() + release_pair = get_current_os_release_pair(application=application) try: index = OPENSTACK_RELEASES_PAIRS.index(release_pair) except ValueError: From ed3b2737d1bc8790ad354e22e2b6136243256960 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 6 Jan 2021 11:38:41 +0000 Subject: [PATCH 054/180] Use ceph-mon to check ceph version not keystone The test class BlueStoreCompressionCharmOperation gates tests on whether the ceph release is mimic or newer but it uses the keystone application to calculate the currently deployed version. This PR switches the test class to ceck the version of ceph-mon instead which makes more sense and the keystone application may not always present in a ceph deployment. --- zaza/openstack/charm_tests/ceph/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 349d514..107eb9f 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -881,7 +881,8 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest): def setUpClass(cls): """Perform class one time initialization.""" super(BlueStoreCompressionCharmOperation, cls).setUpClass() - cls.current_release = zaza_openstack.get_os_release() + cls.current_release = zaza_openstack.get_os_release( + application='ceph-mon') cls.bionic_rocky = zaza_openstack.get_os_release('bionic_rocky') def setUp(self): From ddd9c7402809175ac60889a021bff0da5749b8bc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 6 Jan 2021 12:12:42 +0000 Subject: [PATCH 055/180] Use keystone in the first instance --- zaza/openstack/charm_tests/ceph/tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index 107eb9f..c7d44bd 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -881,8 +881,13 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest): def setUpClass(cls): """Perform class one time initialization.""" super(BlueStoreCompressionCharmOperation, cls).setUpClass() + release_application = 'keystone' + try: + zaza_model.get_application(release_application) + except KeyError: + release_application = 'ceph-mon' cls.current_release = zaza_openstack.get_os_release( - application='ceph-mon') + application=release_application) cls.bionic_rocky = zaza_openstack.get_os_release('bionic_rocky') def setUp(self): From 282b8be577b9398c1af8415d0e521ee71a1ae719 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Wed, 6 Jan 2021 13:09:41 +0000 Subject: [PATCH 056/180] Update openstack upgrade tests for focal (#476) This patch modifies the existing openstack upgrade tests so that they work with focal (by explicitly supporting mysql-innodb-cluster), and are also interruptable and resumable (at a charm level). It also makes them work with the udpated 'get_upgrade_groups()' that ultimately gets a List of Tuples rather than a dictionary. --- .../test_zaza_utilities_openstack_upgrade.py | 118 +++++++++----- .../charm_tests/openstack_upgrade/__init__.py | 15 ++ .../charm_tests/openstack_upgrade/tests.py | 148 ++++++++++++++++++ zaza/openstack/charm_tests/vault/setup.py | 3 +- zaza/openstack/utilities/openstack.py | 33 ++-- zaza/openstack/utilities/openstack_upgrade.py | 136 +++++++++++----- zaza/openstack/utilities/upgrade_utils.py | 119 +++++++++++++- 7 files changed, 483 insertions(+), 89 deletions(-) create mode 100644 zaza/openstack/charm_tests/openstack_upgrade/__init__.py create mode 100644 zaza/openstack/charm_tests/openstack_upgrade/tests.py diff --git a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py index 4d619c5..3782926 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack_upgrade.py @@ -44,25 +44,37 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.patch_object( openstack_upgrade.zaza.model, "get_application_config") + self.patch_object( + openstack_upgrade.zaza.model, + "block_until_all_units_idle") + self.patch_object( + openstack_upgrade, + "block_until_mysql_innodb_cluster_has_rw") def _get_application_config(app, model_name=None): app_config = { - 'ceph-mon': {'verbose': True, 'source': 'old-src'}, - 'neutron-openvswitch': {'verbose': True}, - 'ntp': {'verbose': True}, - 'percona-cluster': {'verbose': True, 'source': 'old-src'}, + 'ceph-mon': {'verbose': {'value': True}, + 'source': {'value': 'old-src'}}, + 'neutron-openvswitch': {'verbose': {'value': True}}, + 'ntp': {'verbose': {'value': True}}, + 'percona-cluster': {'verbose': {'value': True}, + 'source': {'value': 'old-src'}}, 'cinder': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, 'neutron-api': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, 'nova-compute': { - 'verbose': True, - 'openstack-origin': 'old-src', - 'action-managed-upgrade': False}, + 'verbose': {'value': True}, + 'openstack-origin': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': False}}, + 'mysql-innodb-cluster': { + 'verbose': {'value': True}, + 'source': {'value': 'old-src'}, + 'action-managed-upgrade': {'value': True}}, } return app_config[app] self.get_application_config.side_effect = _get_application_config @@ -74,6 +86,10 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): 'subordinate-to': 'nova-compute'}, 'ntp': { # Filter as it has no source option 'charm': 'cs:ntp'}, + 'mysql-innodb-cluster': { + 'charm': 'cs:mysql-innodb-cluster', + 'units': { + 'mysql-innodb-cluster/0': {}}}, 'nova-compute': { 'charm': 'cs:nova-compute', 'units': { @@ -115,7 +131,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): model_name=None, raise_on_failure=True) - def test_action_upgrade_group(self): + def test_action_upgrade_apps(self): self.patch_object(openstack_upgrade, "pause_units") self.patch_object(openstack_upgrade, "action_unit_upgrade") self.patch_object(openstack_upgrade, "resume_units") @@ -127,7 +143,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): 'nova-compute': [mock_nova_compute_0], 'cinder': [mock_cinder_1]} self.get_units.side_effect = lambda app, model_name: units[app] - openstack_upgrade.action_upgrade_group(['nova-compute', 'cinder']) + openstack_upgrade.action_upgrade_apps(['nova-compute', 'cinder']) pause_calls = [ mock.call(['cinder-hacluster/0'], model_name=None), mock.call(['nova-compute/0', 'cinder/1'], model_name=None)] @@ -142,6 +158,30 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): mock.call(['cinder-hacluster/0'], model_name=None)] self.resume_units.assert_has_calls(resume_calls, any_order=False) + def test_action_upgrade_apps_mysql_innodb_cluster(self): + """Verify that mysql-innodb-cluster is settled before complete.""" + self.patch_object(openstack_upgrade, "pause_units") + self.patch_object(openstack_upgrade, "action_unit_upgrade") + self.patch_object(openstack_upgrade, "resume_units") + mock_mysql_innodb_cluster_0 = mock.MagicMock() + mock_mysql_innodb_cluster_0.entity_id = 'mysql-innodb-cluster/0' + units = {'mysql-innodb-cluster': [mock_mysql_innodb_cluster_0]} + self.get_units.side_effect = lambda app, model_name: units[app] + openstack_upgrade.action_upgrade_apps(['mysql-innodb-cluster']) + pause_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.pause_units.assert_has_calls(pause_calls, any_order=False) + action_unit_upgrade_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.action_unit_upgrade.assert_has_calls( + action_unit_upgrade_calls, + any_order=False) + resume_calls = [ + mock.call(['mysql-innodb-cluster/0'], model_name=None)] + self.resume_units.assert_has_calls(resume_calls, any_order=False) + self.block_until_mysql_innodb_cluster_has_rw.assert_called_once_with( + None) + def test_set_upgrade_application_config(self): openstack_upgrade.set_upgrade_application_config( ['neutron-api', 'cinder'], @@ -177,17 +217,23 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.assertFalse( openstack_upgrade.is_action_upgradable('percona-cluster')) + def test_is_already_upgraded(self): + self.assertTrue( + openstack_upgrade.is_already_upgraded('cinder', 'old-src')) + self.assertFalse( + openstack_upgrade.is_already_upgraded('cinder', 'new-src')) + def test_run_action_upgrade(self): self.patch_object(openstack_upgrade, "set_upgrade_application_config") - self.patch_object(openstack_upgrade, "action_upgrade_group") - openstack_upgrade.run_action_upgrade( + self.patch_object(openstack_upgrade, "action_upgrade_apps") + openstack_upgrade.run_action_upgrades( ['cinder', 'neutron-api'], 'new-src') self.set_upgrade_application_config.assert_called_once_with( ['cinder', 'neutron-api'], 'new-src', model_name=None) - self.action_upgrade_group.assert_called_once_with( + self.action_upgrade_apps.assert_called_once_with( ['cinder', 'neutron-api'], model_name=None) @@ -196,7 +242,7 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.patch_object( openstack_upgrade.zaza.model, 'block_until_all_units_idle') - openstack_upgrade.run_all_in_one_upgrade( + openstack_upgrade.run_all_in_one_upgrades( ['percona-cluster'], 'new-src') self.set_upgrade_application_config.assert_called_once_with( @@ -207,34 +253,36 @@ class TestOpenStackUpgradeUtils(ut_utils.BaseTestCase): self.block_until_all_units_idle.assert_called_once_with() def test_run_upgrade(self): - self.patch_object(openstack_upgrade, "run_all_in_one_upgrade") - self.patch_object(openstack_upgrade, "run_action_upgrade") - openstack_upgrade.run_upgrade( + self.patch_object(openstack_upgrade, "run_all_in_one_upgrades") + self.patch_object(openstack_upgrade, "run_action_upgrades") + openstack_upgrade.run_upgrade_on_apps( ['cinder', 'neutron-api', 'ceph-mon'], 'new-src') - self.run_all_in_one_upgrade.assert_called_once_with( + self.run_all_in_one_upgrades.assert_called_once_with( ['ceph-mon'], 'new-src', model_name=None) - self.run_action_upgrade.assert_called_once_with( + self.run_action_upgrades.assert_called_once_with( ['cinder', 'neutron-api'], 'new-src', model_name=None) def test_run_upgrade_tests(self): - self.patch_object(openstack_upgrade, "run_upgrade") + self.patch_object(openstack_upgrade, "run_upgrade_on_apps") self.patch_object(openstack_upgrade, "get_upgrade_groups") - self.get_upgrade_groups.return_value = { - 'Compute': ['nova-compute'], - 'Control Plane': ['cinder', 'neutron-api'], - 'Core Identity': ['keystone'], - 'Storage': ['ceph-mon'], - 'sweep_up': ['designate']} + self.get_upgrade_groups.return_value = [ + ('Compute', ['nova-compute']), + ('Control Plane', ['cinder', 'neutron-api']), + ('Core Identity', ['keystone']), + ('Storage', ['ceph-mon']), + ('sweep_up', ['designate'])] openstack_upgrade.run_upgrade_tests('new-src', model_name=None) run_upgrade_calls = [ + mock.call(['nova-compute'], 'new-src', model_name=None), + mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None), mock.call(['keystone'], 'new-src', model_name=None), mock.call(['ceph-mon'], 'new-src', model_name=None), - mock.call(['cinder', 'neutron-api'], 'new-src', model_name=None), - mock.call(['nova-compute'], 'new-src', model_name=None), - mock.call(['designate'], 'new-src', model_name=None)] - self.run_upgrade.assert_has_calls(run_upgrade_calls, any_order=False) + mock.call(['designate'], 'new-src', model_name=None), + ] + self.run_upgrade_on_apps.assert_has_calls( + run_upgrade_calls, any_order=False) diff --git a/zaza/openstack/charm_tests/openstack_upgrade/__init__.py b/zaza/openstack/charm_tests/openstack_upgrade/__init__.py new file mode 100644 index 0000000..9a1ca53 --- /dev/null +++ b/zaza/openstack/charm_tests/openstack_upgrade/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Code for testing openstack upgrades.""" diff --git a/zaza/openstack/charm_tests/openstack_upgrade/tests.py b/zaza/openstack/charm_tests/openstack_upgrade/tests.py new file mode 100644 index 0000000..82c7fbb --- /dev/null +++ b/zaza/openstack/charm_tests/openstack_upgrade/tests.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Define class for OpenStack Upgrade.""" + +import logging +import unittest + +from zaza.openstack.utilities import ( + cli as cli_utils, + upgrade_utils as upgrade_utils, + openstack as openstack_utils, + openstack_upgrade as openstack_upgrade, +) +from zaza.openstack.charm_tests.nova.tests import LTSGuestCreateTest + + +class OpenStackUpgradeVMLaunchBase(object): + """A base class to peform a simple validation on the cloud. + + This wraps an OpenStack upgrade with a VM launch before and after the + upgrade. + + This test requires a full OpenStack including at least: keystone, glance, + nova-cloud-controller, nova-compute, neutron-gateway, neutron-api and + neutron-openvswitch. + + This class should be used as a base class to the upgrade 'test'. + """ + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenStackUpgradeMixin setUpClass") + super().setUpClass() + cls.lts = LTSGuestCreateTest() + cls.lts.setUpClass() + + def test_100_validate_pre_openstack_upgrade_cloud(self): + """Validate pre openstack upgrade.""" + logging.info("Validate pre-openstack-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + def test_500_validate_openstack_upgraded_cloud(self): + """Validate post openstack upgrade.""" + logging.info("Validate post-openstack-upgrade: Spin up LTS instance") + self.lts.test_launch_small_instance() + + +class WaitForMySQL(unittest.TestCase): + """Helper test to wait on mysql-innodb-cluster to be fully ready. + + In practice this means that there is at least on R/W unit available. + Sometimes, after restarting units in the mysql-innodb-cluster, all the + units are R/O until the cluster picks the R/W unit. + """ + + @classmethod + def setUpClass(cls): + """Set up class.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_100_wait_for_happy_mysql_innodb_cluster(self): + """Wait for mysql cluster to have at least one R/W node.""" + logging.info("Starting wait for an R/W unit.") + openstack_upgrade.block_until_mysql_innodb_cluster_has_rw() + logging.info("Done .. all seems well.") + + +class OpenStackUpgradeTestsFocalUssuri(OpenStackUpgradeVMLaunchBase): + """Upgrade OpenStack from distro -> cloud:focal-victoria.""" + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_200_run_openstack_upgrade(self): + """Run openstack upgrade, but work out what to do.""" + openstack_upgrade.run_upgrade_tests("cloud:focal-victoria") + + +class OpenStackUpgradeTests(OpenStackUpgradeVMLaunchBase): + """A Principal Class to encapsulate OpenStack Upgrade Tests. + + A generic Test class that can discover which Ubuntu version and OpenStack + version to upgrade from. + + TODO: Not used at present. Use the declarative tests directly that choose + the version to upgrade to. The functions that this class depends on need a + bit more work regarding how the determine which version to go to. + """ + + @classmethod + def setUpClass(cls): + """Run setup for OpenStack Upgrades.""" + print("Running OpenstackUpgradeTests setUpClass") + super().setUpClass() + cli_utils.setup_logging() + + def test_200_run_openstack_upgrade(self): + """Run openstack upgrade, but work out what to do. + + TODO: This is really inefficient at the moment, and doesn't (yet) + determine which ubuntu version to work from. Don't use until we can + make it better. + """ + # TODO: work out the most recent Ubuntu version; we assume this is the + # version that OpenStack is running on. + ubuntu_version = "focal" + logging.info("Getting all principle applications ...") + principle_services = upgrade_utils.get_all_principal_applications() + logging.info( + "Getting OpenStack vesions from principal applications ...") + current_versions = openstack_utils.get_current_os_versions( + principle_services) + logging.info("current versions: %s" % current_versions) + # Find the lowest value openstack release across all services and make + # sure all servcies are upgraded to one release higher than the lowest + from_version = upgrade_utils.get_lowest_openstack_version( + current_versions) + logging.info("from version: %s" % from_version) + to_version = upgrade_utils.determine_next_openstack_release( + from_version)[1] + logging.info("to version: %s" % to_version) + # TODO: need to determine the ubuntu base verion that is being upgraded + target_source = upgrade_utils.determine_new_source( + ubuntu_version, from_version, to_version, single_increment=True) + logging.info("target source: %s" % target_source) + assert target_source is not None + openstack_upgrade.run_upgrade_tests(target_source) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index adfb92d..1e4e05e 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -174,7 +174,8 @@ def auto_initialize(cacert=None, validation_application='keystone', wait=True): zaza.model.wait_for_agent_status() test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( - states=test_config.get('target_deploy_status', {})) + states=test_config.get('target_deploy_status', {}), + timeout=7200) if validation_application: validate_ca(cacertificate, application=validation_application) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index a93e61d..3dd5361 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -16,6 +16,23 @@ This module contains a number of functions for interacting with OpenStack. """ +import datetime +import io +import itertools +import juju_wait +import logging +import os +import paramiko +import re +import six +import subprocess +import sys +import tempfile +import tenacity +import textwrap +import urllib + + from .os_versions import ( OPENSTACK_CODENAMES, SWIFT_CODENAMES, @@ -48,21 +65,6 @@ from neutronclient.common import exceptions as neutronexceptions from octaviaclient.api.v2 import octavia as octaviaclient from swiftclient import client as swiftclient -import datetime -import io -import itertools -import juju_wait -import logging -import os -import paramiko -import re -import six -import subprocess -import sys -import tempfile -import tenacity -import textwrap -import urllib import zaza @@ -1554,6 +1556,7 @@ def get_current_os_versions(deployed_applications, model_name=None): for application in UPGRADE_SERVICES: if application['name'] not in deployed_applications: continue + logging.info("looking at application: {}".format(application)) version = generic_utils.get_pkg_version(application['name'], application['type']['pkg'], diff --git a/zaza/openstack/utilities/openstack_upgrade.py b/zaza/openstack/utilities/openstack_upgrade.py index 9ad4d50..c279e58 100755 --- a/zaza/openstack/utilities/openstack_upgrade.py +++ b/zaza/openstack/utilities/openstack_upgrade.py @@ -95,8 +95,8 @@ async def async_action_unit_upgrade(units, model_name=None): action_unit_upgrade = sync_wrapper(async_action_unit_upgrade) -def action_upgrade_group(applications, model_name=None): - """Upgrade units using action managed upgrades. +def action_upgrade_apps(applications, model_name=None): + """Upgrade units in the applications using action managed upgrades. Upgrade all units of the given applications using action managed upgrades. This involves the following process: @@ -142,6 +142,45 @@ def action_upgrade_group(applications, model_name=None): done.extend(target) + # Ensure that mysql-innodb-cluster has at least one R/W group (it can get + # into a state where all are R/O whilst it is sorting itself out after an + # openstack_upgrade + if "mysql-innodb-cluster" in applications: + block_until_mysql_innodb_cluster_has_rw(model_name) + + # Now we need to wait for the model to go back to idle. + zaza.model.block_until_all_units_idle(model_name) + + +async def async_block_until_mysql_innodb_cluster_has_rw(model=None, + timeout=None): + """Block until the mysql-innodb-cluster is in a healthy state. + + Curiously, after a series of pauses and restarts (e.g. during an upgrade) + the mysql-innodb-cluster charms may not yet have agreed which one is the + R/W node; i.e. they are all R/O. Anyway, eventually they sort it out and + one jumps to the front and says "it's me!". This is detected, externally, + by the status line including R/W in the output. + + This function blocks until that happens so that no charm attempts to have a + chat with the mysql server before it has settled, thus breaking the whole + test. + """ + async def async_check_workload_messages_for_rw(model=None): + """Return True if a least one work message contains R/W.""" + status = await zaza.model.async_get_status() + app_status = status.applications.get("mysql-innodb-cluster") + units_data = app_status.units.values() + workload_statuses = [d.workload_status.info for d in units_data] + return any("R/W" in s for s in workload_statuses) + + await zaza.model.async_block_until(async_check_workload_messages_for_rw, + timeout=timeout) + + +block_until_mysql_innodb_cluster_has_rw = sync_wrapper( + async_block_until_mysql_innodb_cluster_has_rw) + def set_upgrade_application_config(applications, new_source, action_managed=True, model_name=None): @@ -150,7 +189,7 @@ def set_upgrade_application_config(applications, new_source, Set the charm config for upgrade. :param applications: List of application names. - :type applications: [] + :type applications: List[str] :param new_source: New package origin. :type new_source: str :param action_managed: Whether to set action-managed-upgrade config option. @@ -180,8 +219,8 @@ def set_upgrade_application_config(applications, new_source, def is_action_upgradable(app, model_name=None): """Can application be upgraded using action managed upgrade method. - :param new_source: New package origin. - :type new_source: str + :param app: The application to check + :type app: str :param model_name: Name of model to query. :type model_name: str :returns: Whether app be upgraded using action managed upgrade method. @@ -196,66 +235,95 @@ def is_action_upgradable(app, model_name=None): return supported -def run_action_upgrade(group, new_source, model_name=None): +def is_already_upgraded(app, new_src, model_name=None): + """Return True if the app has already been upgraded. + + :param app: The application to check + :type app: str + :param new_src: the new source (distro, cloud:x-y, etc.) + :type new_src: str + :param model_name: Name of model to query. + :type model_name: str + :returns: Whether app be upgraded using action managed upgrade method. + :rtype: bool + """ + config = zaza.model.get_application_config(app, model_name=model_name) + try: + src = config['openstack-origin']['value'] + key_was = 'openstack-origin' + except KeyError: + src = config['source']['value'] + key_was = 'source' + logging.info("origin for {} is {}={}".format(app, key_was, src)) + return src == new_src + + +def run_action_upgrades(apps, new_source, model_name=None): """Upgrade payload of all applications in group using action upgrades. - :param group: List of applications to upgrade. - :type group + :param apps: List of applications to upgrade. + :type apps: List[str] :param new_source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ - set_upgrade_application_config(group, new_source, model_name=model_name) - action_upgrade_group(group, model_name=model_name) + set_upgrade_application_config(apps, new_source, model_name=model_name) + action_upgrade_apps(apps, model_name=model_name) -def run_all_in_one_upgrade(group, new_source, model_name=None): +def run_all_in_one_upgrades(apps, new_source, model_name=None): """Upgrade payload of all applications in group using all-in-one method. - :param group: List of applications to upgrade. - :type group: [] + :param apps: List of applications to upgrade. + :type apps: List[str] :source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ set_upgrade_application_config( - group, + apps, new_source, model_name=model_name, action_managed=False) zaza.model.block_until_all_units_idle() -def run_upgrade(group, new_source, model_name=None): +def run_upgrade_on_apps(apps, new_source, model_name=None): """Upgrade payload of all applications in group. Upgrade apps using action managed upgrades where possible and fallback to all_in_one method. - :param group: List of applications to upgrade. - :type group: [] + :param apps: List of applications to upgrade. + :type apps: [] :param new_source: New package origin. :type new_source: str :param model_name: Name of model to query. :type model_name: str """ - action_upgrade = [] - all_in_one_upgrade = [] - for app in group: + action_upgrades = [] + all_in_one_upgrades = [] + for app in apps: + if is_already_upgraded(app, new_source, model_name=model_name): + logging.info("Application '%s' is already upgraded. Skipping.", + app) + continue if is_action_upgradable(app, model_name=model_name): - action_upgrade.append(app) + action_upgrades.append(app) else: - all_in_one_upgrade.append(app) - run_all_in_one_upgrade( - all_in_one_upgrade, - new_source, - model_name=model_name) - run_action_upgrade( - action_upgrade, - new_source, - model_name=model_name) + all_in_one_upgrades.append(app) + if all_in_one_upgrades: + run_all_in_one_upgrades( + all_in_one_upgrades, + new_source, + model_name=model_name) + if action_upgrades: + run_action_upgrades( + action_upgrades, + new_source, + model_name=model_name) def run_upgrade_tests(new_source, model_name=None): @@ -270,8 +338,6 @@ def run_upgrade_tests(new_source, model_name=None): :type model_name: str """ groups = get_upgrade_groups(model_name=model_name) - run_upgrade(groups['Core Identity'], new_source, model_name=model_name) - run_upgrade(groups['Storage'], new_source, model_name=model_name) - run_upgrade(groups['Control Plane'], new_source, model_name=model_name) - run_upgrade(groups['Compute'], new_source, model_name=model_name) - run_upgrade(groups['sweep_up'], new_source, model_name=model_name) + for name, apps in groups: + logging.info("Performing upgrade of %s", name) + run_upgrade_on_apps(apps, new_source, model_name=model_name) diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 995e0bb..8e3a063 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -19,6 +19,12 @@ import logging import re import zaza.model +import zaza.utilities.juju +from zaza.openstack.utilities.os_versions import ( + OPENSTACK_CODENAMES, + UBUNTU_OPENSTACK_RELEASE, + OPENSTACK_RELEASES_PAIRS, +) SERVICE_GROUPS = ( @@ -46,7 +52,7 @@ def get_upgrade_candidates(model_name=None, filters=None): :param filters: List of filter functions to apply :type filters: List[fn] :returns: List of application that can have their payload upgraded. - :rtype: [] + :rtype: Dict[str, Dict[str, ANY]] """ if filters is None: filters = [] @@ -163,8 +169,8 @@ def get_series_upgrade_groups(model_name=None, extra_filters=None): :param model_name: Name of model to query. :type model_name: str - :returns: Dict of group lists keyed on group name. - :rtype: collections.OrderedDict + :returns: List of tuples(group name, applications) + :rtype: List[Tuple[str, Dict[str, ANY]]] """ filters = [_filter_subordinates] filters = _apply_extra_filters(filters, extra_filters) @@ -226,3 +232,110 @@ def extract_charm_name_from_url(charm_url): """ charm_name = re.sub(r'-[0-9]+$', '', charm_url.split('/')[-1]) return charm_name.split(':')[-1] + + +def get_all_principal_applications(model_name=None): + """Return a list of all the prinical applications in the model. + + :param model_name: Optional model name + :type model_name: Optional[str] + :returns: List of principal application names + :rtype: List[str] + """ + status = zaza.utilities.juju.get_full_juju_status(model_name=model_name) + return [application for application in status.applications.keys() + if not status.applications.get(application)['subordinate-to']] + + +def get_lowest_openstack_version(current_versions): + """Get the lowest OpenStack version from the list of current versions. + + :param current_versions: The list of versions + :type current_versions: List[str] + :returns: the lowest version currently installed. + :rtype: str + """ + lowest_version = 'zebra' + for svc in current_versions.keys(): + if current_versions[svc] < lowest_version: + lowest_version = current_versions[svc] + return lowest_version + + +def determine_next_openstack_release(release): + """Determine the next release after the one passed as a str. + + The returned value is a tuple of the form: ('2020.1', 'ussuri') + + :param release: the release to use as the base + :type release: str + :returns: the release tuple immediately after the current one. + :rtype: Tuple[str, str] + :raises: KeyError if the current release doesn't actually exist + """ + old_index = list(OPENSTACK_CODENAMES.values()).index(release) + new_index = old_index + 1 + return list(OPENSTACK_CODENAMES.items())[new_index] + + +def determine_new_source(ubuntu_version, current_source, new_release, + single_increment=True): + """Determine the new source/openstack-origin value based on new release. + + This takes the ubuntu_version and the current_source (in the form of + 'distro' or 'cloud:xenial-mitaka') and either converts it to a new source, + or returns None if the new_release will match the current_source (i.e. it's + already at the right release), or it's simply not possible. + + If single_increment is set, then the returned source will only be returned + if the new_release is one more than the release in the current source. + + :param ubuntu_version: the ubuntu version that the app is installed on. + :type ubuntu_version: str + :param current_source: a source in the form of 'distro' or + 'cloud:xenial-mitaka' + :type current_source: str + :param new_release: a new OpenStack version codename. e.g. 'stein' + :type new_release: str + :param single_increment: If True, only allow single increment upgrade. + :type single_increment: boolean + :returns: The new source in the form of 'cloud:bionic-train' or None if not + possible + :rtype: Optional[str] + :raises: KeyError if any of the strings don't correspond to known values. + """ + logging.warn("determine_new_source: locals: %s", locals()) + if current_source == 'distro': + # convert to a ubuntu-openstack pair + current_source = "cloud:{}-{}".format( + ubuntu_version, UBUNTU_OPENSTACK_RELEASE[ubuntu_version]) + # strip out the current openstack version + if ':' not in current_source: + current_source = "cloud:{}-{}".format(ubuntu_version, current_source) + pair = current_source.split(':')[1] + u_version, os_version = pair.split('-', 2) + if u_version != ubuntu_version: + logging.warn("determine_new_source: ubuntu_versions don't match: " + "%s != %s" % (ubuntu_version, u_version)) + return None + # determine versions + openstack_codenames = list(OPENSTACK_CODENAMES.values()) + old_index = openstack_codenames.index(os_version) + try: + new_os_version = openstack_codenames[old_index + 1] + except IndexError: + logging.warn("determine_new_source: no OpenStack version after " + "'%s'" % os_version) + return None + if single_increment and new_release != new_os_version: + logging.warn("determine_new_source: requested version '%s' not a " + "single increment from '%s' which is '%s'" % ( + new_release, os_version, new_os_version)) + return None + # now check that there is a combination of u_version-new_os_version + new_pair = "{}_{}".format(u_version, new_os_version) + if new_pair not in OPENSTACK_RELEASES_PAIRS: + logging.warn("determine_new_source: now release pair candidate for " + " combination cloud:%s-%s" % (u_version, new_os_version)) + return None + return "cloud:{}-{}".format(u_version, new_os_version) From ebc51b490e92ed3f8d58e5af6a4c9606e52a5cf6 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Wed, 6 Jan 2021 17:19:56 +0100 Subject: [PATCH 057/180] Make the get_os_release default match get_current_os_release_pair --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 3dd5361..33f0e02 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1605,7 +1605,7 @@ def get_current_os_release_pair(application='keystone'): return '{}_{}'.format(series, os_version) -def get_os_release(release_pair=None, application=None): +def get_os_release(release_pair=None, application='keystone'): """Return index of release in OPENSTACK_RELEASES_PAIRS. :param release_pair: OpenStack release pair eg 'focal_ussuri' From cdba5b64ca2d62d812f328be85aca3ad15f5f9df Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 12 Jan 2021 13:11:17 +0000 Subject: [PATCH 058/180] Run update-status before manila api test The manila charm contains a 'band-aid' for Bug #1706699 which relies on update-status to bring up services if needed. When the tests run an update-status hook might not have run so services may still be stopped so force a hook execution. --- zaza/openstack/charm_tests/manila/tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index 8a1fdee..c00b993 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -20,6 +20,7 @@ import tenacity from manilaclient import client as manilaclient +import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils @@ -35,6 +36,12 @@ class ManilaTests(test_utils.OpenStackBaseTest): def test_manila_api(self): """Test that the Manila API is working.""" + # The manila charm contains a 'band-aid' for Bug #1706699 which relies + # on update-status to bring up services if needed. When the tests run + # an update-status hook might not have run so services may still be + # stopped so force a hook execution. + for unit in zaza.model.get_units('manila'): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") self.assertEqual([], self._list_shares()) @tenacity.retry( From 9961c9468bac26dfe6675ada46fdd137316a5b4f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 13 Jan 2021 09:42:34 +0100 Subject: [PATCH 059/180] Sync pertinent tox.ini settings from release-tools The new PIP resolver wreaks havoc for this repository. Long term we should split unit test requirements into a separate file and whip unit tests into shape wrt. mocking out everything as opposed to relying on having random modules installed in the test environment. --- tox.ini | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tox.ini b/tox.ini index 7d67e9f..dc09948 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,22 @@ [tox] envlist = pep8, py3 skipsdist = True +# NOTE: Avoid build/test env pollution by not enabling sitepackages. +sitepackages = False +# NOTE: Avoid false positives by not skipping missing interpreters. +skip_missing_interpreters = False +# NOTES: +# * We avoid the new dependency resolver by pinning pip < 20.3, see +# https://github.com/pypa/pip/issues/9187 +# * Pinning dependencies requires tox >= 3.2.0, see +# https://tox.readthedocs.io/en/latest/config.html#conf-requires +# * It is also necessary to pin virtualenv as a newer virtualenv would still +# lead to fetching the latest pip in the func* tox targets, see +# https://stackoverflow.com/a/38133283 +requires = pip < 20.3 + virtualenv < 20.0 +# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci +minversion = 3.2.0 [testenv] setenv = VIRTUAL_ENV={envdir} From ee0dd80cec01f9bf8649851ba4c052f92cbee463 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 12 Jan 2021 17:11:35 +0100 Subject: [PATCH 060/180] Split configure_gateway_ext_port function The function does three separate things today, and two of its tasks are useful for other provider types such as MAAS. Also fix create_additional_port_for_machines idempotency. We previously added a run time assertion to fail early when attempting to configure networking for an invalid bundle. The check had the side effect of prohibiting subsequent runs on already configured models. --- .../test_zaza_utilities_openstack.py | 78 +++-- zaza/openstack/utilities/openstack.py | 327 ++++++++++++------ 2 files changed, 274 insertions(+), 131 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 48408e0..7e0e8f1 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1262,34 +1262,68 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.get_application.side_effect = KeyError self.assertFalse(openstack_utils.ngw_present()) - def test_configure_gateway_ext_port(self): - # FIXME: this is not a complete unit test for the function as one did - # not exist at all I'm adding this to test one bit and we'll add more - # as we go. + def test_get_charm_networking_data(self): self.patch_object(openstack_utils, 'deprecated_external_networking') self.patch_object(openstack_utils, 'dvr_enabled') self.patch_object(openstack_utils, 'ovn_present') self.patch_object(openstack_utils, 'ngw_present') + self.patch_object(openstack_utils, 'get_ovs_uuids') self.patch_object(openstack_utils, 'get_gateway_uuids') - self.patch_object(openstack_utils, 'get_admin_net') + self.patch_object(openstack_utils, 'get_ovn_uuids') + self.patch_object(openstack_utils.model, 'get_application') self.dvr_enabled.return_value = False self.ovn_present.return_value = False - self.ngw_present.return_value = True - self.get_admin_net.return_value = {'id': 'fakeid'} + self.ngw_present.return_value = False + self.get_ovs_uuids.return_value = [] + self.get_gateway_uuids.return_value = [] + self.get_ovn_uuids.return_value = [] + self.get_application.side_effect = KeyError - novaclient = mock.MagicMock() - neutronclient = mock.MagicMock() - - def _fake_empty_generator(empty=True): - if empty: - return - yield - - self.get_gateway_uuids.side_effect = _fake_empty_generator with self.assertRaises(RuntimeError): - openstack_utils.configure_gateway_ext_port( - novaclient, neutronclient) - # provide a uuid and check that we don't raise RuntimeError - self.get_gateway_uuids.side_effect = ['fake-uuid'] - openstack_utils.configure_gateway_ext_port( - novaclient, neutronclient) + openstack_utils.get_charm_networking_data() + self.ngw_present.return_value = True + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVS, + ['neutron-gateway'], + mock.ANY, + 'data-port', + {})) + self.dvr_enabled.return_value = True + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVS_DVR, + ['neutron-gateway', 'neutron-openvswitch'], + mock.ANY, + 'data-port', + {})) + self.ngw_present.return_value = False + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT, + ['neutron-openvswitch'], + mock.ANY, + 'data-port', + {})) + self.dvr_enabled.return_value = False + self.ovn_present.return_value = True + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVN, + ['ovn-chassis'], + mock.ANY, + 'bridge-interface-mappings', + {'ovn-bridge-mappings': 'physnet1:br-ex'})) + self.get_application.side_effect = None + self.assertEquals( + openstack_utils.get_charm_networking_data(), + openstack_utils.CharmedOpenStackNetworkingData( + openstack_utils.OpenStackNetworkingTopology.ML2_OVN, + ['ovn-chassis', 'ovn-dedicated-chassis'], + mock.ANY, + 'bridge-interface-mappings', + {'ovn-bridge-mappings': 'physnet1:br-ex'})) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 33f0e02..33b8a52 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -16,7 +16,10 @@ This module contains a number of functions for interacting with OpenStack. """ +import collections +import copy import datetime +import enum import io import itertools import juju_wait @@ -721,6 +724,211 @@ def add_interface_to_netplan(server_name, mac_address): model.run_on_unit(unit_name, "sudo netplan apply") +class OpenStackNetworkingTopology(enum.Enum): + """OpenStack Charms Network Topologies.""" + + ML2_OVS = 'ML2+OVS' + ML2_OVS_DVR = 'ML2+OVS+DVR' + ML2_OVS_DVR_SNAT = 'ML2+OVS+DVR, no dedicated GWs' + ML2_OVN = 'ML2+OVN' + + +CharmedOpenStackNetworkingData = collections.namedtuple( + 'CharmedOpenStackNetworkingData', + [ + 'topology', + 'application_names', + 'unit_machine_ids', + 'port_config_key', + 'other_config', + ]) + + +def get_charm_networking_data(limit_gws=None): + """Inspect Juju model, determine networking topology and return data. + + :param limit_gws: Limit the number of gateways that get a port attached + :type limit_gws: Optional[int] + :rtype: CharmedOpenStackNetworkingData[ + OpenStackNetworkingTopology, + List[str], + Iterator[str], + str, + Dict[str,str]] + :returns: Named Tuple with networking data, example: + CharmedOpenStackNetworkingData( + OpenStackNetworkingTopology.ML2_OVN, + ['ovn-chassis', 'ovn-dedicated-chassis'], + ['machine-id-1', 'machine-id-2'], # generator object + 'bridge-interface-mappings', + {'ovn-bridge-mappings': 'physnet1:br-ex'}) + :raises: RuntimeError + """ + # Initialize defaults, these will be amended to fit the reality of the + # model in the checks below. + topology = OpenStackNetworkingTopology.ML2_OVS + other_config = {} + port_config_key = ( + 'data-port' if not deprecated_external_networking() else 'ext-port') + unit_machine_ids = [] + application_names = [] + + if dvr_enabled(): + if ngw_present(): + application_names = ['neutron-gateway', 'neutron-openvswitch'] + topology = OpenStackNetworkingTopology.ML2_OVS_DVR + else: + application_names = ['neutron-openvswitch'] + topology = OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT + unit_machine_ids = itertools.islice( + itertools.chain( + get_ovs_uuids(), + get_gateway_uuids()), + limit_gws) + elif ngw_present(): + unit_machine_ids = itertools.islice( + get_gateway_uuids(), limit_gws) + application_names = ['neutron-gateway'] + elif ovn_present(): + topology = OpenStackNetworkingTopology.ML2_OVN + unit_machine_ids = itertools.islice(get_ovn_uuids(), limit_gws) + application_names = ['ovn-chassis'] + try: + ovn_dc_name = 'ovn-dedicated-chassis' + model.get_application(ovn_dc_name) + application_names.append(ovn_dc_name) + except KeyError: + # ovn-dedicated-chassis not in deployment + pass + port_config_key = 'bridge-interface-mappings' + other_config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) + else: + raise RuntimeError('Unable to determine charm network topology.') + + return CharmedOpenStackNetworkingData( + topology, + application_names, + unit_machine_ids, + port_config_key, + other_config) + + +def create_additional_port_for_machines(novaclient, neutronclient, net_id, + unit_machine_ids, + add_dataport_to_netplan=False): + """Create additional port for machines for use with external networking. + + :param novaclient: Undercloud Authenticated novaclient. + :type novaclient: novaclient.Client object + :param neutronclient: Undercloud Authenticated neutronclient. + :type neutronclient: neutronclient.Client object + :param net_id: Network ID to create ports on. + :type net_id: string + :param unit_machine_ids: Juju provider specific machine IDs for which we + should add ports on. + :type unit_machine_ids: Iterator[str] + :param add_dataport_to_netplan: Whether the newly created port should be + added to instance system configuration so + that it is brought up on instance reboot. + :type add_dataport_to_netplan: Optional[bool] + :returns: List of MAC addresses for created ports. + :rtype: List[str] + :raises: RuntimeError + """ + eligible_machines = 0 + for uuid in unit_machine_ids: + eligible_machines += 1 + server = novaclient.servers.get(uuid) + ext_port_name = "{}_ext-port".format(server.name) + for port in neutronclient.list_ports(device_id=server.id)['ports']: + if port['name'] == ext_port_name: + logging.warning( + 'Instance {} already has additional port, skipping.' + .format(server.id)) + break + else: + logging.info('Attaching additional port to instance ("{}"), ' + 'connected to net id: {}' + .format(uuid, net_id)) + body_value = { + "port": { + "admin_state_up": True, + "name": ext_port_name, + "network_id": net_id, + "port_security_enabled": False, + } + } + port = neutronclient.create_port(body=body_value) + server.interface_attach(port_id=port['port']['id'], + net_id=None, fixed_ip=None) + if add_dataport_to_netplan: + mac_address = get_mac_from_port(port, neutronclient) + add_interface_to_netplan(server.name, + mac_address=mac_address) + if not eligible_machines: + # NOTE: unit_machine_ids may be an iterator so testing it for contents + # or length prior to iterating over it is futile. + raise RuntimeError('Unable to determine UUIDs for machines to attach ' + 'external networking to.') + + # Retrieve the just created ports from Neutron so that we can provide our + # caller with their MAC addresses. + return [ + port['mac_address'] + for port in neutronclient.list_ports(network_id=net_id)['ports'] + if 'ext-port' in port['name'] + ] + + +def configure_networking_charms(networking_data, macs, use_juju_wait=True): + """Configure external networking for networking charms. + + :param networking_data: Data on networking charm topology. + :type networking_data: CharmedOpenStackNetworkingData + :param macs: MAC addresses of ports for use with external networking. + :type macs: Iterator[str] + :param use_juju_wait: Whether to use juju wait to wait for the model to + settle once the gateway has been configured. Default is True + :type use_juju_wait: Optional[bool] + """ + br_mac_fmt = 'br-ex:{}' if not deprecated_external_networking() else '{}' + br_mac = [ + br_mac_fmt.format(mac) + for mac in macs + ] + + config = copy.deepcopy(networking_data.other_config) + config.update({networking_data.port_config_key: ' '.join(sorted(br_mac))}) + + for application_name in networking_data.application_names: + logging.info('Setting {} on {}'.format( + config, application_name)) + current_data_port = get_application_config_option( + application_name, + networking_data.port_config_key) + if current_data_port == config[networking_data.port_config_key]: + logging.info('Config already set to value') + return + + model.set_application_config( + application_name, + configuration=config) + # NOTE(fnordahl): We are stuck with juju_wait until we figure out how + # to deal with all the non ['active', 'idle', 'Unit is ready.'] + # workload/agent states and msgs that our mojo specs are exposed to. + if use_juju_wait: + juju_wait.wait(wait_for_workload=True) + else: + zaza.model.wait_for_agent_status() + # TODO: shouldn't access get_charm_config() here as it relies on + # ./tests/tests.yaml existing by default (regardless of the + # fatal=False) ... it's not great design. + test_config = zaza.charm_lifecycle.utils.get_charm_config( + fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get('target_deploy_status', {})) + + def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, add_dataport_to_netplan=False, limit_gws=None, @@ -739,123 +947,24 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, settle once the gateway has been configured. Default is True :type use_juju_wait: boolean """ - deprecated_extnet_mode = deprecated_external_networking() - - port_config_key = 'data-port' - if deprecated_extnet_mode: - port_config_key = 'ext-port' - - config = {} - if dvr_enabled(): - uuids = itertools.islice(itertools.chain(get_ovs_uuids(), - get_gateway_uuids()), - limit_gws) + networking_data = get_charm_networking_data(limit_gws=limit_gws) + if networking_data.topology in ( + OpenStackNetworkingTopology.ML2_OVS_DVR, + OpenStackNetworkingTopology.ML2_OVS_DVR_SNAT): # If dvr, do not attempt to persist nic in netplan # https://github.com/openstack-charmers/zaza-openstack-tests/issues/78 add_dataport_to_netplan = False - application_names = ['neutron-openvswitch'] - try: - ngw = 'neutron-gateway' - model.get_application(ngw) - application_names.append(ngw) - except KeyError: - # neutron-gateway not in deployment - pass - elif ngw_present(): - uuids = itertools.islice(get_gateway_uuids(), limit_gws) - application_names = ['neutron-gateway'] - elif ovn_present(): - uuids = itertools.islice(get_ovn_uuids(), limit_gws) - application_names = ['ovn-chassis'] - try: - ovn_dc_name = 'ovn-dedicated-chassis' - model.get_application(ovn_dc_name) - application_names.append(ovn_dc_name) - except KeyError: - # ovn-dedicated-chassis not in deployment - pass - port_config_key = 'bridge-interface-mappings' - config.update({'ovn-bridge-mappings': 'physnet1:br-ex'}) - add_dataport_to_netplan = True - else: - raise RuntimeError('Unable to determine charm network topology.') if not net_id: net_id = get_admin_net(neutronclient)['id'] - ports_created = 0 - for uuid in uuids: - server = novaclient.servers.get(uuid) - ext_port_name = "{}_ext-port".format(server.name) - for port in neutronclient.list_ports(device_id=server.id)['ports']: - if port['name'] == ext_port_name: - logging.warning( - 'Neutron Gateway already has additional port') - break - else: - logging.info('Attaching additional port to instance ("{}"), ' - 'connected to net id: {}' - .format(uuid, net_id)) - body_value = { - "port": { - "admin_state_up": True, - "name": ext_port_name, - "network_id": net_id, - "port_security_enabled": False, - } - } - port = neutronclient.create_port(body=body_value) - ports_created += 1 - server.interface_attach(port_id=port['port']['id'], - net_id=None, fixed_ip=None) - if add_dataport_to_netplan: - mac_address = get_mac_from_port(port, neutronclient) - add_interface_to_netplan(server.name, - mac_address=mac_address) - if not ports_created: - # NOTE: uuids is an iterator so testing it for contents or length prior - # to iterating over it is futile. - raise RuntimeError('Unable to determine UUIDs for machines to attach ' - 'external networking to.') + macs = create_additional_port_for_machines( + novaclient, neutronclient, net_id, networking_data.unit_machine_ids, + add_dataport_to_netplan) - ext_br_macs = [] - for port in neutronclient.list_ports(network_id=net_id)['ports']: - if 'ext-port' in port['name']: - if deprecated_extnet_mode: - ext_br_macs.append(port['mac_address']) - else: - ext_br_macs.append('br-ex:{}'.format(port['mac_address'])) - ext_br_macs.sort() - ext_br_macs_str = ' '.join(ext_br_macs) - - if ext_br_macs: - config.update({port_config_key: ext_br_macs_str}) - for application_name in application_names: - logging.info('Setting {} on {}'.format( - config, application_name)) - current_data_port = get_application_config_option(application_name, - port_config_key) - if current_data_port == ext_br_macs_str: - logging.info('Config already set to value') - return - - model.set_application_config( - application_name, - configuration=config) - # NOTE(fnordahl): We are stuck with juju_wait until we figure out how - # to deal with all the non ['active', 'idle', 'Unit is ready.'] - # workload/agent states and msgs that our mojo specs are exposed to. - if use_juju_wait: - juju_wait.wait(wait_for_workload=True) - else: - zaza.model.wait_for_agent_status() - # TODO: shouldn't access get_charm_config() here as it relies on - # ./tests/tests.yaml existing by default (regardless of the - # fatal=False) ... it's not great design. - test_config = zaza.charm_lifecycle.utils.get_charm_config( - fatal=False) - zaza.model.wait_for_application_states( - states=test_config.get('target_deploy_status', {})) + if macs: + configure_networking_charms( + networking_data, macs, use_juju_wait=use_juju_wait) @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), From ec637329740247d8384917c9dbe4c57cf6857de2 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 13 Jan 2021 12:28:39 +0100 Subject: [PATCH 061/180] Support configuring networknig charms on MAAS When on MAAS support doing charm based configuration of OVS by retrieving MAC address of ports attached to external network from MAAS. Note that we should extend the MAAS support to also work with deployments where MAAS does the OVS configuration for us. --- unit_tests/__init__.py | 5 +++++ zaza/openstack/charm_tests/neutron/setup.py | 18 ++++++++++++++-- zaza/openstack/utilities/openstack.py | 23 +++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/unit_tests/__init__.py b/unit_tests/__init__.py index 8203d13..03c4879 100644 --- a/unit_tests/__init__.py +++ b/unit_tests/__init__.py @@ -11,3 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +import sys +import unittest.mock as mock + +sys.modules['zaza.utilities.maas'] = mock.MagicMock() diff --git a/zaza/openstack/charm_tests/neutron/setup.py b/zaza/openstack/charm_tests/neutron/setup.py index f65b87a..a1d1dd4 100644 --- a/zaza/openstack/charm_tests/neutron/setup.py +++ b/zaza/openstack/charm_tests/neutron/setup.py @@ -15,6 +15,7 @@ """Setup for Neutron deployments.""" import functools +import logging from zaza.openstack.configure import ( network, @@ -89,12 +90,25 @@ def basic_overcloud_network(limit_gws=None): 'configure_gateway_ext_port_use_juju_wait', True) # Handle network for OpenStack-on-OpenStack scenarios - if juju_utils.get_provider_type() == "openstack": + provider_type = juju_utils.get_provider_type() + if provider_type == "openstack": undercloud_ks_sess = openstack_utils.get_undercloud_keystone_session() network.setup_gateway_ext_port(network_config, keystone_session=undercloud_ks_sess, - limit_gws=None, + limit_gws=limit_gws, use_juju_wait=use_juju_wait) + elif provider_type == "maas": + # NOTE(fnordahl): After validation of the MAAS+Netplan Open vSwitch + # integration support, we would most likely want to add multiple modes + # of operation with MAAS. + # + # Perform charm based OVS configuration + openstack_utils.configure_charmed_openstack_on_maas( + network_config, limit_gws=limit_gws) + else: + logging.warning('Unknown Juju provider type, "{}", will not perform' + ' charm network configuration.' + .format(provider_type)) # Confugre the overcloud network network.setup_sdn(network_config, keystone_session=keystone_session) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 33b8a52..6e0fa20 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -62,6 +62,7 @@ from keystoneauth1.identity import ( import zaza.openstack.utilities.cert as cert import zaza.utilities.deployment_env as deployment_env import zaza.utilities.juju as juju_utils +import zaza.utilities.maas from novaclient import client as novaclient_client from neutronclient.v2_0 import client as neutronclient from neutronclient.common import exceptions as neutronexceptions @@ -967,6 +968,28 @@ def configure_gateway_ext_port(novaclient, neutronclient, net_id=None, networking_data, macs, use_juju_wait=use_juju_wait) +def configure_charmed_openstack_on_maas(network_config, limit_gws=None): + """Configure networking charms for charm-based OVS config on MAAS provider. + + :param network_config: Network configuration as provided in environment. + :type network_config: Dict[str] + :param limit_gws: Limit the number of gateways that get a port attached + :type limit_gws: Optional[int] + """ + networking_data = get_charm_networking_data(limit_gws=limit_gws) + macs = [ + mim.mac + for mim in zaza.utilities.maas.get_macs_from_cidr( + zaza.utilities.maas.get_maas_client_from_juju_cloud_data( + zaza.model.get_cloud_data()), + network_config['external_net_cidr'], + link_mode=zaza.utilities.maas.LinkMode.LINK_UP) + ] + if macs: + configure_networking_charms( + networking_data, macs, use_juju_wait=False) + + @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=60), reraise=True, retry=tenacity.retry_if_exception_type(KeyError)) def get_mac_from_port(port, neutronclient): From 2b715da7e43b94ad36975063764f33738cc7aa61 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 14 Jan 2021 10:15:34 +0100 Subject: [PATCH 062/180] Make HaclusterScalebackTest more robust (#482) Note that this test will be superseded by #369 in the future. --- zaza/openstack/charm_tests/hacluster/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index af47b76..1690c85 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -125,8 +125,8 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('Waiting for model to settle') zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active') - # NOTE(lourot): the principle application remains blocked after scaling - # back up until lp:1400481 is solved. - zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + # NOTE(lourot): the principle application sometimes remain blocked + # after scaling back up until lp:1400481 is solved. + # zaza.model.block_until_unit_wl_status(other_principle_unit, 'active') zaza.model.block_until_all_units_idle() logging.debug('OK') From 35840a66d6788dc4899847747ee7523895f46e1b Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Sat, 16 Jan 2021 15:28:13 +0100 Subject: [PATCH 063/180] nova: Conditional security checklist based on presence of vault The current test expects Nova to never have TLS connections, let's expect them to be there whenever vault is present. Remove the 'is-volume-encryption-enabled' assertion as it is not a property of the Nova security checks. This was previously masked by the fact that action would always fail due to TLS tests not being enabled for any bundles. --- zaza/openstack/charm_tests/nova/tests.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index b8e7552..edad683 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -495,17 +495,24 @@ class SecurityTests(test_utils.OpenStackBaseTest): # Changes fixing the below expected failures will be made following # this initial work to get validation in. There will be bugs targeted # to each one and resolved independently where possible. - expected_failures = [ - 'is-volume-encryption-enabled', - 'validate-uses-tls-for-glance', - 'validate-uses-tls-for-keystone', ] expected_passes = [ 'validate-file-ownership', 'validate-file-permissions', 'validate-uses-keystone', ] + tls_checks = [ + 'validate-uses-tls-for-glance', + 'validate-uses-tls-for-keystone', + ] + if zaza.model.get_relation_id( + 'nova-cloud-controller', + 'vault', + remote_interface_name='certificates'): + expected_passes.extend(tls_checks) + else: + expected_failures.extend(tls_checks) for unit in zaza.model.get_units(self.application_name, model_name=self.model_name): @@ -519,4 +526,4 @@ class SecurityTests(test_utils.OpenStackBaseTest): action_params={}), expected_passes, expected_failures, - expected_to_pass=False) + expected_to_pass=not len(expected_failures)) From 44adc1fb944403d7d19a7fc2772c910fa8213399 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 19 Jan 2021 10:42:20 +0000 Subject: [PATCH 064/180] Fix policyd call to _login 8681b023 changed the signature of _login but did not update the policyd test. This was not immediately picked up because the policyd test is skipped before groovy due to Bug #1880959 Closes-Bug: #1911923 --- .../charm_tests/openstack_dashboard/tests.py | 102 ++++++++++-------- 1 file changed, 58 insertions(+), 44 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index d3fc625..9588c6f 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -67,7 +67,6 @@ def _login(dashboard_url, domain, username, password, cafile=None): # start session, get csrftoken client = requests.session() client.get(auth_url, verify=cafile) - if 'csrftoken' in client.cookies: csrftoken = client.cookies['csrftoken'] else: @@ -163,7 +162,60 @@ def _do_request(request, cafile=None): return urllib.request.urlopen(request, cafile=cafile) -class OpenStackDashboardTests(test_utils.OpenStackBaseTest): +class OpenStackDashboardBase(): + """Mixin for interacting with Horizon.""" + + def get_base_url(self): + """Return the base url for http(s) requests. + + :returns: URL + :rtype: str + """ + vip = (zaza_model.get_application_config(self.application_name) + .get("vip").get("value")) + if vip: + ip = vip + else: + unit = zaza_model.get_unit_from_name( + zaza_model.get_lead_unit_name(self.application_name)) + ip = unit.public_address + + logging.debug("Dashboard ip is:{}".format(ip)) + scheme = 'http' + if self.use_https: + scheme = 'https' + url = '{}://{}'.format(scheme, ip) + return url + + def get_horizon_url(self): + """Return the url for acccessing horizon. + + :returns: Horizon URL + :rtype: str + """ + url = '{}/horizon'.format(self.get_base_url()) + logging.info("Horizon URL is: {}".format(url)) + return url + + @property + def use_https(self): + """Whether dashboard is using https. + + :returns: Whether dashboard is using https + :rtype: boolean + """ + use_https = False + vault_relation = zaza_model.get_relation_id( + self.application, + 'vault', + remote_interface_name='certificates') + if vault_relation: + use_https = True + return use_https + + +class OpenStackDashboardTests(test_utils.OpenStackBaseTest, + OpenStackDashboardBase): """Encapsulate openstack dashboard charm tests.""" @classmethod @@ -171,13 +223,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): """Run class setup for running openstack dashboard charm tests.""" super(OpenStackDashboardTests, cls).setUpClass() cls.application = 'openstack-dashboard' - cls.use_https = False - vault_relation = zaza_model.get_relation_id( - cls.application, - 'vault', - remote_interface_name='certificates') - if vault_relation: - cls.use_https = True def test_050_local_settings_permissions_regression_check_lp1755027(self): """Assert regression check lp1755027. @@ -302,39 +347,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): mismatches.append(msg) return mismatches - def get_base_url(self): - """Return the base url for http(s) requests. - - :returns: URL - :rtype: str - """ - vip = (zaza_model.get_application_config(self.application_name) - .get("vip").get("value")) - if vip: - ip = vip - else: - unit = zaza_model.get_unit_from_name( - zaza_model.get_lead_unit_name(self.application_name)) - ip = unit.public_address - - logging.debug("Dashboard ip is:{}".format(ip)) - scheme = 'http' - if self.use_https: - scheme = 'https' - url = '{}://{}'.format(scheme, ip) - logging.debug("Base URL is: {}".format(url)) - return url - - def get_horizon_url(self): - """Return the url for acccessing horizon. - - :returns: Horizon URL - :rtype: str - """ - url = '{}/horizon'.format(self.get_base_url()) - logging.info("Horizon URL is: {}".format(url)) - return url - def test_400_connection(self): """Test that dashboard responds to http request. @@ -450,7 +462,8 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest): logging.info("Testing pause resume") -class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): +class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization, + OpenStackDashboardBase): """Test the policyd override using the dashboard.""" good = { @@ -476,6 +489,7 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): super(OpenStackDashboardPolicydTests, cls).setUpClass( application_name="openstack-dashboard") cls.application_name = "openstack-dashboard" + cls.application = cls.application_name def get_client_and_attempt_operation(self, ip): """Attempt to list users on the openstack-dashboard service. @@ -500,7 +514,7 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization): username = 'admin', password = overcloud_auth['OS_PASSWORD'], client, response = _login( - unit.public_address, domain, username, password) + self.get_horizon_url(), domain, username, password) # now attempt to get the domains page _url = self.url.format(unit.public_address) result = client.get(_url) From a828774c4868c6c74fc69e6b0f2a1e690a64d5d9 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sat, 23 Jan 2021 16:17:28 +0000 Subject: [PATCH 065/180] Handle change of CA cert. Closes issue #487 --- zaza/openstack/charm_tests/keystone/setup.py | 4 +- zaza/openstack/charm_tests/keystone/tests.py | 4 +- zaza/openstack/charm_tests/policyd/tests.py | 3 +- zaza/openstack/charm_tests/vault/setup.py | 4 +- zaza/openstack/charm_tests/vault/tests.py | 4 +- zaza/openstack/utilities/openstack.py | 82 +++++++++++++++++--- 6 files changed, 85 insertions(+), 16 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index 748439f..8d64993 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -41,9 +41,11 @@ def wait_for_cacert(model_name=None): :type model_name: str """ logging.info("Waiting for cacert") + cert_file = openstack_utils.get_cert_file_name( + 'keystone') zaza.model.block_until_file_has_contents( 'keystone', - openstack_utils.KEYSTONE_REMOTE_CACERT, + cert_file, 'CERTIFICATE', model_name=model_name) zaza.model.block_until_all_units_idle(model_name=model_name) diff --git a/zaza/openstack/charm_tests/keystone/tests.py b/zaza/openstack/charm_tests/keystone/tests.py index b2830ef..477152d 100644 --- a/zaza/openstack/charm_tests/keystone/tests.py +++ b/zaza/openstack/charm_tests/keystone/tests.py @@ -229,7 +229,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): 'OS_DOMAIN_NAME': DEMO_DOMAIN, } if self.tls_rid: - openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_CACERT'] = openstack_utils.get_cacert() openrc['OS_AUTH_URL'] = ( openrc['OS_AUTH_URL'].replace('http', 'https')) logging.info('keystone IP {}'.format(ip)) @@ -259,7 +259,7 @@ class AuthenticationAuthorizationTest(BaseKeystoneTest): """ def _validate_token_data(openrc): if self.tls_rid: - openrc['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_CACERT'] = openstack_utils.get_cacert() openrc['OS_AUTH_URL'] = ( openrc['OS_AUTH_URL'].replace('http', 'https')) logging.info('keystone IP {}'.format(ip)) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index eca316d..4722755 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -337,8 +337,7 @@ class BasePolicydSpecialization(PolicydTest, logging.info('Authentication for {} on keystone IP {}' .format(openrc['OS_USERNAME'], ip)) if self.tls_rid: - openrc['OS_CACERT'] = \ - openstack_utils.KEYSTONE_LOCAL_CACERT + openrc['OS_CACERT'] = openstack_utils.get_cacert() openrc['OS_AUTH_URL'] = ( openrc['OS_AUTH_URL'].replace('http', 'https')) logging.info('keystone IP {}'.format(ip)) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 1e4e05e..d605063 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -222,9 +222,11 @@ def validate_ca(cacertificate, application="keystone", port=5000): :returns: None :rtype: None """ + cert_file = zaza.openstack.utilities.openstack.get_cert_file_name( + application) zaza.model.block_until_file_has_contents( application, - zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT, + cert_file, cacertificate.decode().strip()) vip = (zaza.model.get_application_config(application) .get("vip").get("value")) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 40227fd..e96f36a 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -154,9 +154,11 @@ class VaultTest(BaseVaultTest): test_config = lifecycle_utils.get_charm_config() del test_config['target_deploy_status']['vault'] + cert_file = zaza.openstack.utilities.openstack.get_cert_file_name( + 'keystone') zaza.model.block_until_file_has_contents( 'keystone', - zaza.openstack.utilities.openstack.KEYSTONE_REMOTE_CACERT, + cert_file, cacert.decode().strip()) zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 6e0fa20..b914c0f 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -69,6 +69,7 @@ from neutronclient.common import exceptions as neutronexceptions from octaviaclient.api.v2 import octavia as octaviaclient from swiftclient import client as swiftclient +from juju.errors import JujuError import zaza @@ -181,10 +182,68 @@ WORKLOAD_STATUS_EXCEPTIONS = { 'ceilometer and gnocchi')}} # For vault TLS certificates +LOCAL_CERT_DIR = "tests" KEYSTONE_CACERT = "keystone_juju_ca_cert.crt" KEYSTONE_REMOTE_CACERT = ( "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT)) -KEYSTONE_LOCAL_CACERT = ("tests/{}".format(KEYSTONE_CACERT)) +KEYSTONE_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, KEYSTONE_CACERT)) + +VAULT_CACERT = "vault_juju_ca_cert.crt" +VAULT_REMOTE_CACERT = ( + "/usr/local/share/ca-certificates/{}".format(VAULT_CACERT)) +VAULT_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, VAULT_CACERT)) + +REMOTE_CERTIFICATES = [VAULT_REMOTE_CACERT, KEYSTONE_REMOTE_CACERT] +LOCAL_CERTIFICATES = [VAULT_LOCAL_CACERT, KEYSTONE_LOCAL_CACERT] + + +async def async_get_cert_file_name(app, cert_files=None, block=True, + model_name=None, timeout=2700): + """Get the name of the CA cert file thats on all units of an application. + + :param app: Name of application + :type capp: str + :param cert_files: List of cert files to search for. + :type cert_files: List[str] + :param block: Whether to block until a consistent cert file is found. + :type block: bool + :param model_name: Name of model to run check in + :type model_name: str + :param timeout: Time to wait for consistent file + :type timeout: int + :returns: Credentials dictionary + :rtype: dict + """ + async def _check_for_file(model, cert_files): + units = model.applications[app].units + results = {u.entity_id: [] for u in units} + for unit in units: + try: + for cf in cert_files: + output = await unit.run('test -e "{}"; echo $?'.format(cf)) + contents = output.data.get('results')['Stdout'] + if "0" in contents: + results[unit.entity_id].append(cf) + except JujuError: + pass + for cert_file in cert_files: + # Check that the certificate file exists on all the units. + if all(cert_file in files for files in results.values()): + return cert_file + else: + return None + + if not cert_files: + cert_files = REMOTE_CERTIFICATES + cert_file = None + async with zaza.model.run_in_model(model_name) as model: + if block: + await zaza.model.async_block_until( + lambda: _check_for_file(model, cert_files), timeout=timeout) + cert_file = await _check_for_file(model, cert_files) + return cert_file + +get_cert_file_name = zaza.model.sync_wrapper(async_get_cert_file_name) def get_cacert(): @@ -193,8 +252,9 @@ def get_cacert(): :returns: Path to CA Certificate bundle or None. :rtype: Optional[str] """ - if os.path.exists(KEYSTONE_LOCAL_CACERT): - return KEYSTONE_LOCAL_CACERT + for _cert in LOCAL_CERTIFICATES: + if os.path.exists(_cert): + return _cert # OpenStack Client helpers @@ -1951,24 +2011,28 @@ def get_overcloud_auth(address=None, model_name=None): 'API_VERSION': 3, } if tls_rid: + cert_file = get_cert_file_name('keystone', model_name=model_name) unit = model.get_first_unit_name('keystone', model_name=model_name) # ensure that the path to put the local cacert in actually exists. The # assumption that 'tests/' exists for, say, mojo is false. # Needed due to: # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 - _dir = os.path.dirname(KEYSTONE_LOCAL_CACERT) + _dir = os.path.dirname(cert_file) if not os.path.exists(_dir): os.makedirs(_dir) + _local_cert_file = "{}/{}".format( + LOCAL_CERT_DIR, + os.path.basename(cert_file)) model.scp_from_unit( unit, - KEYSTONE_REMOTE_CACERT, - KEYSTONE_LOCAL_CACERT) + cert_file, + _local_cert_file) - if os.path.exists(KEYSTONE_LOCAL_CACERT): - os.chmod(KEYSTONE_LOCAL_CACERT, 0o644) - auth_settings['OS_CACERT'] = KEYSTONE_LOCAL_CACERT + if os.path.exists(_local_cert_file): + os.chmod(_local_cert_file, 0o644) + auth_settings['OS_CACERT'] = _local_cert_file return auth_settings From a20733cd1460ada0987ebb2999086eb6dd77cee8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 24 Jan 2021 14:31:29 +0000 Subject: [PATCH 066/180] Refactor ca functions --- zaza/openstack/charm_tests/keystone/setup.py | 4 +- zaza/openstack/charm_tests/vault/setup.py | 7 +- zaza/openstack/charm_tests/vault/tests.py | 4 +- zaza/openstack/utilities/openstack.py | 214 +++++++++++++------ 4 files changed, 153 insertions(+), 76 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index 8d64993..bec43fe 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -41,9 +41,7 @@ def wait_for_cacert(model_name=None): :type model_name: str """ logging.info("Waiting for cacert") - cert_file = openstack_utils.get_cert_file_name( - 'keystone') - zaza.model.block_until_file_has_contents( + zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', cert_file, 'CERTIFICATE', diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index d605063..ad1d535 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -222,9 +222,7 @@ def validate_ca(cacertificate, application="keystone", port=5000): :returns: None :rtype: None """ - cert_file = zaza.openstack.utilities.openstack.get_cert_file_name( - application) - zaza.model.block_until_file_has_contents( + zaza.openstack.utilities.openstack.block_until_ca_exists( application, cert_file, cacertificate.decode().strip()) @@ -238,3 +236,6 @@ def validate_ca(cacertificate, application="keystone", port=5000): fp.write(cacertificate.decode()) fp.flush() requests.get('https://{}:{}'.format(ip, str(port)), verify=fp.name) + +def get_cert(): + print(zaza.openstack.utilities.openstack.get_remote_ca_cert_file('masakari')) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index e96f36a..141a1e6 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -154,9 +154,7 @@ class VaultTest(BaseVaultTest): test_config = lifecycle_utils.get_charm_config() del test_config['target_deploy_status']['vault'] - cert_file = zaza.openstack.utilities.openstack.get_cert_file_name( - 'keystone') - zaza.model.block_until_file_has_contents( + zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', cert_file, cacert.decode().strip()) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index b914c0f..7d2fc9d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -27,6 +27,7 @@ import logging import os import paramiko import re +import shutil import six import subprocess import sys @@ -182,7 +183,10 @@ WORKLOAD_STATUS_EXCEPTIONS = { 'ceilometer and gnocchi')}} # For vault TLS certificates +CACERT_FILENAME_FORMAT = "{}_juju_ca_cert.crt" +CERT_PROVIDORS = ['vault'] LOCAL_CERT_DIR = "tests" +REMOTE_CERT_DIR = "/usr/local/share/ca-certificates" KEYSTONE_CACERT = "keystone_juju_ca_cert.crt" KEYSTONE_REMOTE_CACERT = ( "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT)) @@ -197,53 +201,91 @@ REMOTE_CERTIFICATES = [VAULT_REMOTE_CACERT, KEYSTONE_REMOTE_CACERT] LOCAL_CERTIFICATES = [VAULT_LOCAL_CACERT, KEYSTONE_LOCAL_CACERT] -async def async_get_cert_file_name(app, cert_files=None, block=True, - model_name=None, timeout=2700): - """Get the name of the CA cert file thats on all units of an application. - - :param app: Name of application - :type capp: str - :param cert_files: List of cert files to search for. - :type cert_files: List[str] - :param block: Whether to block until a consistent cert file is found. - :type block: bool - :param model_name: Name of model to run check in - :type model_name: str - :param timeout: Time to wait for consistent file - :type timeout: int - :returns: Credentials dictionary - :rtype: dict - """ - async def _check_for_file(model, cert_files): - units = model.applications[app].units - results = {u.entity_id: [] for u in units} - for unit in units: - try: - for cf in cert_files: - output = await unit.run('test -e "{}"; echo $?'.format(cf)) - contents = output.data.get('results')['Stdout'] - if "0" in contents: - results[unit.entity_id].append(cf) - except JujuError: - pass - for cert_file in cert_files: - # Check that the certificate file exists on all the units. - if all(cert_file in files for files in results.values()): - return cert_file +async def async_block_until_ca_exists(application_name, ca_cert, model_name=None, timeout=2700): + async def _check_ca_present(model, ca_files): + units = model.applications[application_name].units + print(ca_files) + for ca_file in ca_files: + for unit in units: + print(unit) + print(ca_file) + try: + output = await unit.run('cat {}'.format(ca_file)) + contents = output.data.get('results').get('Stdout', '') + if not ca_cert in contents: + print("It's not here!") + print(ca_cert) + print(contents) + break + if ca_cert in contents: + print("It's here!") + # libjuju throws a generic error for connection failure. So we + # cannot differentiate between a connectivity issue and a + # target file not existing error. For now just assume the + # latter. + except JujuError: + continue + else: + return True else: - return None - - if not cert_files: - cert_files = REMOTE_CERTIFICATES - cert_file = None + return False + ca_files = await _async_get_remote_ca_cert_file_candidates(application_name, model_name=model_name) + print(ca_files) async with zaza.model.run_in_model(model_name) as model: - if block: - await zaza.model.async_block_until( - lambda: _check_for_file(model, cert_files), timeout=timeout) - cert_file = await _check_for_file(model, cert_files) - return cert_file + await zaza.model.async_block_until( + lambda: _check_ca_present(model, ca_files), timeout=timeout) -get_cert_file_name = zaza.model.sync_wrapper(async_get_cert_file_name) +block_until_ca_exists = zaza.model.sync_wrapper(async_block_until_ca_exists) + + + +#async def async_get_cert_file_name(app, cert_files=None, block=True, +# model_name=None, timeout=2700): +# """Get the name of the CA cert file thats on all units of an application. +# +# :param app: Name of application +# :type capp: str +# :param cert_files: List of cert files to search for. +# :type cert_files: List[str] +# :param block: Whether to block until a consistent cert file is found. +# :type block: bool +# :param model_name: Name of model to run check in +# :type model_name: str +# :param timeout: Time to wait for consistent file +# :type timeout: int +# :returns: Credentials dictionary +# :rtype: dict +# """ +# async def _check_for_file(model, cert_files): +# units = model.applications[app].units +# results = {u.entity_id: [] for u in units} +# for unit in units: +# try: +# for cf in cert_files: +# output = await unit.run('test -e "{}"; echo $?'.format(cf)) +# contents = output.data.get('results')['Stdout'] +# if "0" in contents: +# results[unit.entity_id].append(cf) +# except JujuError: +# pass +# for cert_file in cert_files: +# # Check that the certificate file exists on all the units. +# if all(cert_file in files for files in results.values()): +# return cert_file +# else: +# return None +# +# if not cert_files: +# cert_files = REMOTE_CERTIFICATES +# cert_file = None +# async with zaza.model.run_in_model(model_name) as model: +# if block: +# await zaza.model.async_block_until( +# lambda: _check_for_file(model, cert_files), timeout=timeout) +# cert_file = await _check_for_file(model, cert_files) +# return cert_file +# +#get_cert_file_name = zaza.model.sync_wrapper(async_get_cert_file_name) def get_cacert(): @@ -2010,32 +2052,70 @@ def get_overcloud_auth(address=None, model_name=None): 'OS_PROJECT_DOMAIN_NAME': 'admin_domain', 'API_VERSION': 3, } - if tls_rid: - cert_file = get_cert_file_name('keystone', model_name=model_name) - unit = model.get_first_unit_name('keystone', model_name=model_name) - - # ensure that the path to put the local cacert in actually exists. The - # assumption that 'tests/' exists for, say, mojo is false. - # Needed due to: - # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 - _dir = os.path.dirname(cert_file) - if not os.path.exists(_dir): - os.makedirs(_dir) - - _local_cert_file = "{}/{}".format( - LOCAL_CERT_DIR, - os.path.basename(cert_file)) - model.scp_from_unit( - unit, - cert_file, - _local_cert_file) - - if os.path.exists(_local_cert_file): - os.chmod(_local_cert_file, 0o644) - auth_settings['OS_CACERT'] = _local_cert_file + local_ca_cert = get_remote_ca_cert_file('keystone', model_name=model_name) + if ca_cert: + auth_settings['OS_CACERT'] = local_ca_cert return auth_settings +async def _async_get_remote_ca_cert_file_candidates(application, model_name=None): + cert_files = [] + # unit = model.get_first_unit_name(application, model_name=model_name) + units = await model.async_get_units(application, model_name=model_name) + unit = units[0].name + for _providor in CERT_PROVIDORS: + tls_rid = await model.async_get_relation_id( + application, + _providor, + model_name=model_name, + remote_interface_name='certificates') + if tls_rid: + cert_files.append(REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format(_providor)) + cert_files.append(REMOTE_CERT_DIR + '/' + KEYSTONE_CACERT) + return cert_files + +_get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper(_async_get_remote_ca_cert_file_candidates) + +def get_remote_ca_cert_file(application, model_name=None): +# CACERT_FILENAME = "{}_juju_ca_cert.crt" +# cert_files = [] +# unit = model.get_first_unit_name(application, model_name=model_name) +# for _providor in CERT_PROVIDORS: +# tls_rid = model.get_relation_id( +# application, +# _providor, +# model_name=model_name, +# remote_interface_name='certificates') +# if tls_rid: +# cert_files.append(CACERT_FILENAME.format(_providor)) +# cert_files.append(KEYSTONE_CACERT) + unit = model.get_first_unit_name(application, model_name=model_name) + local_cert_file = None + cert_files = _get_remote_ca_cert_file_candidates(application, model_name=model_name) + for cert_file in cert_files: + _local_cert_file = "{}/{}".format( + LOCAL_CERT_DIR, + os.path.basename(cert_file)) + with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca_file: + try: + model.scp_from_unit( + unit, + cert_file, + _tmp_ca_file.name) + except JujuError: + continue + # ensure that the path to put the local cacert in actually exists. The + # assumption that 'tests/' exists for, say, mojo is false. + # Needed due to: + # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 + _dir = os.path.dirname(_local_cert_file) + if not os.path.exists(_dir): + os.makedirs(_dir) + shutil.move(_tmp_ca_file.name, _local_cert_file) + os.chmod(_local_cert_file, 0o644) + local_cert_file = _local_cert_file + break + return local_cert_file def get_urllib_opener(): """Create a urllib opener taking into account proxy settings. From 24fbc068c94f99a702f7b3f8a2a01b7de0bf855d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 24 Jan 2021 15:27:06 +0000 Subject: [PATCH 067/180] Correct args to block_until_ca_exists --- zaza/openstack/charm_tests/keystone/setup.py | 1 - zaza/openstack/charm_tests/vault/setup.py | 1 - zaza/openstack/charm_tests/vault/tests.py | 1 - 3 files changed, 3 deletions(-) diff --git a/zaza/openstack/charm_tests/keystone/setup.py b/zaza/openstack/charm_tests/keystone/setup.py index bec43fe..73264cd 100644 --- a/zaza/openstack/charm_tests/keystone/setup.py +++ b/zaza/openstack/charm_tests/keystone/setup.py @@ -43,7 +43,6 @@ def wait_for_cacert(model_name=None): logging.info("Waiting for cacert") zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', - cert_file, 'CERTIFICATE', model_name=model_name) zaza.model.block_until_all_units_idle(model_name=model_name) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index ad1d535..25389d5 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -224,7 +224,6 @@ def validate_ca(cacertificate, application="keystone", port=5000): """ zaza.openstack.utilities.openstack.block_until_ca_exists( application, - cert_file, cacertificate.decode().strip()) vip = (zaza.model.get_application_config(application) .get("vip").get("value")) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 141a1e6..68130a0 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -156,7 +156,6 @@ class VaultTest(BaseVaultTest): del test_config['target_deploy_status']['vault'] zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', - cert_file, cacert.decode().strip()) zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) From d637646a9e139245029a4506afc2d0b71f2060bc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 24 Jan 2021 16:19:45 +0000 Subject: [PATCH 068/180] Fix typo --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 7d2fc9d..5fb4980 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2053,7 +2053,7 @@ def get_overcloud_auth(address=None, model_name=None): 'API_VERSION': 3, } local_ca_cert = get_remote_ca_cert_file('keystone', model_name=model_name) - if ca_cert: + if local_ca_cert: auth_settings['OS_CACERT'] = local_ca_cert return auth_settings From 401829f0a7b657f1eb9b3d129844476ce869839d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Sun, 24 Jan 2021 17:24:20 +0000 Subject: [PATCH 069/180] Code tidy and docstrings --- zaza/openstack/charm_tests/vault/setup.py | 3 - zaza/openstack/utilities/openstack.py | 152 ++++++++-------------- 2 files changed, 53 insertions(+), 102 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/setup.py b/zaza/openstack/charm_tests/vault/setup.py index 25389d5..c792508 100644 --- a/zaza/openstack/charm_tests/vault/setup.py +++ b/zaza/openstack/charm_tests/vault/setup.py @@ -235,6 +235,3 @@ def validate_ca(cacertificate, application="keystone", port=5000): fp.write(cacertificate.decode()) fp.flush() requests.get('https://{}:{}'.format(ip, str(port)), verify=fp.name) - -def get_cert(): - print(zaza.openstack.utilities.openstack.get_remote_ca_cert_file('masakari')) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 5fb4980..a975206 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -184,7 +184,7 @@ WORKLOAD_STATUS_EXCEPTIONS = { # For vault TLS certificates CACERT_FILENAME_FORMAT = "{}_juju_ca_cert.crt" -CERT_PROVIDORS = ['vault'] +CERT_PROVIDERS = ['vault'] LOCAL_CERT_DIR = "tests" REMOTE_CERT_DIR = "/usr/local/share/ca-certificates" KEYSTONE_CACERT = "keystone_juju_ca_cert.crt" @@ -192,33 +192,29 @@ KEYSTONE_REMOTE_CACERT = ( "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT)) KEYSTONE_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, KEYSTONE_CACERT)) -VAULT_CACERT = "vault_juju_ca_cert.crt" -VAULT_REMOTE_CACERT = ( - "/usr/local/share/ca-certificates/{}".format(VAULT_CACERT)) -VAULT_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, VAULT_CACERT)) -REMOTE_CERTIFICATES = [VAULT_REMOTE_CACERT, KEYSTONE_REMOTE_CACERT] -LOCAL_CERTIFICATES = [VAULT_LOCAL_CACERT, KEYSTONE_LOCAL_CACERT] +async def async_block_until_ca_exists(application_name, ca_cert, + model_name=None, timeout=2700): + """Block until a CA cert is on all units of application_name. - -async def async_block_until_ca_exists(application_name, ca_cert, model_name=None, timeout=2700): + :param application_name: Name of application to check + :type application_name: str + :param ca_cert: The certificate to look for. + :type ca_cert: str + :param model_name: Name of model to query. + :type model_name: str + :param timeout: How long in seconds to wait + :type timeout: int + """ async def _check_ca_present(model, ca_files): units = model.applications[application_name].units - print(ca_files) for ca_file in ca_files: for unit in units: - print(unit) - print(ca_file) try: output = await unit.run('cat {}'.format(ca_file)) contents = output.data.get('results').get('Stdout', '') - if not ca_cert in contents: - print("It's not here!") - print(ca_cert) - print(contents) + if ca_cert not in contents: break - if ca_cert in contents: - print("It's here!") # libjuju throws a generic error for connection failure. So we # cannot differentiate between a connectivity issue and a # target file not existing error. For now just assume the @@ -229,8 +225,9 @@ async def async_block_until_ca_exists(application_name, ca_cert, model_name=None return True else: return False - ca_files = await _async_get_remote_ca_cert_file_candidates(application_name, model_name=model_name) - print(ca_files) + ca_files = await _async_get_remote_ca_cert_file_candidates( + application_name, + model_name=model_name) async with zaza.model.run_in_model(model_name) as model: await zaza.model.async_block_until( lambda: _check_ca_present(model, ca_files), timeout=timeout) @@ -238,65 +235,19 @@ async def async_block_until_ca_exists(application_name, ca_cert, model_name=None block_until_ca_exists = zaza.model.sync_wrapper(async_block_until_ca_exists) - -#async def async_get_cert_file_name(app, cert_files=None, block=True, -# model_name=None, timeout=2700): -# """Get the name of the CA cert file thats on all units of an application. -# -# :param app: Name of application -# :type capp: str -# :param cert_files: List of cert files to search for. -# :type cert_files: List[str] -# :param block: Whether to block until a consistent cert file is found. -# :type block: bool -# :param model_name: Name of model to run check in -# :type model_name: str -# :param timeout: Time to wait for consistent file -# :type timeout: int -# :returns: Credentials dictionary -# :rtype: dict -# """ -# async def _check_for_file(model, cert_files): -# units = model.applications[app].units -# results = {u.entity_id: [] for u in units} -# for unit in units: -# try: -# for cf in cert_files: -# output = await unit.run('test -e "{}"; echo $?'.format(cf)) -# contents = output.data.get('results')['Stdout'] -# if "0" in contents: -# results[unit.entity_id].append(cf) -# except JujuError: -# pass -# for cert_file in cert_files: -# # Check that the certificate file exists on all the units. -# if all(cert_file in files for files in results.values()): -# return cert_file -# else: -# return None -# -# if not cert_files: -# cert_files = REMOTE_CERTIFICATES -# cert_file = None -# async with zaza.model.run_in_model(model_name) as model: -# if block: -# await zaza.model.async_block_until( -# lambda: _check_for_file(model, cert_files), timeout=timeout) -# cert_file = await _check_for_file(model, cert_files) -# return cert_file -# -#get_cert_file_name = zaza.model.sync_wrapper(async_get_cert_file_name) - - def get_cacert(): """Return path to CA Certificate bundle for verification during test. :returns: Path to CA Certificate bundle or None. - :rtype: Optional[str] + :rtype: Union[str, None] """ - for _cert in LOCAL_CERTIFICATES: + for _provider in CERT_PROVIDERS: + _cert = LOCAL_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format( + _provider) if os.path.exists(_cert): return _cert + if os.path.exists(KEYSTONE_LOCAL_CACERT): + return KEYSTONE_LOCAL_CACERT # OpenStack Client helpers @@ -2058,65 +2009,68 @@ def get_overcloud_auth(address=None, model_name=None): return auth_settings -async def _async_get_remote_ca_cert_file_candidates(application, model_name=None): + +async def _async_get_remote_ca_cert_file_candidates(application, + model_name=None): cert_files = [] - # unit = model.get_first_unit_name(application, model_name=model_name) - units = await model.async_get_units(application, model_name=model_name) - unit = units[0].name - for _providor in CERT_PROVIDORS: + for _provider in CERT_PROVIDERS: tls_rid = await model.async_get_relation_id( application, - _providor, + _provider, model_name=model_name, remote_interface_name='certificates') if tls_rid: - cert_files.append(REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format(_providor)) - cert_files.append(REMOTE_CERT_DIR + '/' + KEYSTONE_CACERT) + cert_files.append( + REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format( + _provider)) + cert_files.append(KEYSTONE_LOCAL_CACERT) return cert_files -_get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper(_async_get_remote_ca_cert_file_candidates) +_get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper( + _async_get_remote_ca_cert_file_candidates) + def get_remote_ca_cert_file(application, model_name=None): -# CACERT_FILENAME = "{}_juju_ca_cert.crt" -# cert_files = [] -# unit = model.get_first_unit_name(application, model_name=model_name) -# for _providor in CERT_PROVIDORS: -# tls_rid = model.get_relation_id( -# application, -# _providor, -# model_name=model_name, -# remote_interface_name='certificates') -# if tls_rid: -# cert_files.append(CACERT_FILENAME.format(_providor)) -# cert_files.append(KEYSTONE_CACERT) + """Collect CA certificate from application. + + :param application: Name of application to collect file from. + :type application: str + :param model_name: Name of model to query. + :type model_name: str + :returns: Path to cafile + :rtype: str + """ unit = model.get_first_unit_name(application, model_name=model_name) local_cert_file = None - cert_files = _get_remote_ca_cert_file_candidates(application, model_name=model_name) + cert_files = _get_remote_ca_cert_file_candidates( + application, + model_name=model_name) for cert_file in cert_files: _local_cert_file = "{}/{}".format( LOCAL_CERT_DIR, os.path.basename(cert_file)) - with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca_file: + with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca: try: model.scp_from_unit( unit, cert_file, - _tmp_ca_file.name) + _tmp_ca.name) except JujuError: continue - # ensure that the path to put the local cacert in actually exists. The - # assumption that 'tests/' exists for, say, mojo is false. + # ensure that the path to put the local cacert in actually exists. + # The assumption that 'tests/' exists for, say, mojo is false. # Needed due to: # commit: 537473ad3addeaa3d1e4e2d0fd556aeaa4018eb2 _dir = os.path.dirname(_local_cert_file) if not os.path.exists(_dir): os.makedirs(_dir) - shutil.move(_tmp_ca_file.name, _local_cert_file) + shutil.move(_tmp_ca.name, _local_cert_file) os.chmod(_local_cert_file, 0o644) local_cert_file = _local_cert_file break return local_cert_file + def get_urllib_opener(): """Create a urllib opener taking into account proxy settings. From e047150f5bae0b8a996cb57542491131348b92f1 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 25 Jan 2021 09:21:00 +0000 Subject: [PATCH 070/180] Add unit tests --- .../test_zaza_utilities_openstack.py | 157 +++++++++++++++++- ..._zaza_utilities_parallel_series_upgrade.py | 24 +-- unit_tests/utils.py | 22 +++ zaza/openstack/utilities/openstack.py | 12 +- 4 files changed, 190 insertions(+), 25 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 7e0e8f1..e4abfb1 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -17,6 +17,8 @@ import datetime import io import mock import subprocess +import sys +import unittest import tenacity import unit_tests.utils as ut_utils @@ -191,6 +193,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.patch_object(openstack_utils, 'get_application_config_option') self.patch_object(openstack_utils, 'get_keystone_ip') self.patch_object(openstack_utils, "get_current_os_versions") + self.patch_object(openstack_utils, "get_remote_ca_cert_file") self.patch_object(openstack_utils.juju_utils, 'leader_get') if tls_relation: self.patch_object(openstack_utils.model, "scp_from_unit") @@ -204,6 +207,7 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.get_relation_id.return_value = None self.get_application_config_option.return_value = None self.leader_get.return_value = 'openstack' + self.get_remote_ca_cert_file.return_value = None if tls_relation or ssl_cert: port = 35357 transport = 'https' @@ -245,7 +249,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'API_VERSION': 3, } if tls_relation: - expect['OS_CACERT'] = openstack_utils.KEYSTONE_LOCAL_CACERT + self.get_remote_ca_cert_file.return_value = '/tmp/a.cert' + expect['OS_CACERT'] = '/tmp/a.cert' self.assertEqual(openstack_utils.get_overcloud_auth(), expect) @@ -1327,3 +1332,153 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): mock.ANY, 'bridge-interface-mappings', {'ovn-bridge-mappings': 'physnet1:br-ex'})) + + def test_get_cacert(self): + self.patch_object(openstack_utils.os.path, 'exists') + results = { + 'tests/vault_juju_ca_cert.crt': True} + self.exists.side_effect = lambda x: results[x] + self.assertEqual( + openstack_utils.get_cacert(), + 'tests/vault_juju_ca_cert.crt') + + results = { + 'tests/vault_juju_ca_cert.crt': False, + 'tests/keystone_juju_ca_cert.crt': True} + self.assertEqual( + openstack_utils.get_cacert(), + 'tests/keystone_juju_ca_cert.crt') + + results = { + 'tests/vault_juju_ca_cert.crt': False, + 'tests/keystone_juju_ca_cert.crt': False} + self.assertIsNone(openstack_utils.get_cacert()) + + def test_get_remote_ca_cert_file(self): + self.patch_object(openstack_utils.model, 'get_first_unit_name') + self.patch_object( + openstack_utils, + '_get_remote_ca_cert_file_candidates') + self.patch_object(openstack_utils.model, 'scp_from_unit') + self.patch_object(openstack_utils.os.path, 'exists') + self.patch_object(openstack_utils.shutil, 'move') + self.patch_object(openstack_utils.os, 'chmod') + self.patch_object(openstack_utils.tempfile, 'NamedTemporaryFile') + enter_mock = mock.MagicMock() + enter_mock.__enter__.return_value.name = 'tempfilename' + self.NamedTemporaryFile.return_value = enter_mock + self.get_first_unit_name.return_value = 'neutron-api/0' + self._get_remote_ca_cert_file_candidates.return_value = [ + '/tmp/ca1.cert'] + self.exists.return_value = True + + openstack_utils.get_remote_ca_cert_file('neutron-api') + self.scp_from_unit.assert_called_once_with( + 'neutron-api/0', + '/tmp/ca1.cert', + 'tempfilename') + self.chmod.assert_called_once_with('tests/ca1.cert', 0o644) + self.move.assert_called_once_with('tempfilename', 'tests/ca1.cert') + + +class TestAsyncOpenstackUtils(ut_utils.AioTestCase): + + def setUp(self): + super(TestAsyncOpenstackUtils, self).setUp() + if sys.version_info < (3, 6, 0): + raise unittest.SkipTest("Can't AsyncMock in py35") + model_mock = mock.MagicMock() + test_mock = mock.MagicMock() + + class AsyncContextManagerMock(test_mock): + async def __aenter__(self): + yield model_mock + + async def __aexit__(self, *args): + pass + + self.model_mock = model_mock + self.patch_object(openstack_utils.zaza.model, "async_block_until") + + async def _block_until(f, timeout): + # Store the result of the call to _check_ca_present to validate + # tests + self.result = await f() + self.async_block_until.side_effect = _block_until + self.patch('zaza.model.run_in_model', name='_run_in_model') + self._run_in_model.return_value = AsyncContextManagerMock + self._run_in_model().__aenter__.return_value = self.model_mock + + async def test_async_block_until_ca_exists(self): + def _get_action_output(stdout, code, stderr=None): + stderr = stderr or '' + action = mock.MagicMock() + action.data = { + 'results': { + 'Code': code, + 'Stderr': stderr, + 'Stdout': stdout}} + return action + results = { + '/tmp/missing.cert': _get_action_output( + '', + '1', + 'cat: /tmp/missing.cert: No such file or directory'), + '/tmp/good.cert': _get_action_output('CERTIFICATE', '0')} + + async def _run(command, timeout=None): + return results[command.split()[-1]] + self.unit1 = mock.MagicMock() + self.unit2 = mock.MagicMock() + self.unit2.run.side_effect = _run + self.unit1.run.side_effect = _run + self.units = [self.unit1, self.unit2] + _units = mock.MagicMock() + _units.units = self.units + self.model_mock.applications = { + 'keystone': _units + } + self.patch_object( + openstack_utils, + "_async_get_remote_ca_cert_file_candidates") + + # Test a missing cert then a good cert. + self._async_get_remote_ca_cert_file_candidates.return_value = [ + '/tmp/missing.cert', + '/tmp/good.cert'] + await openstack_utils.async_block_until_ca_exists( + 'keystone', + 'CERTIFICATE') + self.assertTrue(self.result) + + # Test a single missing + self._async_get_remote_ca_cert_file_candidates.return_value = [ + '/tmp/missing.cert'] + await openstack_utils.async_block_until_ca_exists( + 'keystone', + 'CERTIFICATE') + self.assertFalse(self.result) + + async def test__async_get_remote_ca_cert_file_candidates(self): + self.patch_object(openstack_utils.zaza.model, "async_get_relation_id") + rel_id_out = { + } + + def _get_relation_id(app, cert_app, model_name, remote_interface_name): + return rel_id_out[cert_app] + self.async_get_relation_id.side_effect = _get_relation_id + + rel_id_out['vault'] = 'certs:1' + r = await openstack_utils._async_get_remote_ca_cert_file_candidates( + 'neutron-api', 'mymodel') + self.assertEqual( + r, + ['/usr/local/share/ca-certificates/vault_juju_ca_cert.crt', + '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt']) + + rel_id_out['vault'] = None + r = await openstack_utils._async_get_remote_ca_cert_file_candidates( + 'neutron-api', 'mymodel') + self.assertEqual( + r, + ['/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt']) diff --git a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py index 0231dbf..70ffd3b 100644 --- a/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py +++ b/unit_tests/utilities/test_zaza_utilities_parallel_series_upgrade.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio import mock import sys import unittest @@ -139,28 +138,7 @@ class Test_ParallelSeriesUpgradeSync(ut_utils.BaseTestCase): self.assertEqual(expected, config) -class AioTestCase(ut_utils.BaseTestCase): - def __init__(self, methodName='runTest', loop=None): - self.loop = loop or asyncio.get_event_loop() - self._function_cache = {} - super(AioTestCase, self).__init__(methodName=methodName) - - def coroutine_function_decorator(self, func): - def wrapper(*args, **kw): - return self.loop.run_until_complete(func(*args, **kw)) - return wrapper - - def __getattribute__(self, item): - attr = object.__getattribute__(self, item) - if asyncio.iscoroutinefunction(attr) and item.startswith('test_'): - if item not in self._function_cache: - self._function_cache[item] = ( - self.coroutine_function_decorator(attr)) - return self._function_cache[item] - return attr - - -class TestParallelSeriesUpgrade(AioTestCase): +class TestParallelSeriesUpgrade(ut_utils.AioTestCase): def setUp(self): super(TestParallelSeriesUpgrade, self).setUp() if sys.version_info < (3, 6, 0): diff --git a/unit_tests/utils.py b/unit_tests/utils.py index 4694d0d..8e31f45 100644 --- a/unit_tests/utils.py +++ b/unit_tests/utils.py @@ -19,6 +19,7 @@ """Module to provide helper for writing unit tests.""" +import asyncio import contextlib import io import mock @@ -96,3 +97,24 @@ class BaseTestCase(unittest.TestCase): started.return_value = return_value self._patches_start[name] = started setattr(self, name, started) + + +class AioTestCase(BaseTestCase): + def __init__(self, methodName='runTest', loop=None): + self.loop = loop or asyncio.get_event_loop() + self._function_cache = {} + super(AioTestCase, self).__init__(methodName=methodName) + + def coroutine_function_decorator(self, func): + def wrapper(*args, **kw): + return self.loop.run_until_complete(func(*args, **kw)) + return wrapper + + def __getattribute__(self, item): + attr = object.__getattribute__(self, item) + if asyncio.iscoroutinefunction(attr) and item.startswith('test_'): + if item not in self._function_cache: + self._function_cache[item] = ( + self.coroutine_function_decorator(attr)) + return self._function_cache[item] + return attr diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index a975206..d883e0a 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -222,6 +222,7 @@ async def async_block_until_ca_exists(application_name, ca_cert, except JujuError: continue else: + # The CA was found in `ca_file` on all units. return True else: return False @@ -2012,6 +2013,15 @@ def get_overcloud_auth(address=None, model_name=None): async def _async_get_remote_ca_cert_file_candidates(application, model_name=None): + """Return a list of possible remote CA file names. + + :param application: Name of application to examine. + :type application: str + :param model_name: Name of model to query. + :type model_name: str + :returns: List of paths to possible ca files. + :rtype: List[str] + """ cert_files = [] for _provider in CERT_PROVIDERS: tls_rid = await model.async_get_relation_id( @@ -2023,7 +2033,7 @@ async def _async_get_remote_ca_cert_file_candidates(application, cert_files.append( REMOTE_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format( _provider)) - cert_files.append(KEYSTONE_LOCAL_CACERT) + cert_files.append(KEYSTONE_REMOTE_CACERT) return cert_files _get_remote_ca_cert_file_candidates = zaza.model.sync_wrapper( From 5be8fc377179e8fce9de8e8e7d9744fc169d65a4 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 25 Jan 2021 11:41:39 +0000 Subject: [PATCH 071/180] Fix docstring and bug --- zaza/openstack/utilities/openstack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index d883e0a..522df31 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -199,7 +199,7 @@ async def async_block_until_ca_exists(application_name, ca_cert, :param application_name: Name of application to check :type application_name: str - :param ca_cert: The certificate to look for. + :param ca_cert: The certificate content. :type ca_cert: str :param model_name: Name of model to query. :type model_name: str @@ -220,7 +220,7 @@ async def async_block_until_ca_exists(application_name, ca_cert, # target file not existing error. For now just assume the # latter. except JujuError: - continue + break else: # The CA was found in `ca_file` on all units. return True From 93a9aff92796ce626ae969fc0554f3411dc3adf2 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 25 Jan 2021 11:47:42 +0000 Subject: [PATCH 072/180] Fix context manager mock --- unit_tests/utilities/test_zaza_utilities_openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index e4abfb1..67cd417 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1392,7 +1392,7 @@ class TestAsyncOpenstackUtils(ut_utils.AioTestCase): class AsyncContextManagerMock(test_mock): async def __aenter__(self): - yield model_mock + return self async def __aexit__(self, *args): pass From dc886f65d5a6fa5093a8cfdb9a0f293e6154a82f Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 28 Jan 2021 08:49:11 +0100 Subject: [PATCH 073/180] Add workaround for OVS-OVN migration on Groovy The root of the issue is in Open vSwitch itself and it is not easilly workaroundable in the charms. We'll pursue a upstream/package level fix. --- zaza/openstack/charm_tests/ovn/tests.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index c5eae5e..9bea951 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -143,6 +143,13 @@ class ChassisCharmOperationTest(BaseCharmOperationTest): class OVSOVNMigrationTest(test_utils.BaseCharmTest): """OVS to OVN migration tests.""" + @classmethod + def setUpClass(cls): + """Run class setup for OVN migration tests.""" + super(OVSOVNMigrationTest, cls).setUpClass() + cls.current_release = openstack_utils.get_os_release( + openstack_utils.get_current_os_release_pair()) + def setUp(self): """Perform migration steps prior to validation.""" super(OVSOVNMigrationTest, self).setUp() @@ -410,3 +417,17 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): zaza.model.wait_for_agent_status() zaza.model.wait_for_application_states( states=self.target_deploy_status) + # Workaround for our old friend LP: #1852221 which hit us again on + # Groovy. We make the os_release check explicit so that we can + # re-evaluate the need for the workaround at the next release. + if self.current_release == openstack_utils.get_os_release( + 'groovy_victoria'): + try: + for application in ('ovn-chassis', 'ovn-dedicated-chassis'): + for unit in zaza.model.get_units(application): + zaza.model.run_on_unit( + unit.entity_id, + 'systemctl restart ovs-vswitchd') + except KeyError: + # One of the applications is not in the model, which is fine + pass From c4691ef1c74365bf2a33f4922f852247e205e8bb Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 29 Jan 2021 13:51:25 +0000 Subject: [PATCH 074/180] Skip OVN provider octavia test on Victoria The OVN provider octavia test on Victoria is currently broken due to Bug #1896603. Until it is fixed skip the test. --- zaza/openstack/charm_tests/octavia/tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index e95a33f..26941a2 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -178,6 +178,14 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): for provider in octavia_client.provider_list().get('providers', []) if provider['name'] != 'octavia' # alias for `amphora`, skip } + if (openstack_utils.get_os_release() in [ + openstack_utils.get_os_release('focal_victoria'), + openstack_utils.get_os_release('groovy_victoria')]): + logging.info("Skipping tests of ovn backed lb (Bug #1896603)") + try: + del providers['ovn'] + except KeyError: + pass return providers def _create_lb_resources(self, octavia_client, provider, vip_subnet_id, From ea03e36273ea616a14e53be6640339ba750597e3 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Mon, 1 Feb 2021 15:50:31 -0800 Subject: [PATCH 075/180] Replace get_relation_from_unit for designate test --- zaza/openstack/charm_tests/designate/tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/designate/tests.py b/zaza/openstack/charm_tests/designate/tests.py index 3c56ee1..78a5f31 100644 --- a/zaza/openstack/charm_tests/designate/tests.py +++ b/zaza/openstack/charm_tests/designate/tests.py @@ -23,7 +23,7 @@ import designateclient.v1.records as records import designateclient.v1.servers as servers import zaza.model -import zaza.openstack.utilities.juju as zaza_juju +import zaza.utilities.juju as juju_utils import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.designate.utils as designate_utils @@ -170,7 +170,7 @@ class DesignateAPITests(BaseDesignateTest): reraise=True ) def _wait_to_resolve_test_record(self): - dns_ip = zaza_juju.get_relation_from_unit( + dns_ip = juju_utils.get_relation_from_unit( 'designate/0', 'designate-bind/0', 'dns-backend' From 33ab875eb25668ce4c65ce548ef97aaf55947219 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Tue, 2 Feb 2021 09:58:13 -0800 Subject: [PATCH 076/180] Replace get_relation_from_unit for ceph test --- zaza/openstack/charm_tests/ceph/tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/tests.py b/zaza/openstack/charm_tests/ceph/tests.py index c7d44bd..27fefbd 100644 --- a/zaza/openstack/charm_tests/ceph/tests.py +++ b/zaza/openstack/charm_tests/ceph/tests.py @@ -33,7 +33,7 @@ import zaza.model as zaza_model import zaza.openstack.utilities.ceph as zaza_ceph import zaza.openstack.utilities.exceptions as zaza_exceptions import zaza.openstack.utilities.generic as zaza_utils -import zaza.openstack.utilities.juju as zaza_juju +import zaza.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as zaza_openstack @@ -124,7 +124,7 @@ class CephRelationTest(test_utils.OpenStackBaseTest): relation_name = 'osd' remote_unit = zaza_model.get_unit_from_name(remote_unit_name) remote_ip = remote_unit.public_address - relation = zaza_juju.get_relation_from_unit( + relation = juju_utils.get_relation_from_unit( unit_name, remote_unit_name, relation_name @@ -153,7 +153,7 @@ class CephRelationTest(test_utils.OpenStackBaseTest): 'ceph-public-address': remote_ip, 'fsid': fsid, } - relation = zaza_juju.get_relation_from_unit( + relation = juju_utils.get_relation_from_unit( unit_name, remote_unit_name, relation_name @@ -980,7 +980,7 @@ class BlueStoreCompressionCharmOperation(test_utils.BaseCharmTest): ceph_pools_detail = zaza_ceph.get_ceph_pool_details( model_name=self.model_name) logging.debug('AFTER: {}'.format(ceph_pools_detail)) - logging.debug(zaza_juju.get_relation_from_unit( + logging.debug(juju_utils.get_relation_from_unit( 'ceph-mon', self.application_name, None, model_name=self.model_name)) logging.info('Checking Ceph pool compression_mode after restoring ' From 7878523e3457116497cd48bdc7fa8b3761d0c621 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Wed, 3 Feb 2021 18:01:28 +0000 Subject: [PATCH 077/180] Rename zaza_juju to be consistent with all of repo --- unit_tests/utilities/test_zaza_utilities_ceph.py | 2 +- zaza/openstack/utilities/ceph.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_ceph.py b/unit_tests/utilities/test_zaza_utilities_ceph.py index ea718b7..e6ac337 100644 --- a/unit_tests/utilities/test_zaza_utilities_ceph.py +++ b/unit_tests/utilities/test_zaza_utilities_ceph.py @@ -118,7 +118,7 @@ class TestCephUtils(ut_utils.BaseTestCase): model_name='amodel') def test_pools_from_broker_req(self): - self.patch_object(ceph_utils.zaza_juju, 'get_relation_from_unit') + self.patch_object(ceph_utils.juju_utils, 'get_relation_from_unit') self.get_relation_from_unit.return_value = { 'broker_req': ( '{"api-version": 1, "ops": [' diff --git a/zaza/openstack/utilities/ceph.py b/zaza/openstack/utilities/ceph.py index 6613154..a01f56e 100644 --- a/zaza/openstack/utilities/ceph.py +++ b/zaza/openstack/utilities/ceph.py @@ -3,7 +3,7 @@ import json import logging import zaza.model as zaza_model -import zaza.utilities.juju as zaza_juju +import zaza.utilities.juju as juju_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -225,7 +225,7 @@ def get_pools_from_broker_req(application_or_unit, model_name=None): """ # NOTE: we do not pass on a name for the remote_interface_name as that # varies between the Ceph consuming applications. - relation_data = zaza_juju.get_relation_from_unit( + relation_data = juju_utils.get_relation_from_unit( 'ceph-mon', application_or_unit, None, model_name=model_name) # NOTE: we probably should consume the Ceph broker code from c-h but c-h is From 2ab9cebbf67c8e88aabbe09250bd1c27396c5613 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 4 Feb 2021 08:48:17 +0100 Subject: [PATCH 078/180] Use per-model tmp-dir to store local copy of CA cert (#493) The current approach of storing the deployment CA certificate in the 'test/' relative path does not allow for executing tests for multiple targets from the same environment. We have previously moved (7a90110) the local copy of the SSH private key for similar reasons. Remove the global constants as we cannot build them without making function calls, and we'd rather avoid doing that at module import time. Code using the location of the local CA certificate has already been changed to use helper functions. --- .../test_zaza_utilities_openstack.py | 30 +++++++++++++------ zaza/openstack/utilities/openstack.py | 26 +++++++++++----- 2 files changed, 39 insertions(+), 17 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index 67cd417..c42fc9d 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -1333,25 +1333,34 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'bridge-interface-mappings', {'ovn-bridge-mappings': 'physnet1:br-ex'})) + def test_get_cacert_absolute_path(self): + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir') + self.get_tmpdir.return_value = '/tmp/default' + self.assertEqual( + openstack_utils.get_cacert_absolute_path('filename'), + '/tmp/default/filename') + def test_get_cacert(self): + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir') + self.get_tmpdir.return_value = '/tmp/default' self.patch_object(openstack_utils.os.path, 'exists') results = { - 'tests/vault_juju_ca_cert.crt': True} + '/tmp/default/vault_juju_ca_cert.crt': True} self.exists.side_effect = lambda x: results[x] self.assertEqual( openstack_utils.get_cacert(), - 'tests/vault_juju_ca_cert.crt') + '/tmp/default/vault_juju_ca_cert.crt') results = { - 'tests/vault_juju_ca_cert.crt': False, - 'tests/keystone_juju_ca_cert.crt': True} + '/tmp/default/vault_juju_ca_cert.crt': False, + '/tmp/default/keystone_juju_ca_cert.crt': True} self.assertEqual( openstack_utils.get_cacert(), - 'tests/keystone_juju_ca_cert.crt') + '/tmp/default/keystone_juju_ca_cert.crt') results = { - 'tests/vault_juju_ca_cert.crt': False, - 'tests/keystone_juju_ca_cert.crt': False} + '/tmp/default/vault_juju_ca_cert.crt': False, + '/tmp/default/keystone_juju_ca_cert.crt': False} self.assertIsNone(openstack_utils.get_cacert()) def test_get_remote_ca_cert_file(self): @@ -1364,6 +1373,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.patch_object(openstack_utils.shutil, 'move') self.patch_object(openstack_utils.os, 'chmod') self.patch_object(openstack_utils.tempfile, 'NamedTemporaryFile') + self.patch_object(openstack_utils.deployment_env, 'get_tmpdir') + self.get_tmpdir.return_value = '/tmp/default' enter_mock = mock.MagicMock() enter_mock.__enter__.return_value.name = 'tempfilename' self.NamedTemporaryFile.return_value = enter_mock @@ -1377,8 +1388,9 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): 'neutron-api/0', '/tmp/ca1.cert', 'tempfilename') - self.chmod.assert_called_once_with('tests/ca1.cert', 0o644) - self.move.assert_called_once_with('tempfilename', 'tests/ca1.cert') + self.chmod.assert_called_once_with('/tmp/default/ca1.cert', 0o644) + self.move.assert_called_once_with( + 'tempfilename', '/tmp/default/ca1.cert') class TestAsyncOpenstackUtils(ut_utils.AioTestCase): diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 522df31..d277656 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -185,12 +185,10 @@ WORKLOAD_STATUS_EXCEPTIONS = { # For vault TLS certificates CACERT_FILENAME_FORMAT = "{}_juju_ca_cert.crt" CERT_PROVIDERS = ['vault'] -LOCAL_CERT_DIR = "tests" REMOTE_CERT_DIR = "/usr/local/share/ca-certificates" KEYSTONE_CACERT = "keystone_juju_ca_cert.crt" KEYSTONE_REMOTE_CACERT = ( "/usr/local/share/ca-certificates/{}".format(KEYSTONE_CACERT)) -KEYSTONE_LOCAL_CACERT = ("{}/{}".format(LOCAL_CERT_DIR, KEYSTONE_CACERT)) async def async_block_until_ca_exists(application_name, ca_cert, @@ -236,6 +234,18 @@ async def async_block_until_ca_exists(application_name, ca_cert, block_until_ca_exists = zaza.model.sync_wrapper(async_block_until_ca_exists) +def get_cacert_absolute_path(filename): + """Build string containing location of the CA Certificate file. + + :param filename: Expected filename for CA Certificate file. + :type filename: str + :returns: Absolute path to file containing CA Certificate + :rtype: str + """ + return os.path.join( + deployment_env.get_tmpdir(), filename) + + def get_cacert(): """Return path to CA Certificate bundle for verification during test. @@ -243,12 +253,13 @@ def get_cacert(): :rtype: Union[str, None] """ for _provider in CERT_PROVIDERS: - _cert = LOCAL_CERT_DIR + '/' + CACERT_FILENAME_FORMAT.format( - _provider) + _cert = get_cacert_absolute_path( + CACERT_FILENAME_FORMAT.format(_provider)) if os.path.exists(_cert): return _cert - if os.path.exists(KEYSTONE_LOCAL_CACERT): - return KEYSTONE_LOCAL_CACERT + _keystone_local_cacert = get_cacert_absolute_path(KEYSTONE_CACERT) + if os.path.exists(_keystone_local_cacert): + return _keystone_local_cacert # OpenStack Client helpers @@ -2056,8 +2067,7 @@ def get_remote_ca_cert_file(application, model_name=None): application, model_name=model_name) for cert_file in cert_files: - _local_cert_file = "{}/{}".format( - LOCAL_CERT_DIR, + _local_cert_file = get_cacert_absolute_path( os.path.basename(cert_file)) with tempfile.NamedTemporaryFile(mode="w", delete=False) as _tmp_ca: try: From 4cbf70dd5e931285973ab230e65ba3678fa87994 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 4 Feb 2021 08:56:22 +0100 Subject: [PATCH 079/180] octavia: Configure SSH key to allow debugging of Amphorae (#495) --- zaza/openstack/charm_tests/octavia/setup.py | 50 +++++++++++++++++---- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/setup.py b/zaza/openstack/charm_tests/octavia/setup.py index 677c368..4b09aa0 100644 --- a/zaza/openstack/charm_tests/octavia/setup.py +++ b/zaza/openstack/charm_tests/octavia/setup.py @@ -25,6 +25,9 @@ import zaza.openstack.charm_tests.glance.setup as glance_setup import zaza.openstack.utilities.openstack as openstack import zaza.openstack.configure.guest +import zaza.openstack.charm_tests.nova.setup as nova_setup +import zaza.openstack.charm_tests.nova.utils as nova_utils + def ensure_lts_images(): """Ensure that bionic and focal images are available for the tests.""" @@ -51,13 +54,32 @@ def add_amphora_image(image_url=None): def configure_octavia(): - """Do mandatory post deployment configuration of Octavia.""" - # Tell Octavia charm it is safe to create cloud resources - logging.info('Running `configure-resources` action on Octavia leader unit') - zaza.model.run_action_on_leader( - 'octavia', - 'configure-resources', - action_params={}) + """Do post deployment configuration and initialization of Octavia. + + Certificates for the private Octavia worker <-> Amphorae communication must + be generated and set trough charm configuration. + + The optional SSH configuration options are set to enable debug and log + collection from Amphorae, we will use the same keypair as Zaza uses for + instance creation. + + The `configure-resources` action must be run to have the charm create + in-cloud resources such as management network and associated ports and + security groups. + """ + # Set up Nova client to create/retrieve keypair for Amphora debug purposes. + # + # We reuse the Nova setup code for this and in most cases the test + # declaration will already defined that the Nova manage_ssh_key setup + # helper to run before we get here. Re-run here to make sure this setup + # function can be used separately, manage_ssh_key is idempotent. + keystone_session = openstack.get_overcloud_keystone_session() + nova_client = openstack.get_nova_session_client( + keystone_session) + nova_setup.manage_ssh_key(nova_client) + ssh_public_key = openstack.get_public_key( + nova_client, nova_utils.KEYPAIR_NAME) + # Generate certificates for controller/load balancer instance communication (issuing_cakey, issuing_cacert) = cert.generate_cert( 'OSCI Zaza Issuer', @@ -71,7 +93,7 @@ def configure_octavia(): issuer_name='OSCI Zaza Octavia Controller', signing_key=controller_cakey) controller_bundle = controller_cert + controller_key - cert_config = { + charm_config = { 'lb-mgmt-issuing-cacert': base64.b64encode( issuing_cacert).decode('utf-8'), 'lb-mgmt-issuing-ca-private-key': base64.b64encode( @@ -81,6 +103,9 @@ def configure_octavia(): controller_cacert).decode('utf-8'), 'lb-mgmt-controller-cert': base64.b64encode( controller_bundle).decode('utf-8'), + 'amp-ssh-key-name': 'octavia', + 'amp-ssh-pub-key': base64.b64encode( + bytes(ssh_public_key, 'utf-8')).decode('utf-8'), } logging.info('Configuring certificates for mandatory Octavia ' 'client/server authentication ' @@ -93,10 +118,17 @@ def configure_octavia(): _singleton = zaza.openstack.charm_tests.test_utils.OpenStackBaseTest() _singleton.setUpClass(application_name='octavia') - with _singleton.config_change(cert_config, cert_config): + with _singleton.config_change(charm_config, charm_config): # wait for configuration to be applied then return pass + # Tell Octavia charm it is safe to create cloud resources + logging.info('Running `configure-resources` action on Octavia leader unit') + zaza.model.run_action_on_leader( + 'octavia', + 'configure-resources', + action_params={}) + def centralized_fip_network(): """Create network with centralized router for connecting lb and fips. From 83e5dc798c6a1b3114b8526d7a3cc1a5aa8eb618 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Tue, 9 Feb 2021 19:47:40 -0800 Subject: [PATCH 080/180] Refactor security checklist test to another class This will allow security checks to be logically separate from other functional tests, and adhere to a similar design pattern that is used in other charm tests (see nova [0]) for an example. It will also highight which security checks are currently passing, and which are not. [0] https://github.com/openstack-charmers/zaza-openstack-tests/blob/35840a66d6788dc4899847747ee7523895f46e1b/zaza/openstack/charm_tests/nova/tests.py#L485-L529 Partial-Bug: #1883196 --- .../charm_tests/openstack_dashboard/tests.py | 53 +++++++++++++++---- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 9588c6f..5944964 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -407,17 +407,6 @@ class OpenStackDashboardTests(test_utils.OpenStackBaseTest, self.assertEqual(e.code, 404, msg) logging.info('OK') - def test_501_security_checklist_action(self): - """Verify expected result on a default install. - - Ported from amulet tests. - """ - logging.info("Testing security-checklist") - unit_name = zaza_model.get_lead_unit_name('openstack-dashboard') - action = zaza_model.run_action(unit_name, 'security-checklist') - assert action.data.get(u"status") == "failed", \ - "Security check is expected to not pass by default" - def test_900_restart_on_config_change(self): """Verify that the specified services are restarted on config changed. @@ -520,3 +509,45 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization, result = client.get(_url) if result.status_code == 403: raise policyd.PolicydOperationFailedException("Not authenticated") + + +class SecurityTests(test_utils.OpenStackBaseTest, + OpenStackDashboardBase): + """Openstack-dashboard security tests.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running openstack-dashboard SecurityTests.""" + super(SecurityTests, cls).setUpClass() + + def test_security_checklist(self): + """Verify expected state with security checklist.""" + logging.info("Testing security checklist.") + + expected_failures = [ + 'csrf_cookie_set', + 'disable_password_reveal', + 'disallow-iframe-embed', + 'password-validator-is-not-default', + 'securie_proxy_ssl_header_is_set', + 'session_cookie-httponly', + 'session-cookie-store', + ] + expected_passes = [ + 'disable_password_autocomplete', + 'enforce-password-check', + 'validate-file-ownership', + 'validate-file-permissions' + ] + + logging.info('Running `security-checklist` action' + ' on {} leader'.format(self.application_name)) + test_utils.audit_assertions( + zaza_model.run_action_on_leader( + self.application_name, + 'security-checklist', + model_name=self.model_name, + action_params={}), + expected_passes, + expected_failures, + expected_to_pass=False) From c327e4c3a294eaeda27c07bafb54ab5a7ba625a4 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 10 Feb 2021 10:22:13 +0100 Subject: [PATCH 081/180] Pin cryptography<3.4 Newer versions require a Rust compiler to build. --- requirements.txt | 5 +++++ setup.py | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cec0286..78a41f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,3 +44,8 @@ paramiko sphinx sphinxcontrib-asyncio git+https://github.com/openstack-charmers/zaza#egg=zaza + +# Newer versions require a Rust compiler to build, see +# * https://github.com/openstack-charmers/zaza/issues/421 +# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html +cryptography<3.4 diff --git a/setup.py b/setup.py index b18cd9a..fb4377d 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,12 @@ install_require = [ 'futurist<2.0.0', 'async_generator', 'boto3', - 'cryptography', + + # Newer versions require a Rust compiler to build, see + # * https://github.com/openstack-charmers/zaza/issues/421 + # * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html + 'cryptography<3.4', + 'dnspython', 'hvac<0.7.0', 'jinja2', From 5b9db73fc7c63979741a52fabff449499a477132 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Wed, 10 Feb 2021 06:40:09 -0800 Subject: [PATCH 082/180] Fix indentation for pep8 --- zaza/openstack/charm_tests/openstack_dashboard/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 5944964..5cd341c 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -512,7 +512,7 @@ class OpenStackDashboardPolicydTests(policyd.BasePolicydSpecialization, class SecurityTests(test_utils.OpenStackBaseTest, - OpenStackDashboardBase): + OpenStackDashboardBase): """Openstack-dashboard security tests.""" @classmethod @@ -541,9 +541,9 @@ class SecurityTests(test_utils.OpenStackBaseTest, ] logging.info('Running `security-checklist` action' - ' on {} leader'.format(self.application_name)) + ' on {} leader'.format(self.application_name)) test_utils.audit_assertions( - zaza_model.run_action_on_leader( + zaza_model.run_action_on_leader( self.application_name, 'security-checklist', model_name=self.model_name, From 1210e0c47e9dc18edca047a2c784ad9409c35589 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Wed, 10 Feb 2021 18:42:32 -0800 Subject: [PATCH 083/180] Skip security_checklist test for xenial_mitaka See bug 1915293 [0] for more details, but this test is already skipped in the repo's current state, this is just making it more explicit as part of the current refactor. [0] https://bugs.launchpad.net/charm-openstack-dashboard/+bug/1915293 --- zaza/openstack/charm_tests/openstack_dashboard/tests.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 5cd341c..36e7175 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -535,10 +535,15 @@ class SecurityTests(test_utils.OpenStackBaseTest, ] expected_passes = [ 'disable_password_autocomplete', - 'enforce-password-check', 'validate-file-ownership', 'validate-file-permissions' ] + if (openstack_utils.get_os_release() <= + openstack_utils.get_os_release('xenial_mitaka')): + logging.info("Test expected to fail until bug #1915293 is fixed") + expected_failures.append('enforce-password-check') + else: + expected_passes.append('enforce-password-check') logging.info('Running `security-checklist` action' ' on {} leader'.format(self.application_name)) From ce4fe120219a3d23e9a4d917d37df33bee7f6433 Mon Sep 17 00:00:00 2001 From: Garrett Thompson Date: Thu, 11 Feb 2021 12:43:04 -0800 Subject: [PATCH 084/180] Revert "Skip security_checklist test for xenial_mitaka" This reverts commit 1210e0c47e9dc18edca047a2c784ad9409c35589. Rather than using an if statement for gating xenial_mitaka and earlier, test cases can be customized in tests/tests.yaml for individual bundles. This will be handled in review 774305. [0] https://review.opendev.org/c/openstack/charm-openstack-dashboard/+/774305 --- zaza/openstack/charm_tests/openstack_dashboard/tests.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/openstack_dashboard/tests.py b/zaza/openstack/charm_tests/openstack_dashboard/tests.py index 36e7175..5cd341c 100644 --- a/zaza/openstack/charm_tests/openstack_dashboard/tests.py +++ b/zaza/openstack/charm_tests/openstack_dashboard/tests.py @@ -535,15 +535,10 @@ class SecurityTests(test_utils.OpenStackBaseTest, ] expected_passes = [ 'disable_password_autocomplete', + 'enforce-password-check', 'validate-file-ownership', 'validate-file-permissions' ] - if (openstack_utils.get_os_release() <= - openstack_utils.get_os_release('xenial_mitaka')): - logging.info("Test expected to fail until bug #1915293 is fixed") - expected_failures.append('enforce-password-check') - else: - expected_passes.append('enforce-password-check') logging.info('Running `security-checklist` action' ' on {} leader'.format(self.application_name)) From af2c6243e2c2ea5429f89cb4f36f3f2e216261de Mon Sep 17 00:00:00 2001 From: David Ames Date: Sat, 6 Feb 2021 00:31:28 +0000 Subject: [PATCH 085/180] Fix race condition in MySQL tests In the MySQL test we destroy a unit and then attempt to remove it from metadata. The metadata removal (remove_instance action) gets stuck when it is started while the destroyed unit has not been entirely removed and is still running MySQL. Wait for all the units to have cluster incomplete workload status message before attempting the remove_instance. --- zaza/openstack/charm_tests/mysql/tests.py | 32 +++++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index d4c89ec..982f0d2 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -753,13 +753,19 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Scale in test: remove leader") leader, nons = self.get_leaders_and_non_leaders() leader_unit = zaza.model.get_unit_from_name(leader) - zaza.model.destroy_unit(self.application_name, leader) - - logging.info("Wait until unit is in waiting state ...") - zaza.model.block_until_unit_wl_status(nons[0], "waiting") + # Wait until we are idle in the hopes clients are not running + # update-status hooks logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() + zaza.model.destroy_unit(self.application_name, leader) + + logging.info("Wait until all only 2 units ...") + zaza.model.block_until_unit_count(self.application, 2) + + logging.info("Wait until all units are cluster incomplete ...") + zaza.model.block_until_wl_status_info_starts_with( + self.application, "'cluster' incomplete") logging.info( "Removing old unit from cluster: {} " @@ -786,6 +792,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Adding unit after removed unit ...") zaza.model.add_unit(self.application_name) + logging.info("Wait until 3 units ...") + zaza.model.block_until_unit_count(self.application, 3) + logging.info("Wait for application states ...") zaza.model.wait_for_application_states(states=self.states) @@ -801,6 +810,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Adding unit after full cluster ...") zaza.model.add_unit(self.application_name) + logging.info("Wait until 4 units ...") + zaza.model.block_until_unit_count(self.application, 4) + logging.info("Wait for application states ...") zaza.model.wait_for_application_states(states=self.states) @@ -810,17 +822,21 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): We start with a four node full cluster, remove one, down to a three node full cluster. """ - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() - leader, nons = self.get_leaders_and_non_leaders() non_leader_unit = zaza.model.get_unit_from_name(nons[0]) - zaza.model.destroy_unit(self.application_name, nons[0]) + # Wait until we are idle in the hopes clients are not running + # update-status hooks logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() + zaza.model.destroy_unit(self.application_name, nons[0]) + logging.info("Scale in test: back down to three") + logging.info("Wait until 3 units ...") + zaza.model.block_until_unit_count(self.application, 3) + + logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.states) logging.info( From 1c4d2e05aff4e56859afec4ce80b4e079a41175e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 18 Feb 2021 09:51:00 +0100 Subject: [PATCH 086/180] Let test_930_scaleback remove 2 units instead of 1 --- zaza/openstack/charm_tests/hacluster/tests.py | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 28d0383..b1c0713 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -137,7 +137,10 @@ class HaclusterScalebackTest(HaclusterBaseTest): class HaclusterScaleBackAndForthTest(HaclusterBaseTest): - """hacluster tests scaling back and forth.""" + """hacluster tests scaling back and forth. + + Supersedes HaclusterScalebackTest. + """ @classmethod def setUpClass(cls): @@ -148,38 +151,46 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): cls._hacluster_charm_name = test_config['hacluster-charm-name'] def test_930_scaleback(self): - """Remove a unit, recalculate quorum and add a new one.""" + """Remove two units, recalculate quorum and re-add two units. + + NOTE(lourot): before lp:1400481 was fixed, quorum wasn't recalculated + when removing units. Within a cluster of 3 units, removing two units + and re-adding them led to a situation where the expected quorum became + 5 and was still considered unreached. + """ principle_units = sorted(zaza.model.get_status().applications[ self._principle_app_name]['units'].keys()) self.assertEqual(len(principle_units), 3) - doomed_principle_unit = principle_units[0] - other_principle_unit = principle_units[1] - doomed_hacluster_unit = juju_utils.get_subordinate_units( - [doomed_principle_unit], charm_name=self._hacluster_charm_name)[0] + doomed_principle_units = principle_units[:2] + other_principle_unit = principle_units[2] + doomed_hacluster_units = juju_utils.get_subordinate_units( + doomed_principle_units, charm_name=self._hacluster_charm_name) other_hacluster_unit = juju_utils.get_subordinate_units( [other_principle_unit], charm_name=self._hacluster_charm_name)[0] - logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) - zaza.model.run_action( - doomed_hacluster_unit, - 'pause', - raise_on_failure=True) - logging.info('OK') + for doomed_hacluster_unit in doomed_hacluster_units: + logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) + zaza.model.run_action( + doomed_hacluster_unit, + 'pause', + raise_on_failure=True) - logging.info('Removing {}'.format(doomed_principle_unit)) - zaza.model.destroy_unit( - self._principle_app_name, - doomed_principle_unit, - wait_disappear=True) - logging.info('OK') + for doomed_principle_unit in doomed_principle_units: + logging.info('Removing {}'.format(doomed_principle_unit)) + zaza.model.destroy_unit( + self._principle_app_name, + doomed_principle_unit, + wait_disappear=True) logging.info('Waiting for model to settle') zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') - zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') + # NOTE(lourot): the principle unit (usually a keystone unit) isn't + # guaranteed to be blocked, so we don't validate that. zaza.model.block_until_all_units_idle() - logging.info('OK') logging.info('Updating corosync ring') + # NOTE(lourot): with Juju >= 2.8 this isn't actually necessary as it + # has already been done in hacluster's hanode departing hook: hacluster_app_name = zaza.model.get_unit_from_name( other_hacluster_unit).application zaza.model.run_action_on_leader( @@ -188,9 +199,9 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): action_params={'i-really-mean-it': True}, raise_on_failure=True) - logging.info('Adding an hacluster unit') - zaza.model.add_unit(self._principle_app_name, wait_appear=True) - logging.info('OK') + for article in ('an', 'another'): + logging.info('Adding {} hacluster unit'.format(article)) + zaza.model.add_unit(self._principle_app_name, wait_appear=True) logging.info('Waiting for model to settle') expected_states = {hacluster_app_name: { From f598eb93c29f0c7aec99ac22f1d5d5e5ec7cd7e1 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 18 Feb 2021 10:38:45 +0100 Subject: [PATCH 087/180] Improve comments --- zaza/openstack/charm_tests/hacluster/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index b1c0713..710d3e7 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -185,14 +185,14 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): logging.info('Waiting for model to settle') zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') # NOTE(lourot): the principle unit (usually a keystone unit) isn't - # guaranteed to be blocked, so we don't validate that. + # guaranteed to be blocked, so we don't validate that here. zaza.model.block_until_all_units_idle() logging.info('Updating corosync ring') - # NOTE(lourot): with Juju >= 2.8 this isn't actually necessary as it - # has already been done in hacluster's hanode departing hook: hacluster_app_name = zaza.model.get_unit_from_name( other_hacluster_unit).application + # NOTE(lourot): with Juju >= 2.8 this isn't actually necessary as it + # has already been done in hacluster's hanode departing hook: zaza.model.run_action_on_leader( hacluster_app_name, 'update-ring', From f97c717ba002adb5df348f97dee975884a02a386 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 18 Feb 2021 14:38:01 +0000 Subject: [PATCH 088/180] Add short-term workaround to Trilio bug --- zaza/openstack/charm_tests/trilio/setup.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/zaza/openstack/charm_tests/trilio/setup.py b/zaza/openstack/charm_tests/trilio/setup.py index a7ba7b3..44f1d9d 100644 --- a/zaza/openstack/charm_tests/trilio/setup.py +++ b/zaza/openstack/charm_tests/trilio/setup.py @@ -81,3 +81,17 @@ def basic_setup(): else: logging.error("Unable to find Trilio License file") + + +def python2_workaround(): + """Workaround for Bug #1915914. + + Trilio code currently has a bug which assumes an executable called 'python' + will be on the path. To workaround this install a package which adds a + symlink to python + """ + for unit in zaza_model.get_units('trilio-wlm'): + zaza_model.run_on_unit( + unit.entity_id, + ("apt install --yes python-is-python3; " + "systemctl restart wlm\\*.service")) From c00aa18ffbd03dfad9b196b86a54b51e1ba3ef7e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 18 Feb 2021 16:22:43 +0100 Subject: [PATCH 089/180] Assert that corosync considers all nodes to be online --- zaza/openstack/charm_tests/hacluster/tests.py | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 710d3e7..f720d20 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -153,20 +153,22 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): def test_930_scaleback(self): """Remove two units, recalculate quorum and re-add two units. - NOTE(lourot): before lp:1400481 was fixed, quorum wasn't recalculated - when removing units. Within a cluster of 3 units, removing two units - and re-adding them led to a situation where the expected quorum became - 5 and was still considered unreached. + NOTE(lourot): before lp:1400481 was fixed, the corosync ring wasn't + recalculated when removing units. So within a cluster of 3 units, + removing two units and re-adding them led to a situation where corosync + considers having 3 nodes online out of 5, instead of just 3 out of 3. + This test covers this scenario. """ principle_units = sorted(zaza.model.get_status().applications[ self._principle_app_name]['units'].keys()) self.assertEqual(len(principle_units), 3) doomed_principle_units = principle_units[:2] - other_principle_unit = principle_units[2] + surviving_principle_unit = principle_units[2] doomed_hacluster_units = juju_utils.get_subordinate_units( doomed_principle_units, charm_name=self._hacluster_charm_name) - other_hacluster_unit = juju_utils.get_subordinate_units( - [other_principle_unit], charm_name=self._hacluster_charm_name)[0] + surviving_hacluster_unit = juju_utils.get_subordinate_units( + [surviving_principle_unit], + charm_name=self._hacluster_charm_name)[0] for doomed_hacluster_unit in doomed_hacluster_units: logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) @@ -183,14 +185,15 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): wait_disappear=True) logging.info('Waiting for model to settle') - zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') + zaza.model.block_until_unit_wl_status(surviving_hacluster_unit, + 'blocked') # NOTE(lourot): the principle unit (usually a keystone unit) isn't # guaranteed to be blocked, so we don't validate that here. zaza.model.block_until_all_units_idle() logging.info('Updating corosync ring') hacluster_app_name = zaza.model.get_unit_from_name( - other_hacluster_unit).application + surviving_hacluster_unit).application # NOTE(lourot): with Juju >= 2.8 this isn't actually necessary as it # has already been done in hacluster's hanode departing hook: zaza.model.run_action_on_leader( @@ -200,7 +203,7 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): raise_on_failure=True) for article in ('an', 'another'): - logging.info('Adding {} hacluster unit'.format(article)) + logging.info('Re-adding {} hacluster unit'.format(article)) zaza.model.add_unit(self._principle_app_name, wait_appear=True) logging.info('Waiting for model to settle') @@ -208,4 +211,16 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): "workload-status": "active", "workload-status-message": "Unit is ready and clustered"}} zaza.model.wait_for_application_states(states=expected_states) - logging.debug('OK') + + # At this point if the corosync ring has been properly updated, there + # shouldn't be any trace of the deleted units anymore: + logging.info('Checking that corosync considers all nodes to be online') + cmd = 'sudo crm status' + result = zaza.model.run_on_unit(surviving_hacluster_unit, cmd) + code = result.get('Code') + if code != '0': + raise zaza.model.CommandRunFailed(cmd, result) + output = result.get('Stdout').strip() + logging.debug('Output received: {}'.format(output)) + self.assertNotIn('OFFLINE', output, + "corosync shouldn't list any offline node") From 135da1e1a361e01cadfe17df9a25cfa92f4f1b63 Mon Sep 17 00:00:00 2001 From: David Ames Date: Thu, 18 Feb 2021 09:37:49 -0800 Subject: [PATCH 090/180] Fix reboot-cluster-from-complete-outage race --- zaza/openstack/charm_tests/mysql/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 982f0d2..6322684 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -589,6 +589,10 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): unit.entity_id, 'blocked') + # Wait until update-status hooks have completed + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + logging.info("Execute reboot-cluster-from-complete-outage " "action after cold boot ...") # We do not know which unit has the most up to date data From 8c819814fb01741575bee97ab74d202b00947123 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 18 Feb 2021 10:50:53 +0100 Subject: [PATCH 091/180] Don't wait for principle to get back to active --- zaza/openstack/charm_tests/hacluster/tests.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index f720d20..f340022 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -130,8 +130,7 @@ class HaclusterScalebackTest(HaclusterBaseTest): logging.info('Waiting for model to settle') zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active') # NOTE(lourot): the principle application sometimes remain blocked - # after scaling back up until lp:1400481 is solved. - # zaza.model.block_until_unit_wl_status(other_principle_unit, 'active') + # after scaling back up. zaza.model.block_until_all_units_idle() logging.debug('OK') @@ -207,10 +206,15 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): zaza.model.add_unit(self._principle_app_name, wait_appear=True) logging.info('Waiting for model to settle') - expected_states = {hacluster_app_name: { - "workload-status": "active", - "workload-status-message": "Unit is ready and clustered"}} - zaza.model.wait_for_application_states(states=expected_states) + # NOTE(lourot): the principle charm may remain blocked here. This seems + # to happen often when it is keystone and has a mysql-router as other + # subordinate charm. The keystone units seems to often remain blocked + # with 'Database not initialised'. This is not the hacluster charm's + # fault and this is why we don't validate here that the entire model + # goes back to active/idle. + zaza.model.block_until_unit_wl_status(surviving_hacluster_unit, + 'active') + zaza.model.block_until_all_units_idle() # At this point if the corosync ring has been properly updated, there # shouldn't be any trace of the deleted units anymore: From d728458afa1dca1ca4bbf59541ce248e78f26bbc Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Fri, 8 Jan 2021 15:59:28 +0000 Subject: [PATCH 092/180] Add multi-backend testing for Keystone SAML Mellon The new updated tests rely on new testing bundles deployed with two local IdPs via the Juju charm https://jaas.ai/u/ionutbalutoiu/test-saml-idp. --- .../charm_tests/saml_mellon/setup.py | 121 +++++++++-- .../charm_tests/saml_mellon/tests.py | 199 ++++++++++++++++++ zaza/openstack/utilities/generic.py | 26 +++ 3 files changed, 329 insertions(+), 17 deletions(-) diff --git a/zaza/openstack/charm_tests/saml_mellon/setup.py b/zaza/openstack/charm_tests/saml_mellon/setup.py index 7e5f96c..20b8cbd 100644 --- a/zaza/openstack/charm_tests/saml_mellon/setup.py +++ b/zaza/openstack/charm_tests/saml_mellon/setup.py @@ -25,6 +25,7 @@ from zaza.openstack.utilities import ( cert as cert_utils, cli as cli_utils, openstack as openstack_utils, + generic as generic_utils, ) @@ -34,8 +35,8 @@ FEDERATED_DOMAIN = "federated_domain" FEDERATED_GROUP = "federated_users" MEMBER = "Member" IDP = "samltest" +LOCAL_IDP_REMOTE_ID = "http://{}/simplesaml/saml2/idp/metadata.php" REMOTE_ID = "https://samltest.id/saml/idp" -MAP_NAME = "{}_mapping".format(IDP) PROTOCOL_NAME = "mapped" MAP_TEMPLATE = ''' [{{ @@ -45,7 +46,7 @@ MAP_TEMPLATE = ''' "name": "{{0}}" }}, "group": {{ - "name": "federated_users", + "name": "{group_id}", "domain": {{ "id": "{domain_id}" }} @@ -55,7 +56,7 @@ MAP_TEMPLATE = ''' "name": "{{0}}_project", "roles": [ {{ - "name": "Member" + "name": "{role_name}" }} ] }} @@ -81,7 +82,10 @@ SP_SIGNING_KEY_INFO_XML_TEMPLATE = ''' ''' -def keystone_federation_setup(): +def keystone_federation_setup(federated_domain=FEDERATED_DOMAIN, + federated_group=FEDERATED_GROUP, + idp_name=IDP, + idp_remote_id=REMOTE_ID): """Configure Keystone Federation.""" cli_utils.setup_logging() keystone_session = openstack_utils.get_overcloud_keystone_session() @@ -89,19 +93,19 @@ def keystone_federation_setup(): keystone_session) try: - domain = keystone_client.domains.find(name=FEDERATED_DOMAIN) + domain = keystone_client.domains.find(name=federated_domain) except keystoneauth1.exceptions.http.NotFound: domain = keystone_client.domains.create( - FEDERATED_DOMAIN, + federated_domain, description="Federated Domain", enabled=True) try: group = keystone_client.groups.find( - name=FEDERATED_GROUP, domain=domain) + name=federated_group, domain=domain) except keystoneauth1.exceptions.http.NotFound: group = keystone_client.groups.create( - FEDERATED_GROUP, + federated_group, domain=domain, enabled=True) @@ -109,30 +113,33 @@ def keystone_federation_setup(): keystone_client.roles.grant(role, group=group, domain=domain) try: - idp = keystone_client.federation.identity_providers.find( - name=IDP, domain_id=domain.id) + idp = keystone_client.federation.identity_providers.get(idp_name) except keystoneauth1.exceptions.http.NotFound: idp = keystone_client.federation.identity_providers.create( - IDP, - remote_ids=[REMOTE_ID], + idp_name, + remote_ids=[idp_remote_id], domain_id=domain.id, enabled=True) - JSON_RULES = json.loads(MAP_TEMPLATE.format(domain_id=domain.id)) + JSON_RULES = json.loads(MAP_TEMPLATE.format( + domain_id=domain.id, group_id=group.id, role_name=MEMBER)) + map_name = "{}_mapping".format(idp_name) try: - keystone_client.federation.mappings.find(name=MAP_NAME) + keystone_client.federation.mappings.get(map_name) except keystoneauth1.exceptions.http.NotFound: keystone_client.federation.mappings.create( - MAP_NAME, rules=JSON_RULES) + map_name, rules=JSON_RULES) try: - keystone_client.federation.protocols.get(IDP, PROTOCOL_NAME) + keystone_client.federation.protocols.get(idp_name, PROTOCOL_NAME) except keystoneauth1.exceptions.http.NotFound: keystone_client.federation.protocols.create( - PROTOCOL_NAME, mapping=MAP_NAME, identity_provider=idp) + PROTOCOL_NAME, mapping=map_name, identity_provider=idp) +# This setup method is deprecated. It will be removed once we fully drop the +# `samltest.id` dependency. def attach_saml_resources(application="keystone-saml-mellon"): """Attach resource to the Keystone SAML Mellon charm.""" test_idp_metadata_xml = "samltest.xml" @@ -161,3 +168,83 @@ def attach_saml_resources(application="keystone-saml-mellon"): fp.flush() zaza.model.attach_resource( application, sp_signing_keyinfo_name, fp.name) + + +def _attach_saml_resources_local_idp(keystone_saml_mellon_app_name=None, + test_saml_idp_app_name=None): + """Attach resources to the Keystone SAML Mellon and the local IdP.""" + action_result = zaza.model.run_action_on_leader( + test_saml_idp_app_name, 'get-idp-metadata') + idp_metadata = action_result.data['results']['output'] + + generic_utils.attach_file_resource( + keystone_saml_mellon_app_name, + 'idp-metadata', + idp_metadata, + '.xml') + + (key, cert) = cert_utils.generate_cert('SP Signing Key') + + cert = cert.decode().strip("-----BEGIN CERTIFICATE-----") + cert = cert.strip("-----END CERTIFICATE-----") + + generic_utils.attach_file_resource( + keystone_saml_mellon_app_name, + 'sp-private-key', + key.decode(), + '.pem') + generic_utils.attach_file_resource( + keystone_saml_mellon_app_name, + 'sp-signing-keyinfo', + SP_SIGNING_KEY_INFO_XML_TEMPLATE.format(cert), + '.xml') + + action_result = zaza.model.run_action_on_leader( + keystone_saml_mellon_app_name, 'get-sp-metadata') + sp_metadata = action_result.data['results']['output'] + + generic_utils.attach_file_resource( + test_saml_idp_app_name, + 'sp-metadata', + sp_metadata, + '.xml') + + +def attach_saml_resources_idp1(): + """Attach the SAML resources for the local IdP #1.""" + _attach_saml_resources_local_idp( + keystone_saml_mellon_app_name="keystone-saml-mellon1", + test_saml_idp_app_name="test-saml-idp1") + + +def attach_saml_resources_idp2(): + """Attach the SAML resources for the local IdP #2.""" + _attach_saml_resources_local_idp( + keystone_saml_mellon_app_name="keystone-saml-mellon2", + test_saml_idp_app_name="test-saml-idp2") + + +def keystone_federation_setup_idp1(): + """Configure Keystone Federation for the local IdP #1.""" + test_saml_idp_unit = zaza.model.get_units("test-saml-idp1")[0] + idp_remote_id = LOCAL_IDP_REMOTE_ID.format( + test_saml_idp_unit.public_address) + + keystone_federation_setup( + federated_domain="federated_domain_idp1", + federated_group="federated_users_idp1", + idp_name="test-saml-idp1", + idp_remote_id=idp_remote_id) + + +def keystone_federation_setup_idp2(): + """Configure Keystone Federation for the local IdP #2.""" + test_saml_idp_unit = zaza.model.get_units("test-saml-idp2")[0] + idp_remote_id = LOCAL_IDP_REMOTE_ID.format( + test_saml_idp_unit.public_address) + + keystone_federation_setup( + federated_domain="federated_domain_idp2", + federated_group="federated_users_idp2", + idp_name="test-saml-idp2", + idp_remote_id=idp_remote_id) diff --git a/zaza/openstack/charm_tests/saml_mellon/tests.py b/zaza/openstack/charm_tests/saml_mellon/tests.py index 9b5b211..962fa92 100644 --- a/zaza/openstack/charm_tests/saml_mellon/tests.py +++ b/zaza/openstack/charm_tests/saml_mellon/tests.py @@ -30,6 +30,8 @@ class FailedToReachIDP(Exception): pass +# This testing class is deprecated. It will be removed once we fully drop the +# `samltest.id` dependency. class CharmKeystoneSAMLMellonTest(BaseKeystoneTest): """Charm Keystone SAML Mellon tests.""" @@ -156,3 +158,200 @@ class CharmKeystoneSAMLMellonTest(BaseKeystoneTest): # We may need to try/except to allow horizon to build its pages _do_redirect_check(url, region, idp_expect, horizon_expect) logging.info("SUCCESS") + + +class BaseCharmKeystoneSAMLMellonTest(BaseKeystoneTest): + """Charm Keystone SAML Mellon tests.""" + + @classmethod + def setUpClass(cls, + application_name="keystone-saml-mellon", + test_saml_idp_app_name="test-saml-idp", + horizon_idp_option_name="myidp_mapped", + horizon_idp_display_name="myidp via mapped"): + """Run class setup for running Keystone SAML Mellon charm tests.""" + super(BaseCharmKeystoneSAMLMellonTest, cls).setUpClass() + # Note: The BaseKeystoneTest class sets the application_name to + # "keystone" which breaks keystone-saml-mellon actions. Explicitly set + # application name here. + cls.application_name = application_name + cls.test_saml_idp_app_name = test_saml_idp_app_name + cls.horizon_idp_option_name = horizon_idp_option_name + cls.horizon_idp_display_name = horizon_idp_display_name + cls.action = "get-sp-metadata" + cls.current_release = openstack_utils.get_os_release() + cls.FOCAL_USSURI = openstack_utils.get_os_release("focal_ussuri") + + @staticmethod + def check_horizon_redirect(horizon_url, horizon_expect, + horizon_idp_option_name, horizon_region, + idp_url, idp_expect): + """Validate the Horizon -> Keystone -> IDP redirects. + + This validation is done through `requests.session()`, and the proper + get / post http calls. + + :param horizon_url: The login page for the Horizon OpenStack dashboard. + :type horizon_url: string + :param horizon_expect: Information that needs to be displayed by + Horizon login page, when there is a proper + SAML IdP configuration. + :type horizon_expect: string + :param horizon_idp_option_name: The name of the IdP that is chosen + in the Horizon dropdown from the login + screen. This will go in the post body + as 'auth_type'. + :type horizon_idp_option_name: string + :param horizon_region: Information needed to complete the http post + data for the Horizon login. + :type horizon_region: string + :param idp_url: The url for the IdP where the user needs to be + redirected. + :type idp_url: string + :param idp_expect: Information that needs to be displayed by the IdP + after the user is redirected there. + :type idp_expect: string + :returns: None + """ + # start session, get csrftoken + client = requests.session() + # Verify=False see note below + login_page = client.get(horizon_url, verify=False) + + # Validate SAML method is available + assert horizon_expect in login_page.text + + # Get cookie + if "csrftoken" in client.cookies: + csrftoken = client.cookies["csrftoken"] + else: + raise Exception("Missing csrftoken") + + # Build and send post request + form_data = { + "auth_type": horizon_idp_option_name, + "csrfmiddlewaretoken": csrftoken, + "next": "/horizon/project/api_access", + "region": horizon_region, + } + + # Verify=False due to CA certificate bundles. + # If we don't set it validation fails for keystone/horizon + # We would have to install the keystone CA onto the system + # to validate end to end. + response = client.post( + horizon_url, + data=form_data, + headers={"Referer": horizon_url}, + allow_redirects=True, + verify=False) + + if idp_expect not in response.text: + msg = "FAILURE code={} text={}".format(response, response.text) + # Raise a custom exception. + raise FailedToReachIDP(msg) + + # Validate that we were redirected to the proper IdP + assert response.url.startswith(idp_url) + assert idp_url in response.text + + def test_run_get_sp_metadata_action(self): + """Validate the get-sp-metadata action.""" + unit = zaza.model.get_units(self.application_name)[0] + ip = self.vip if self.vip else unit.public_address + + action = zaza.model.run_action(unit.entity_id, self.action) + self.assertNotIn( + "failed", + action.data["status"], + msg="The action failed: {}".format(action.data["message"])) + + output = action.data["results"]["output"] + root = etree.fromstring(output) + for item in root.items(): + if "entityID" in item[0]: + self.assertIn(ip, item[1]) + + for appt in root.getchildren(): + for elem in appt.getchildren(): + for item in elem.items(): + if "Location" in item[0]: + self.assertIn(ip, item[1]) + + logging.info("Successul get-sp-metadata action") + + def test_saml_mellon_redirects(self): + """Validate the horizon -> keystone -> IDP redirects.""" + unit = zaza.model.get_units(self.application_name)[0] + keystone_ip = self.vip if self.vip else unit.public_address + + horizon = "openstack-dashboard" + horizon_config = zaza.model.get_application_config(horizon) + horizon_vip = horizon_config.get("vip").get("value") + unit = zaza.model.get_units("openstack-dashboard")[0] + + horizon_ip = horizon_vip if horizon_vip else unit.public_address + proto = "https" if self.tls_rid else "http" + + # Use Keystone URL for < Focal + if self.current_release < self.FOCAL_USSURI: + region = "{}://{}:5000/v3".format(proto, keystone_ip) + else: + region = "default" + + idp_address = zaza.model.get_units( + self.test_saml_idp_app_name)[0].public_address + + horizon_url = "{}://{}/horizon/auth/login/".format(proto, horizon_ip) + horizon_expect = ''.format( + self.horizon_idp_option_name, self.horizon_idp_display_name) + idp_url = ("http://{0}/simplesaml/" + "module.php/core/loginuserpass.php").format(idp_address) + # This is the message the local test-saml-idp displays after you are + # redirected. It shows we have been directed to: + # horizon -> keystone -> test-saml-idp + idp_expect = ( + "A service has requested you to authenticate yourself. Please " + "enter your username and password in the form below.") + + # Execute the check + BaseCharmKeystoneSAMLMellonTest.check_horizon_redirect( + horizon_url=horizon_url, + horizon_expect=horizon_expect, + horizon_idp_option_name=self.horizon_idp_option_name, + horizon_region=region, + idp_url=idp_url, + idp_expect=idp_expect) + logging.info("SUCCESS") + + +class CharmKeystoneSAMLMellonIDP1Test(BaseCharmKeystoneSAMLMellonTest): + """Charm Keystone SAML Mellon tests class for the local IDP #1.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Keystone SAML Mellon charm tests. + + It does the necessary setup for the local IDP #1. + """ + super(CharmKeystoneSAMLMellonIDP1Test, cls).setUpClass( + application_name="keystone-saml-mellon1", + test_saml_idp_app_name="test-saml-idp1", + horizon_idp_option_name="test-saml-idp1_mapped", + horizon_idp_display_name="Test SAML IDP #1") + + +class CharmKeystoneSAMLMellonIDP2Test(BaseCharmKeystoneSAMLMellonTest): + """Charm Keystone SAML Mellon tests class for the local IDP #2.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running Keystone SAML Mellon charm tests. + + It does the necessary setup for the local IDP #2. + """ + super(CharmKeystoneSAMLMellonIDP2Test, cls).setUpClass( + application_name="keystone-saml-mellon2", + test_saml_idp_app_name="test-saml-idp2", + horizon_idp_option_name="test-saml-idp2_mapped", + horizon_idp_display_name="Test SAML IDP #2") diff --git a/zaza/openstack/utilities/generic.py b/zaza/openstack/utilities/generic.py index dd8b1e0..fcd7989 100644 --- a/zaza/openstack/utilities/generic.py +++ b/zaza/openstack/utilities/generic.py @@ -20,6 +20,7 @@ import os import socket import subprocess import telnetlib +import tempfile import yaml from zaza import model @@ -680,3 +681,28 @@ def get_mojo_cacert_path(): return cacert else: raise zaza_exceptions.CACERTNotFound("Could not find cacert.pem") + + +def attach_file_resource(application_name, resource_name, + file_content, file_suffix=".txt"): + """Attaches a file as a Juju resource given the file content and suffix. + + The file content will be written into a temporary file with the given + suffix, and it will be attached to the Juju application. + + :param application_name: Juju application name. + :type application_name: string + :param resource_name: Juju resource name. + :type resource_name: string + :param file_content: The content of the file that will be attached + :type file_content: string + :param file_suffix: File suffix. This should be used to set the file + extension for applications that are sensitive to this. + :type file_suffix: string + :returns: None + """ + with tempfile.NamedTemporaryFile(mode='w', suffix=file_suffix) as fp: + fp.write(file_content) + fp.flush() + model.attach_resource( + application_name, resource_name, fp.name) From d41f8b37240b794442c873327696ffe2ed9d4fbb Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Wed, 20 Jan 2021 20:10:34 +0200 Subject: [PATCH 093/180] Update Ceph RBD Mirror tests The updated tests add the possibility of testing deployments with `image` RBD mirroring mode implemented as part of the Cinder Ceph Replication charm spec. --- .../charm_tests/ceph/rbd_mirror/tests.py | 308 ++++++++++++++---- 1 file changed, 242 insertions(+), 66 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py index 6c2fa1b..75a3f42 100644 --- a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py @@ -16,6 +16,9 @@ import json import logging import re +import time + +import cinderclient.exceptions as cinder_exceptions import zaza.openstack.charm_tests.test_utils as test_utils @@ -39,16 +42,20 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): cls.site_a_model = cls.site_b_model = zaza.model.get_juju_model() cls.site_b_app_suffix = '-b' - def run_status_action(self, application_name=None, model_name=None): + def run_status_action(self, application_name=None, model_name=None, + pools=[]): """Run status action, decode and return response.""" + action_params = { + 'verbose': True, + 'format': 'json', + } + if len(pools) > 0: + action_params['pools'] = ','.join(pools) result = zaza.model.run_action_on_leader( application_name or self.application_name, 'status', model_name=model_name, - action_params={ - 'verbose': True, - 'format': 'json', - }) + action_params=action_params) return json.loads(result.results['output']) def get_pools(self): @@ -71,7 +78,8 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): def wait_for_mirror_state(self, state, application_name=None, model_name=None, check_entries_behind_master=False, - require_images_in=[]): + require_images_in=[], + pools=[]): """Wait until all images reach requested state. This function runs the ``status`` action and examines the data it @@ -90,6 +98,9 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): :type check_entries_behind_master: bool :param require_images_in: List of pools to require images in :type require_images_in: list of str + :param pools: List of pools to run status on. If this is empty, the + status action will run on all the pools. + :type pools: list of str :returns: True on success, never returns on failure """ rep = re.compile(r'.*entries_behind_master=(\d+)') @@ -97,7 +108,8 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): try: # encapsulate in try except to work around LP: #1820976 pool_status = self.run_status_action( - application_name=application_name, model_name=model_name) + application_name=application_name, model_name=model_name, + pools=pools) except KeyError: continue for pool, status in pool_status.items(): @@ -124,6 +136,119 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): # all images with state has expected state return True + def get_cinder_rbd_mirroring_mode(self, + cinder_ceph_app_name='cinder-ceph'): + """Get the RBD mirroring mode for the Cinder Ceph pool. + + :returns: A string representing the RBD mirroring mode. It can be + either 'pool' or 'image'. + """ + DEFAULT_RBD_MIRRORING_MODE = 'pool' + + rbd_mirroring_mode_config = zaza.model.get_application_config( + cinder_ceph_app_name).get('rbd-mirroring-mode') + if rbd_mirroring_mode_config: + rbd_mirroring_mode = rbd_mirroring_mode_config.get( + 'value', DEFAULT_RBD_MIRRORING_MODE).lower() + else: + rbd_mirroring_mode = DEFAULT_RBD_MIRRORING_MODE + + return rbd_mirroring_mode + + def create_cinder_volume(self, session, from_image=False): + """Create Cinder Volume from image. + + :rtype: :class:`Volume`. + """ + def get_glance_image(session): + glance = openstack.get_glance_session_client(session) + images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) + if images: + return images[0] + logging.info("Failed to find {} image, falling back to {}".format( + CIRROS_IMAGE_NAME, + LTS_IMAGE_NAME)) + return openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] + + def create_volume_type(cinder): + try: + vol_type = cinder.volume_types.find(name='repl') + except cinder_exceptions.NotFound: + vol_type = cinder.volume_types.create('repl') + vol_type.set_keys(metadata={ + 'volume_backend_name': 'cinder-ceph', + 'replication_enabled': ' True', + }) + return vol_type + + # NOTE(fnordahl): for some reason create volume from image often fails + # when run just after deployment is finished. We should figure out + # why, resolve the underlying issue and then remove this. + # + # We do not use tenacity here as it will interfere with tenacity used + # in ``resource_reaches_status`` + def create_volume(cinder, volume_params, retry=20): + if retry < 1: + return + volume = cinder.volumes.create(**volume_params) + try: + # Note(coreycb): stop_after_attempt is increased because using + # juju storage for ceph-osd backed by cinder on undercloud + # takes longer than the prior method of directory-backed OSD + # devices. + openstack.resource_reaches_status( + cinder.volumes, volume.id, msg='volume', + stop_after_attempt=20) + return volume + except AssertionError: + logging.info('retrying') + volume.delete() + return create_volume(cinder, volume_params, retry=retry - 1) + + volume_params = { + 'size': 8, + 'name': 'zaza', + } + if from_image: + volume_params['imageRef'] = get_glance_image(session).id + cinder = openstack.get_cinder_session_client(session) + if self.get_cinder_rbd_mirroring_mode() == 'image': + volume_params['volume_type'] = create_volume_type(cinder).id + + return create_volume(cinder, volume_params) + + def failover_cinder_volume_host(self, cinder_client, + backend_name='cinder-ceph', + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over', + timeout=300): + """Failover Cinder volume host.""" + host = 'cinder@{}'.format(backend_name) + logging.info( + 'Failover Cinder host %s to backend_id %s', + host, target_backend_id) + cinder_client.services.failover_host( + host=host, + backend_id=target_backend_id) + start = time.time() + while True: + elapsed = time.time() - start + if elapsed > timeout: + raise cinder_exceptions.TimeoutException( + obj=cinder_client.services, + action='failover_host') + service = cinder_client.services.list( + host=host, + binary='cinder-volume')[0] + if (service.status == target_status and + service.replication_status == target_replication_status): + break + time.sleep(5) + logging.info( + 'Successfully failed-over Cinder host %s to backend_id %s', + host, target_backend_id) + class CephRBDMirrorTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` tests.""" @@ -196,43 +321,7 @@ class CephRBDMirrorTest(CephRBDMirrorBase): test. """ session = openstack.get_overcloud_keystone_session() - glance = openstack.get_glance_session_client(session) - cinder = openstack.get_cinder_session_client(session) - - images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) - if images: - image = images[0] - else: - logging.info("Failed to find {} image, falling back to {}".format( - CIRROS_IMAGE_NAME, - LTS_IMAGE_NAME)) - image = openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] - - # NOTE(fnordahl): for some reason create volume from image often fails - # when run just after deployment is finished. We should figure out - # why, resolve the underlying issue and then remove this. - # - # We do not use tenacity here as it will interfere with tenacity used - # in ``resource_reaches_status`` - def create_volume_from_image(cinder, image, retry=20): - if retry < 1: - return - volume = cinder.volumes.create(8, name='zaza', imageRef=image.id) - try: - # Note(coreycb): stop_after_attempt is increased because using - # juju storage for ceph-osd backed by cinder on undercloud - # takes longer than the prior method of directory-backed OSD - # devices. - openstack.resource_reaches_status( - cinder.volumes, volume.id, msg='volume', - stop_after_attempt=20) - return volume - except AssertionError: - logging.info('retrying') - volume.delete() - return create_volume_from_image(cinder, image, retry=retry - 1) - volume = create_volume_from_image(cinder, image) - + volume = self.create_cinder_volume(session, from_image=True) site_a_hash = zaza.openstack.utilities.ceph.get_rbd_hash( zaza.model.get_lead_unit_name('ceph-mon', model_name=self.site_a_model), @@ -258,85 +347,170 @@ class CephRBDMirrorTest(CephRBDMirrorBase): class CephRBDMirrorControlledFailoverTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` controlled failover tests.""" + def cinder_fail_over_fall_back(self): + """Validate controlled fail over and fall back via the Cinder API.""" + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session) + volume = self.create_cinder_volume(session, from_image=True) + self.wait_for_mirror_state( + 'up+replaying', + check_entries_behind_master=True, + application_name=self.application_name + self.site_b_app_suffix, + model_name=self.site_b_model, + pools=['cinder-ceph']) + self.failover_cinder_volume_host( + cinder_client=cinder) + self.assertEqual(cinder.volumes.get(volume.id).status, 'available') + self.failover_cinder_volume_host( + cinder_client=cinder, + target_backend_id='default', + target_status='enabled', + target_replication_status='enabled') + self.assertEqual(cinder.volumes.get(volume.id).status, 'available') + def test_fail_over_fall_back(self): """Validate controlled fail over and fall back.""" site_a_pools, site_b_pools = self.get_pools() + site_a_action_params = {} + site_b_action_params = {} + if self.get_cinder_rbd_mirroring_mode() == 'image': + site_a_pools.remove('cinder-ceph') + site_a_action_params['pools'] = ','.join(site_a_pools) + site_b_pools.remove('cinder-ceph') + site_b_action_params['pools'] = ','.join(site_b_pools) result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror', 'demote', model_name=self.site_a_model, - action_params={}) + action_params=site_a_action_params) logging.info(result.results) n_pools_demoted = len(result.results['output'].split('\n')) self.assertEqual(len(site_a_pools), n_pools_demoted) - self.wait_for_mirror_state('up+unknown', model_name=self.site_a_model) + self.wait_for_mirror_state( + 'up+unknown', + model_name=self.site_a_model, + pools=site_a_pools) self.wait_for_mirror_state( 'up+unknown', application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model) + model_name=self.site_b_model, + pools=site_b_pools) result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror' + self.site_b_app_suffix, 'promote', model_name=self.site_b_model, - action_params={}) + action_params=site_b_action_params) logging.info(result.results) n_pools_promoted = len(result.results['output'].split('\n')) self.assertEqual(len(site_b_pools), n_pools_promoted) self.wait_for_mirror_state( 'up+replaying', - model_name=self.site_a_model) + model_name=self.site_a_model, + pools=site_a_pools) self.wait_for_mirror_state( 'up+stopped', application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model) + model_name=self.site_b_model, + pools=site_b_pools) result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror' + self.site_b_app_suffix, 'demote', model_name=self.site_b_model, - action_params={ - }) + action_params=site_b_action_params) logging.info(result.results) n_pools_demoted = len(result.results['output'].split('\n')) self.assertEqual(len(site_a_pools), n_pools_demoted) self.wait_for_mirror_state( 'up+unknown', - model_name=self.site_a_model) + model_name=self.site_a_model, + pools=site_a_pools) self.wait_for_mirror_state( 'up+unknown', application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model) + model_name=self.site_b_model, + pools=site_b_pools) result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror', 'promote', model_name=self.site_a_model, - action_params={ - }) + action_params=site_a_action_params) logging.info(result.results) n_pools_promoted = len(result.results['output'].split('\n')) self.assertEqual(len(site_b_pools), n_pools_promoted) self.wait_for_mirror_state( 'up+stopped', - model_name=self.site_a_model) + model_name=self.site_a_model, + pools=site_a_pools) + action_params = { + 'i-really-mean-it': True, + } + if self.get_cinder_rbd_mirroring_mode() == 'image': + action_params['pools'] = site_b_action_params['pools'] result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror' + self.site_b_app_suffix, 'resync-pools', model_name=self.site_b_model, - action_params={ - 'i-really-mean-it': True, - }) + action_params=action_params) logging.info(result.results) self.wait_for_mirror_state( 'up+replaying', application_name=self.application_name + self.site_b_app_suffix, model_name=self.site_b_model, - require_images_in=['cinder-ceph', 'glance']) + require_images_in=['cinder-ceph', 'glance'], + pools=site_a_pools) + if self.get_cinder_rbd_mirroring_mode() == 'image': + self.cinder_fail_over_fall_back() class CephRBDMirrorDisasterFailoverTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` destructive tests.""" + def forced_failover_cinder_volume_host(self, cinder_client): + """Validate forced Cinder volume host fail over.""" + def apply_cinder_workaround(): + """Set minimal timeouts / retries to the Cinder Ceph backend. + + This is needed because the failover via Cinder will try to do a + demotion of the site-a, and with the default timeouts / retries, + the operation takes an unreasonably amount of time. + """ + cinder_configs = { + 'rados_connect_timeout': '1', + 'rados_connection_retries': '1', + 'rados_connection_interval': '0', + 'replication_connect_timeout': '1', + } + update_cinder_conf_cmd = ( + "import configparser; " + "config = configparser.ConfigParser(); " + "config.read('/etc/cinder/cinder.conf'); " + "{}" + "f = open('/etc/cinder/cinder.conf', 'w'); " + "config.write(f); " + "f.close()") + cmd = '' + for config in cinder_configs: + cmd += "config.set('cinder-ceph', '{0}', '{1}'); ".format( + config, cinder_configs[config]) + cmd = update_cinder_conf_cmd.format(cmd) + zaza.model.run_on_leader( + 'cinder-ceph', + 'python3 -c "{}"; systemctl restart cinder-volume'.format(cmd)) + + apply_cinder_workaround() + self.failover_cinder_volume_host(cinder_client) + + for volume in cinder_client.volumes.list(): + self.assertEqual(volume.status, 'available') + def test_kill_site_a_fail_over(self): """Validate fail over after uncontrolled shutdown of primary.""" + action_params = {} + if self.get_cinder_rbd_mirroring_mode() == 'image': + _, site_b_pools = self.get_pools() + site_b_pools.remove('cinder-ceph') + action_params['pools'] = ','.join(site_b_pools) + for application in 'ceph-rbd-mirror', 'ceph-mon', 'ceph-osd': zaza.model.remove_application( application, @@ -346,14 +520,16 @@ class CephRBDMirrorDisasterFailoverTest(CephRBDMirrorBase): 'ceph-rbd-mirror' + self.site_b_app_suffix, 'promote', model_name=self.site_b_model, - action_params={ - }) + action_params=action_params) self.assertEqual(result.status, 'failed') + action_params['force'] = True result = zaza.model.run_action_on_leader( 'ceph-rbd-mirror' + self.site_b_app_suffix, 'promote', model_name=self.site_b_model, - action_params={ - 'force': True, - }) + action_params=action_params) self.assertEqual(result.status, 'completed') + if self.get_cinder_rbd_mirroring_mode() == 'image': + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session) + self.forced_failover_cinder_volume_host(cinder) From 2054b7c4775ea8c79d99d60f23a684f6e1910db3 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 25 Feb 2021 09:16:34 +0000 Subject: [PATCH 094/180] Add test for deferred restarts --- zaza/openstack/charm_tests/neutron/tests.py | 14 ++ zaza/openstack/charm_tests/test_utils.py | 236 ++++++++++++++++++++ 2 files changed, 250 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index bebe3b6..412a352 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -952,3 +952,17 @@ class NeutronNetworkingVRRPTests(NeutronNetworkingBase): uc_neutron_client, gateway_hostname) self.check_connectivity(instance_1, instance_2) + + +class NeutronOVSDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super(NeutronOVSDeferredRestartTest, cls).setUpClass( + restart_config_file='/etc/neutron/neutron.conf', + test_service='neutron-openvswitch-agent', + restart_package='openvswitch-switch', + restart_package_service='openvswitch-switch', + application_name='neutron-openvswitch') diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 5a3e6c1..bd82a38 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -18,6 +18,7 @@ import subprocess import sys import tenacity import unittest +import yaml import novaclient @@ -701,3 +702,238 @@ class OpenStackBaseTest(BaseCharmTest): instance_2 = self.retrieve_guest( '{}-ins-1'.format(self.RESOURCE_PREFIX)) return instance_1, instance_2 + + +class BaseDeferredRestartTest(OpenStackBaseTest): + """Check deferred restarts. + + Example of adding a deferred restart test:: + + class NeutronOVSDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + + @classmethod + def setUpClass(cls): + super(NeutronOVSDeferredRestartTest, cls).setUpClass( + restart_config_file = '/etc/neutron/neutron.conf', + test_service = 'neutron-openvswitch-agent', + restart_package = 'openvswitch-switch', + restart_package_service = 'openvswitch-switch', + application_name = 'neutron-openvswitch') + + NOTE: The test has been broken into various class methods which may require + specialisation if the charm being tested is not a standard OpenStack + charm e.g. `trigger_deferred_restart_via_charm` if the charm is not + an oslo config or does not have a debug option. + """ + + @classmethod + def setUpClass(cls, restart_config_file, test_service, restart_package, + restart_package_service, application_name): + """Run test setup. + + :param restart_config_file: Config file that will be changed to trigger + a service restart. + :type restart_config_file: str + :param test_service: Service that will require a restart after + restart_config_file has changed. + :type test_service: str + :param restart_package: Package that will be changed to trigger a + service restart. + :type restart_package: str + :param restart_package_service: Service that will require a restart + after restart_package has changed. + :type restart_package_service: str + :param application_name: Name of application to run tests against. + :type application_name: str + """ + cls.restart_config_file = restart_config_file + cls.test_service = test_service + cls.restart_package = restart_package + cls.restart_package_service = restart_package_service + cls.application_name = application_name + super(BaseDeferredRestartTest, cls).setUpClass( + application_name=cls.application_name) + + def check_clear_restarts(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in model.get_units(self.application_name): + model.run_action( + unit.entity_id, + 'restart-services', + action_params={'deferred': True}) + + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + assert unit.workload_status_message == 'Unit is ready' + + def check_show_deferred_restarts_action(self, test_service, + restart_reason): + """Check the output from the action to list deferred restarts. + + Run the action to list any deferred restarts and check it has entry for + the given service and reason. + + :param test_service: Service that should need a restart + :type test_service: str + :param restart_reason: The reason the action should list for the + service needing to be restarted. This can be a + substring. + :type restart_reason: str + """ + # Ensure that the deferred restart and cause are listed via action + for unit in model.get_units(self.application_name): + action = model.run_action( + unit.entity_id, + 'show-deferred-restarts') + logging.info( + ("Checking {} is marked as needing restart in " + "show-deferred-restarts action on {}").format( + test_service, + unit.entity_id)) + for event in yaml.safe_load(action.data['results']['output']): + if test_service in event and restart_reason in event: + break + else: + msg = 'No entry for restart of {} for reason {} found'.format( + test_service, + restart_reason) + raise Exception(msg) + + def check_show_deferred_restarts_wlm(self, test_service): + """Check the workload status message lists deferred restart. + + :param test_service: Service that should need a restart + :type test_service: str + """ + # Ensure that the deferred restarts are visible in Juju status + for unit in model.get_units(self.application_name): + # Just checking one example service should we be checking all? + logging.info( + ("Checking {} is marked as needing restart in workload " + "message of {}".format(test_service, unit.entity_id))) + assert test_service in unit.workload_status_message + + def trigger_deferred_restart_via_charm(self): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be renderred in + applications config file. + + NOTE: The implementation assumes the charm has a `debug` option and + self.restart_config_file in an oslo config file where that + debug option is renderred. If that is not true the specaliasation + class should override this method. + """ + app_config = model.get_application_config(self.application_name) + logging.info("Triggering deferred restart via config change") + new_debug_value = str(not app_config['debug']['value']) + logging.info("Setting debug: {}".format(new_debug_value)) + model.set_application_config( + self.application_name, + {'debug': new_debug_value}) + expected_contents = { + 'DEFAULT': { + 'debug': [new_debug_value]}} + logging.info("Waiting for debug to be {} in {}".format( + new_debug_value, + self.restart_config_file)) + model.block_until_oslo_config_entries_match( + self.application_name, + self.restart_config_file, + expected_contents) + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + + def trigger_deferred_restart_via_package(self): + """Update a package which requires a service restart.""" + logging.info("Triggering deferred restart via package change") + # Test restart requested by package + for unit in model.get_units(self.application_name): + model.run_on_unit( + unit.entity_id, + 'dpkg-reconfigure {}; ./hooks/update-status'.format( + self.restart_package)) + + def run_charm_change_test(self): + """Trigger a deferred restart by updating a config file via the charm. + + Trigger a config-changed hook in the charm which will update a config + file and add a deferred restart. + + NOTE: If this test is not relevant for the target charm then override + this method and raise unittest.SkipTest + """ + self.trigger_deferred_restart_via_charm() + + self.check_show_deferred_restarts_wlm(self.test_service) + self.check_show_deferred_restarts_action( + self.test_service, + self.restart_config_file) + logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def run_package_change_test(self): + """Trigger a deferred restart by updating a package. + + Update a package which requires will add a deferred restart. + + NOTE: If this test is not relevant for the target charm then override + this method and raise unittest.SkipTest + """ + self.trigger_deferred_restart_via_package() + + self.check_show_deferred_restarts_wlm(self.restart_package_service) + self.check_show_deferred_restarts_action( + self.restart_package_service, + 'Pkg Update') + logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def test_deferred_restarts(self): + """Run deferred restart tests.""" + app_config = model.get_application_config(self.application_name) + auto_restart_config_key = 'enable-auto-restarts' + if auto_restart_config_key not in app_config: + raise unittest.SkipTest("Deferred restarts not implemented") + + # Ensure auto restarts are off. + policy_file = '/etc/policy-rc.d/charm-{}.policy'.format( + self.application_name) + if app_config[auto_restart_config_key]['value']: + logging.info("Turning off auto restarts") + model.set_application_config( + self.application_name, {auto_restart_config_key: 'False'}) + logging.info("Waiting for {} to appear on units of {}".format( + policy_file, + self.application_name)) + model.block_until_file_has_contents( + self.application_name, + policy_file, + 'policy_requestor_name') + # The block_until_file_has_contents ensures the change we waiting + # for has happened, now just wait for any hooks to finish. + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + else: + logging.info("Auto restarts already disabled") + + # Trigger a config change which requires a restart + self.run_charm_change_test() + + # Trigger a package change which requires a restart + self.run_package_change_test() + + # Finished so turn auto-restarts back on. + logging.info("Turning on auto restarts") + model.set_application_config( + self.application_name, {auto_restart_config_key: 'True'}) + model.block_until_file_missing( + self.application_name, + policy_file) + model.block_until_all_units_idle() From d4b8fbaed037df0fb9050c9837c5f9efdc33455d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 25 Feb 2021 11:01:22 +0000 Subject: [PATCH 095/180] Rerun self.trigger_deferred_restart_via_charm --- zaza/openstack/charm_tests/test_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index bd82a38..3aff816 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -875,6 +875,8 @@ class BaseDeferredRestartTest(OpenStackBaseTest): self.check_show_deferred_restarts_action( self.test_service, self.restart_config_file) + # Rerunning to flip config option back to previous value. + self.trigger_deferred_restart_via_charm() logging.info("Running restart action to clear deferred restarts") self.check_clear_restarts() From 20bd5dfb5908f79b6e2c1c19b5e6e893baad86ee Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Thu, 25 Feb 2021 13:05:30 +0100 Subject: [PATCH 096/180] Remove obsolete comment The automatic corosync cleanup on Juju >= 2.8 turned out not to work and has been removed from charm-hacluster. The 'update-ring' action is now always needed. --- zaza/openstack/charm_tests/hacluster/tests.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index f340022..ea613ec 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -193,8 +193,6 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): logging.info('Updating corosync ring') hacluster_app_name = zaza.model.get_unit_from_name( surviving_hacluster_unit).application - # NOTE(lourot): with Juju >= 2.8 this isn't actually necessary as it - # has already been done in hacluster's hanode departing hook: zaza.model.run_action_on_leader( hacluster_app_name, 'update-ring', From adda37269958edad8c6087d264e58b8267f5261e Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 2 Mar 2021 22:44:40 +0000 Subject: [PATCH 097/180] Show cluster status before removal --- zaza/openstack/charm_tests/mysql/tests.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 6322684..15256e7 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -771,6 +771,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): zaza.model.block_until_wl_status_info_starts_with( self.application, "'cluster' incomplete") + # Show status + logging.info(self.get_cluster_status()) + logging.info( "Removing old unit from cluster: {} " .format(leader_unit.public_address)) @@ -843,6 +846,9 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): logging.info("Wait for status ready ...") zaza.model.wait_for_application_states(states=self.states) + # Show status + logging.info(self.get_cluster_status()) + logging.info( "Removing old unit from cluster: {} " .format(non_leader_unit.public_address)) From c80f5fdef8fe80564406b084961d5a6b4f7e03a0 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 3 Mar 2021 10:50:53 +0100 Subject: [PATCH 098/180] Make HaclusterScaleBackAndForthTest less stressful Removing and re-adding two units out of a cluster of three so fast seems to be too stressful for corosync/pacemaker, which makes the test fail in rare cases. Removing and re-adding one unit only is enough in order to validate the functionality. This also makes the test execution time much shorter. --- zaza/openstack/charm_tests/hacluster/tests.py | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index ea613ec..b365960 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -154,40 +154,39 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): NOTE(lourot): before lp:1400481 was fixed, the corosync ring wasn't recalculated when removing units. So within a cluster of 3 units, - removing two units and re-adding them led to a situation where corosync - considers having 3 nodes online out of 5, instead of just 3 out of 3. + removing a unit and re-adding one led to a situation where corosync + considers having 3 nodes online out of 4, instead of just 3 out of 3. This test covers this scenario. """ principle_units = sorted(zaza.model.get_status().applications[ self._principle_app_name]['units'].keys()) self.assertEqual(len(principle_units), 3) - doomed_principle_units = principle_units[:2] - surviving_principle_unit = principle_units[2] - doomed_hacluster_units = juju_utils.get_subordinate_units( - doomed_principle_units, charm_name=self._hacluster_charm_name) + surviving_principle_unit = principle_units[0] + doomed_principle_unit = principle_units[1] surviving_hacluster_unit = juju_utils.get_subordinate_units( [surviving_principle_unit], charm_name=self._hacluster_charm_name)[0] + doomed_hacluster_unit = juju_utils.get_subordinate_units( + [doomed_principle_unit], + charm_name=self._hacluster_charm_name)[0] - for doomed_hacluster_unit in doomed_hacluster_units: - logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) - zaza.model.run_action( - doomed_hacluster_unit, - 'pause', - raise_on_failure=True) + logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) + zaza.model.run_action( + doomed_hacluster_unit, + 'pause', + raise_on_failure=True) - for doomed_principle_unit in doomed_principle_units: - logging.info('Removing {}'.format(doomed_principle_unit)) - zaza.model.destroy_unit( - self._principle_app_name, - doomed_principle_unit, - wait_disappear=True) + logging.info('Removing {}'.format(doomed_principle_unit)) + zaza.model.destroy_unit( + self._principle_app_name, + doomed_principle_unit, + wait_disappear=True) logging.info('Waiting for model to settle') zaza.model.block_until_unit_wl_status(surviving_hacluster_unit, 'blocked') - # NOTE(lourot): the principle unit (usually a keystone unit) isn't - # guaranteed to be blocked, so we don't validate that here. + # NOTE(lourot): the surviving principle units (usually keystone units) + # aren't guaranteed to be blocked, so we don't validate that here. zaza.model.block_until_all_units_idle() logging.info('Updating corosync ring') @@ -199,9 +198,8 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): action_params={'i-really-mean-it': True}, raise_on_failure=True) - for article in ('an', 'another'): - logging.info('Re-adding {} hacluster unit'.format(article)) - zaza.model.add_unit(self._principle_app_name, wait_appear=True) + logging.info('Re-adding an hacluster unit') + zaza.model.add_unit(self._principle_app_name, wait_appear=True) logging.info('Waiting for model to settle') # NOTE(lourot): the principle charm may remain blocked here. This seems @@ -215,7 +213,7 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): zaza.model.block_until_all_units_idle() # At this point if the corosync ring has been properly updated, there - # shouldn't be any trace of the deleted units anymore: + # shouldn't be any trace of the deleted unit anymore: logging.info('Checking that corosync considers all nodes to be online') cmd = 'sudo crm status' result = zaza.model.run_on_unit(surviving_hacluster_unit, cmd) From 0931ebb456574fb3b4b25344e3ed26861eca7429 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 3 Mar 2021 10:56:15 +0100 Subject: [PATCH 099/180] Fix docstring --- zaza/openstack/charm_tests/hacluster/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index b365960..66d4f45 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -150,7 +150,7 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): cls._hacluster_charm_name = test_config['hacluster-charm-name'] def test_930_scaleback(self): - """Remove two units, recalculate quorum and re-add two units. + """Remove one unit, recalculate quorum and re-add one unit. NOTE(lourot): before lp:1400481 was fixed, the corosync ring wasn't recalculated when removing units. So within a cluster of 3 units, From 2fefca5a406647a1efcd91e5f67441766f6976b7 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Tue, 2 Mar 2021 18:03:06 +0200 Subject: [PATCH 100/180] Code cleanup * Move general function out of the testing class. This will make them easy to be reused. * Properly organize the tests, and add tons of docstrings and comments to have them as clear as possible. * Add `failover_cinder_volume_host` to the Zaza `utilities/openstack.py`, since this is a general purpose function. --- .../charm_tests/ceph/rbd_mirror/tests.py | 776 ++++++++++++------ zaza/openstack/utilities/openstack.py | 34 + 2 files changed, 558 insertions(+), 252 deletions(-) diff --git a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py index 75a3f42..d8d7967 100644 --- a/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py +++ b/zaza/openstack/charm_tests/ceph/rbd_mirror/tests.py @@ -16,7 +16,6 @@ import json import logging import re -import time import cinderclient.exceptions as cinder_exceptions @@ -31,6 +30,129 @@ from zaza.openstack.charm_tests.glance.setup import ( CIRROS_IMAGE_NAME) +DEFAULT_CINDER_RBD_MIRRORING_MODE = 'pool' + + +def get_cinder_rbd_mirroring_mode(cinder_ceph_app_name='cinder-ceph'): + """Get the RBD mirroring mode for the Cinder Ceph pool. + + :param cinder_ceph_app_name: Cinder Ceph Juju application name. + :type cinder_ceph_app_name: str + :returns: A string representing the RBD mirroring mode. It can be + either 'pool' or 'image'. + :rtype: str + """ + rbd_mirroring_mode_config = zaza.model.get_application_config( + cinder_ceph_app_name).get('rbd-mirroring-mode') + if rbd_mirroring_mode_config: + rbd_mirroring_mode = rbd_mirroring_mode_config.get( + 'value', DEFAULT_CINDER_RBD_MIRRORING_MODE).lower() + else: + rbd_mirroring_mode = DEFAULT_CINDER_RBD_MIRRORING_MODE + + return rbd_mirroring_mode + + +def get_glance_image(glance): + """Get the Glance image object to be used by the Ceph tests. + + It looks for the Cirros Glance image, and it's returned if it's found. + If the Cirros image is not found, it will try and find the Ubuntu + LTS image. + + :param glance: Authenticated glanceclient + :type glance: glanceclient.Client + :returns: Glance image object + :rtype: glanceclient.image + """ + images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) + if images: + return images[0] + logging.info("Failed to find {} image, falling back to {}".format( + CIRROS_IMAGE_NAME, + LTS_IMAGE_NAME)) + return openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] + + +def setup_cinder_repl_volume_type(cinder, type_name='repl', + backend_name='cinder-ceph'): + """Set up the Cinder volume replication type. + + :param cinder: Authenticated cinderclient + :type cinder: cinder.Client + :param type_name: Cinder volume type name + :type type_name: str + :param backend_name: Cinder volume backend name with replication enabled. + :type backend_name: str + :returns: Cinder volume type object + :rtype: cinderclient.VolumeType + """ + try: + vol_type = cinder.volume_types.find(name=type_name) + except cinder_exceptions.NotFound: + vol_type = cinder.volume_types.create(type_name) + + vol_type.set_keys(metadata={ + 'volume_backend_name': backend_name, + 'replication_enabled': ' True', + }) + return vol_type + + +# TODO: This function should be incorporated into +# 'zaza.openstack.utilities.openstack.create_volume' helper, once the below +# flakiness comments are addressed. +def create_cinder_volume(cinder, name='zaza', image_id=None, type_id=None): + """Create a new Cinder volume. + + :param cinder: Authenticated cinderclient. + :type cinder: cinder.Client + :param name: Volume name. + :type name: str + :param image_id: Glance image id, if the volume is created from image. + :type image_id: str + :param type_id: Cinder Volume type id, if the volume needs to use an + explicit volume type. + :type type_id: boolean + :returns: Cinder volume + :rtype: :class:`Volume`. + """ + # NOTE(fnordahl): for some reason create volume from image often fails + # when run just after deployment is finished. We should figure out + # why, resolve the underlying issue and then remove this. + # + # We do not use tenacity here as it will interfere with tenacity used + # in ``resource_reaches_status`` + def create_volume(cinder, volume_params, retry=20): + if retry < 1: + return + volume = cinder.volumes.create(**volume_params) + try: + # Note(coreycb): stop_after_attempt is increased because using + # juju storage for ceph-osd backed by cinder on undercloud + # takes longer than the prior method of directory-backed OSD + # devices. + openstack.resource_reaches_status( + cinder.volumes, volume.id, msg='volume', + stop_after_attempt=20) + return volume + except AssertionError: + logging.info('retrying') + volume.delete() + return create_volume(cinder, volume_params, retry=retry - 1) + + volume_params = { + 'size': 8, + 'name': name, + } + if image_id: + volume_params['imageRef'] = image_id + if type_id: + volume_params['volume_type'] = type_id + + return create_volume(cinder, volume_params) + + class CephRBDMirrorBase(test_utils.OpenStackBaseTest): """Base class for ``ceph-rbd-mirror`` tests.""" @@ -38,6 +160,8 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run setup for ``ceph-rbd-mirror`` tests.""" super().setUpClass() + cls.cinder_ceph_app_name = 'cinder-ceph' + cls.test_cinder_volume_name = 'test-cinder-ceph-volume' # get ready for multi-model Zaza cls.site_a_model = cls.site_b_model = zaza.model.get_juju_model() cls.site_b_app_suffix = '-b' @@ -75,6 +199,21 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): model_name=self.site_b_model) return sorted(site_a_pools.keys()), sorted(site_b_pools.keys()) + def get_failover_pools(self): + """Get the failover Ceph pools' names, from both sites. + + If the Cinder RBD mirroring mode is 'image', the 'cinder-ceph' pool + needs to be excluded, since Cinder orchestrates the failover then. + + :returns: Tuple with site-a pools and site-b pools. + :rtype: Tuple[List[str], List[str]] + """ + site_a_pools, site_b_pools = self.get_pools() + if get_cinder_rbd_mirroring_mode(self.cinder_ceph_app_name) == 'image': + site_a_pools.remove(self.cinder_ceph_app_name) + site_b_pools.remove(self.cinder_ceph_app_name) + return site_a_pools, site_b_pools + def wait_for_mirror_state(self, state, application_name=None, model_name=None, check_entries_behind_master=False, @@ -136,118 +275,40 @@ class CephRBDMirrorBase(test_utils.OpenStackBaseTest): # all images with state has expected state return True - def get_cinder_rbd_mirroring_mode(self, - cinder_ceph_app_name='cinder-ceph'): - """Get the RBD mirroring mode for the Cinder Ceph pool. + def setup_test_cinder_volume(self): + """Set up the test Cinder volume into the Ceph RBD mirror environment. - :returns: A string representing the RBD mirroring mode. It can be - either 'pool' or 'image'. - """ - DEFAULT_RBD_MIRRORING_MODE = 'pool' + If the volume already exists, then it's returned. - rbd_mirroring_mode_config = zaza.model.get_application_config( - cinder_ceph_app_name).get('rbd-mirroring-mode') - if rbd_mirroring_mode_config: - rbd_mirroring_mode = rbd_mirroring_mode_config.get( - 'value', DEFAULT_RBD_MIRRORING_MODE).lower() - else: - rbd_mirroring_mode = DEFAULT_RBD_MIRRORING_MODE - - return rbd_mirroring_mode - - def create_cinder_volume(self, session, from_image=False): - """Create Cinder Volume from image. + Also, if the Cinder RBD mirroring mode is 'image', the volume will + use an explicit volume type with the appropriate replication flags. + Otherwise, it is just a simple Cinder volume using the default backend. + :returns: Cinder volume :rtype: :class:`Volume`. """ - def get_glance_image(session): - glance = openstack.get_glance_session_client(session) - images = openstack.get_images_by_name(glance, CIRROS_IMAGE_NAME) - if images: - return images[0] - logging.info("Failed to find {} image, falling back to {}".format( - CIRROS_IMAGE_NAME, - LTS_IMAGE_NAME)) - return openstack.get_images_by_name(glance, LTS_IMAGE_NAME)[0] + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session, version=3) - def create_volume_type(cinder): - try: - vol_type = cinder.volume_types.find(name='repl') - except cinder_exceptions.NotFound: - vol_type = cinder.volume_types.create('repl') - vol_type.set_keys(metadata={ - 'volume_backend_name': 'cinder-ceph', - 'replication_enabled': ' True', - }) - return vol_type + try: + return cinder.volumes.find(name=self.test_cinder_volume_name) + except cinder_exceptions.NotFound: + logging.info("Test Cinder volume doesn't exist. Creating it") - # NOTE(fnordahl): for some reason create volume from image often fails - # when run just after deployment is finished. We should figure out - # why, resolve the underlying issue and then remove this. - # - # We do not use tenacity here as it will interfere with tenacity used - # in ``resource_reaches_status`` - def create_volume(cinder, volume_params, retry=20): - if retry < 1: - return - volume = cinder.volumes.create(**volume_params) - try: - # Note(coreycb): stop_after_attempt is increased because using - # juju storage for ceph-osd backed by cinder on undercloud - # takes longer than the prior method of directory-backed OSD - # devices. - openstack.resource_reaches_status( - cinder.volumes, volume.id, msg='volume', - stop_after_attempt=20) - return volume - except AssertionError: - logging.info('retrying') - volume.delete() - return create_volume(cinder, volume_params, retry=retry - 1) - - volume_params = { - 'size': 8, - 'name': 'zaza', + glance = openstack.get_glance_session_client(session) + image = get_glance_image(glance) + kwargs = { + 'cinder': cinder, + 'name': self.test_cinder_volume_name, + 'image_id': image.id, } - if from_image: - volume_params['imageRef'] = get_glance_image(session).id - cinder = openstack.get_cinder_session_client(session) - if self.get_cinder_rbd_mirroring_mode() == 'image': - volume_params['volume_type'] = create_volume_type(cinder).id + if get_cinder_rbd_mirroring_mode(self.cinder_ceph_app_name) == 'image': + volume_type = setup_cinder_repl_volume_type( + cinder, + backend_name=self.cinder_ceph_app_name) + kwargs['type_id'] = volume_type.id - return create_volume(cinder, volume_params) - - def failover_cinder_volume_host(self, cinder_client, - backend_name='cinder-ceph', - target_backend_id='ceph', - target_status='disabled', - target_replication_status='failed-over', - timeout=300): - """Failover Cinder volume host.""" - host = 'cinder@{}'.format(backend_name) - logging.info( - 'Failover Cinder host %s to backend_id %s', - host, target_backend_id) - cinder_client.services.failover_host( - host=host, - backend_id=target_backend_id) - start = time.time() - while True: - elapsed = time.time() - start - if elapsed > timeout: - raise cinder_exceptions.TimeoutException( - obj=cinder_client.services, - action='failover_host') - service = cinder_client.services.list( - host=host, - binary='cinder-volume')[0] - if (service.status == target_status and - service.replication_status == target_replication_status): - break - time.sleep(5) - logging.info( - 'Successfully failed-over Cinder host %s to backend_id %s', - host, target_backend_id) + return create_cinder_volume(**kwargs) class CephRBDMirrorTest(CephRBDMirrorBase): @@ -320,8 +381,7 @@ class CephRBDMirrorTest(CephRBDMirrorBase): site B and subsequently comparing the contents we get a full end to end test. """ - session = openstack.get_overcloud_keystone_session() - volume = self.create_cinder_volume(session, from_image=True) + volume = self.setup_test_cinder_volume() site_a_hash = zaza.openstack.utilities.ceph.get_rbd_hash( zaza.model.get_lead_unit_name('ceph-mon', model_name=self.site_a_model), @@ -333,6 +393,8 @@ class CephRBDMirrorTest(CephRBDMirrorBase): check_entries_behind_master=True, application_name=self.application_name + self.site_b_app_suffix, model_name=self.site_b_model) + logging.info('Checking the Ceph RBD hashes of the primary and ' + 'the secondary Ceph images') site_b_hash = zaza.openstack.utilities.ceph.get_rbd_hash( zaza.model.get_lead_unit_name('ceph-mon' + self.site_b_app_suffix, model_name=self.site_b_model), @@ -347,189 +409,399 @@ class CephRBDMirrorTest(CephRBDMirrorBase): class CephRBDMirrorControlledFailoverTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` controlled failover tests.""" - def cinder_fail_over_fall_back(self): - """Validate controlled fail over and fall back via the Cinder API.""" + def execute_failover_juju_actions(self, + primary_site_app_name, + primary_site_model, + primary_site_pools, + secondary_site_app_name, + secondary_site_model, + secondary_site_pools): + """Execute the failover Juju actions. + + The failover / failback via Juju actions shares the same workflow. The + failback is just a failover with sites in reversed order. + + This function encapsulates the tasks to failover a primary site to + a secondary site: + 1. Demote primary site + 2. Validation of the primary site demotion + 3. Promote secondary site + 4. Validation of the secondary site promotion + + :param primary_site_app_name: Primary site Ceph RBD mirror app name. + :type primary_site_app_name: str + :param primary_site_model: Primary site Juju model name. + :type primary_site_model: str + :param primary_site_pools: Primary site pools. + :type primary_site_pools: List[str] + :param secondary_site_app_name: Secondary site Ceph RBD mirror + app name. + :type secondary_site_app_name: str + :param secondary_site_model: Secondary site Juju model name. + :type secondary_site_model: str + :param secondary_site_pools: Secondary site pools. + :type secondary_site_pools: List[str] + """ + # Check if primary and secondary pools sizes are the same. + self.assertEqual(len(primary_site_pools), len(secondary_site_pools)) + + # Run the 'demote' Juju action against the primary site pools. + logging.info('Demoting {} from model {}.'.format( + primary_site_app_name, primary_site_model)) + result = zaza.model.run_action_on_leader( + primary_site_app_name, + 'demote', + model_name=primary_site_model, + action_params={ + 'pools': ','.join(primary_site_pools) + }) + logging.info(result.results) + self.assertEqual(int(result.results['Code']), 0) + + # Validate that the demoted pools count matches the total primary site + # pools count. + n_pools_demoted = len(result.results['output'].split('\n')) + self.assertEqual(len(primary_site_pools), n_pools_demoted) + + # At this point, both primary and secondary sites are demoted. Validate + # that the Ceph images, from both sites, report 'up+unknown', since + # there isn't a primary site at the moment. + logging.info('Waiting until {} is demoted.'.format( + primary_site_app_name)) + self.wait_for_mirror_state( + 'up+unknown', + application_name=primary_site_app_name, + model_name=primary_site_model, + pools=primary_site_pools) + self.wait_for_mirror_state( + 'up+unknown', + application_name=secondary_site_app_name, + model_name=secondary_site_model, + pools=secondary_site_pools) + + # Run the 'promote' Juju against the secondary site. + logging.info('Promoting {} from model {}.'.format( + secondary_site_app_name, secondary_site_model)) + result = zaza.model.run_action_on_leader( + secondary_site_app_name, + 'promote', + model_name=secondary_site_model, + action_params={ + 'pools': ','.join(secondary_site_pools) + }) + logging.info(result.results) + self.assertEqual(int(result.results['Code']), 0) + + # Validate that the promoted pools count matches the total secondary + # site pools count. + n_pools_promoted = len(result.results['output'].split('\n')) + self.assertEqual(len(secondary_site_pools), n_pools_promoted) + + # Validate that the Ceph images from the newly promoted site + # report 'up+stopped' state (which is reported by primary Ceph images). + logging.info('Waiting until {} is promoted.'.format( + secondary_site_app_name)) + self.wait_for_mirror_state( + 'up+stopped', + application_name=secondary_site_app_name, + model_name=secondary_site_model, + pools=secondary_site_pools) + + # Validate that the Ceph images from site-a report 'up+replaying' + # (which is reported by secondary Ceph images). + self.wait_for_mirror_state( + 'up+replaying', + check_entries_behind_master=True, + application_name=primary_site_app_name, + model_name=primary_site_model, + pools=primary_site_pools) + + def test_100_cinder_failover(self): + """Validate controlled failover via the Cinder API. + + This test only makes sense if Cinder RBD mirroring mode is 'image'. + It will return early, if this is not the case. + """ + cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode( + self.cinder_ceph_app_name) + if cinder_rbd_mirroring_mode != 'image': + logging.warning( + "Skipping 'test_100_cinder_failover' since Cinder RBD " + "mirroring mode is {}.".format(cinder_rbd_mirroring_mode)) + return + session = openstack.get_overcloud_keystone_session() - cinder = openstack.get_cinder_session_client(session) - volume = self.create_cinder_volume(session, from_image=True) + cinder = openstack.get_cinder_session_client(session, version=3) + + # Check if the Cinder volume host is available with replication + # enabled. + host = 'cinder@{}'.format(self.cinder_ceph_app_name) + svc = cinder.services.list(host=host, binary='cinder-volume')[0] + self.assertEqual(svc.replication_status, 'enabled') + self.assertEqual(svc.status, 'enabled') + + # Setup the test Cinder volume + volume = self.setup_test_cinder_volume() + + # Check if the volume is properly mirrored self.wait_for_mirror_state( 'up+replaying', check_entries_behind_master=True, application_name=self.application_name + self.site_b_app_suffix, model_name=self.site_b_model, - pools=['cinder-ceph']) - self.failover_cinder_volume_host( - cinder_client=cinder) + pools=[self.cinder_ceph_app_name]) + + # Execute the Cinder volume failover + openstack.failover_cinder_volume_host( + cinder=cinder, + backend_name=self.cinder_ceph_app_name, + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over') + + # Check if the test volume is still available after failover self.assertEqual(cinder.volumes.get(volume.id).status, 'available') - self.failover_cinder_volume_host( - cinder_client=cinder, + + def test_101_cinder_failback(self): + """Validate controlled failback via the Cinder API. + + This test only makes sense if Cinder RBD mirroring mode is 'image'. + It will return early, if this is not the case. + + The test needs to be executed when the Cinder volume host is already + failed-over with the test volume on it. + """ + cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode( + self.cinder_ceph_app_name) + if cinder_rbd_mirroring_mode != 'image': + logging.warning( + "Skipping 'test_101_cinder_failback' since Cinder RBD " + "mirroring mode is {}.".format(cinder_rbd_mirroring_mode)) + return + + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session, version=3) + + # Check if the Cinder volume host is already failed-over + host = 'cinder@{}'.format(self.cinder_ceph_app_name) + svc = cinder.services.list(host=host, binary='cinder-volume')[0] + self.assertEqual(svc.replication_status, 'failed-over') + self.assertEqual(svc.status, 'disabled') + + # Check if the test Cinder volume is already present. The method + # 'cinder.volumes.find' raises 404 if the volume is not found. + volume = cinder.volumes.find(name=self.test_cinder_volume_name) + + # Execute the Cinder volume failback + openstack.failover_cinder_volume_host( + cinder=cinder, + backend_name=self.cinder_ceph_app_name, target_backend_id='default', target_status='enabled', target_replication_status='enabled') + + # Check if the test volume is still available after failback self.assertEqual(cinder.volumes.get(volume.id).status, 'available') - def test_fail_over_fall_back(self): - """Validate controlled fail over and fall back.""" - site_a_pools, site_b_pools = self.get_pools() - site_a_action_params = {} - site_b_action_params = {} - if self.get_cinder_rbd_mirroring_mode() == 'image': - site_a_pools.remove('cinder-ceph') - site_a_action_params['pools'] = ','.join(site_a_pools) - site_b_pools.remove('cinder-ceph') - site_b_action_params['pools'] = ','.join(site_b_pools) + def test_200_juju_failover(self): + """Validate controlled failover via Juju actions.""" + # Get the Ceph pools needed to failover + site_a_pools, site_b_pools = self.get_failover_pools() + + # Execute the failover Juju actions with the appropriate parameters. + site_b_app_name = self.application_name + self.site_b_app_suffix + self.execute_failover_juju_actions( + primary_site_app_name=self.application_name, + primary_site_model=self.site_a_model, + primary_site_pools=site_a_pools, + secondary_site_app_name=site_b_app_name, + secondary_site_model=self.site_b_model, + secondary_site_pools=site_b_pools) + + def test_201_juju_failback(self): + """Validate controlled failback via Juju actions.""" + # Get the Ceph pools needed to failback + site_a_pools, site_b_pools = self.get_failover_pools() + + # Execute the failover Juju actions with the appropriate parameters. + # The failback operation is just a failover with sites in reverse + # order. + site_b_app_name = self.application_name + self.site_b_app_suffix + self.execute_failover_juju_actions( + primary_site_app_name=site_b_app_name, + primary_site_model=self.site_b_model, + primary_site_pools=site_b_pools, + secondary_site_app_name=self.application_name, + secondary_site_model=self.site_a_model, + secondary_site_pools=site_a_pools) + + def test_203_juju_resync(self): + """Validate the 'resync-pools' Juju action. + + The 'resync-pools' Juju action is meant to flag Ceph images from the + secondary site to re-sync against the Ceph images from the primary + site. + + This use case is useful when the Ceph secondary images are out of sync. + """ + # Get the Ceph pools needed to failback + _, site_b_pools = self.get_failover_pools() + + # Run the 'resync-pools' Juju action against the pools from site-b. + # This will make sure that the Ceph images from site-b are properly + # synced with the primary images from site-a. + site_b_app_name = self.application_name + self.site_b_app_suffix + logging.info('Re-syncing {} from model {}'.format( + site_b_app_name, self.site_b_model)) result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror', - 'demote', - model_name=self.site_a_model, - action_params=site_a_action_params) - logging.info(result.results) - n_pools_demoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_a_pools), n_pools_demoted) - self.wait_for_mirror_state( - 'up+unknown', - model_name=self.site_a_model, - pools=site_a_pools) - self.wait_for_mirror_state( - 'up+unknown', - application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model, - pools=site_b_pools) - result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, - 'promote', - model_name=self.site_b_model, - action_params=site_b_action_params) - logging.info(result.results) - n_pools_promoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_b_pools), n_pools_promoted) - self.wait_for_mirror_state( - 'up+replaying', - model_name=self.site_a_model, - pools=site_a_pools) - self.wait_for_mirror_state( - 'up+stopped', - application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model, - pools=site_b_pools) - result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, - 'demote', - model_name=self.site_b_model, - action_params=site_b_action_params) - logging.info(result.results) - n_pools_demoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_a_pools), n_pools_demoted) - self.wait_for_mirror_state( - 'up+unknown', - model_name=self.site_a_model, - pools=site_a_pools) - self.wait_for_mirror_state( - 'up+unknown', - application_name=self.application_name + self.site_b_app_suffix, - model_name=self.site_b_model, - pools=site_b_pools) - result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror', - 'promote', - model_name=self.site_a_model, - action_params=site_a_action_params) - logging.info(result.results) - n_pools_promoted = len(result.results['output'].split('\n')) - self.assertEqual(len(site_b_pools), n_pools_promoted) - self.wait_for_mirror_state( - 'up+stopped', - model_name=self.site_a_model, - pools=site_a_pools) - action_params = { - 'i-really-mean-it': True, - } - if self.get_cinder_rbd_mirroring_mode() == 'image': - action_params['pools'] = site_b_action_params['pools'] - result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, + site_b_app_name, 'resync-pools', model_name=self.site_b_model, - action_params=action_params) + action_params={ + 'pools': ','.join(site_b_pools), + 'i-really-mean-it': True, + }) logging.info(result.results) + self.assertEqual(int(result.results['Code']), 0) + + # Validate that the Ceph images from site-b report 'up+replaying' + # (which is reported by secondary Ceph images). And check that images + # exist in Cinder and Glance pools. self.wait_for_mirror_state( 'up+replaying', - application_name=self.application_name + self.site_b_app_suffix, + check_entries_behind_master=True, + application_name=site_b_app_name, model_name=self.site_b_model, - require_images_in=['cinder-ceph', 'glance'], - pools=site_a_pools) - if self.get_cinder_rbd_mirroring_mode() == 'image': - self.cinder_fail_over_fall_back() + require_images_in=[self.cinder_ceph_app_name, 'glance'], + pools=site_b_pools) class CephRBDMirrorDisasterFailoverTest(CephRBDMirrorBase): """Encapsulate ``ceph-rbd-mirror`` destructive tests.""" - def forced_failover_cinder_volume_host(self, cinder_client): - """Validate forced Cinder volume host fail over.""" - def apply_cinder_workaround(): - """Set minimal timeouts / retries to the Cinder Ceph backend. + def apply_cinder_ceph_workaround(self): + """Set minimal timeouts / retries to the Cinder Ceph backend. - This is needed because the failover via Cinder will try to do a - demotion of the site-a, and with the default timeouts / retries, - the operation takes an unreasonably amount of time. - """ - cinder_configs = { - 'rados_connect_timeout': '1', - 'rados_connection_retries': '1', - 'rados_connection_interval': '0', - 'replication_connect_timeout': '1', - } - update_cinder_conf_cmd = ( - "import configparser; " - "config = configparser.ConfigParser(); " - "config.read('/etc/cinder/cinder.conf'); " - "{}" - "f = open('/etc/cinder/cinder.conf', 'w'); " - "config.write(f); " - "f.close()") - cmd = '' - for config in cinder_configs: - cmd += "config.set('cinder-ceph', '{0}', '{1}'); ".format( - config, cinder_configs[config]) - cmd = update_cinder_conf_cmd.format(cmd) - zaza.model.run_on_leader( - 'cinder-ceph', - 'python3 -c "{}"; systemctl restart cinder-volume'.format(cmd)) + This is needed because the failover via Cinder API will try to do a + demotion of the site-a. However, when site-a is down, and with the + default timeouts / retries, the operation takes an unreasonably amount + of time (or sometimes it never finishes). + """ + # These new config options need to be set under the Cinder Ceph backend + # section in the main Cinder config file. + # At the moment, we don't the possibility of using Juju config to set + # these options. And also, it's not even a good practice to have them + # in production. + # These should be set only to do the Ceph failover via Cinder API, and + # they need to be removed after. + configs = { + 'rados_connect_timeout': '1', + 'rados_connection_retries': '1', + 'rados_connection_interval': '0', + 'replication_connect_timeout': '1', + } - apply_cinder_workaround() - self.failover_cinder_volume_host(cinder_client) + # Small Python script that will be executed via Juju run to update + # the Cinder config file. + update_cinder_conf_script = ( + "import configparser; " + "config = configparser.ConfigParser(); " + "config.read('/etc/cinder/cinder.conf'); " + "{}" + "f = open('/etc/cinder/cinder.conf', 'w'); " + "config.write(f); " + "f.close()") + set_cmd = '' + for cfg_name in configs: + set_cmd += "config.set('{0}', '{1}', '{2}'); ".format( + self.cinder_ceph_app_name, cfg_name, configs[cfg_name]) + script = update_cinder_conf_script.format(set_cmd) - for volume in cinder_client.volumes.list(): - self.assertEqual(volume.status, 'available') + # Run the workaround script via Juju run + zaza.model.run_on_leader( + self.cinder_ceph_app_name, + 'python3 -c "{}"; systemctl restart cinder-volume'.format(script)) - def test_kill_site_a_fail_over(self): - """Validate fail over after uncontrolled shutdown of primary.""" - action_params = {} - if self.get_cinder_rbd_mirroring_mode() == 'image': - _, site_b_pools = self.get_pools() - site_b_pools.remove('cinder-ceph') - action_params['pools'] = ','.join(site_b_pools) - - for application in 'ceph-rbd-mirror', 'ceph-mon', 'ceph-osd': + def kill_primary_site(self): + """Simulate an unexpected primary site shutdown.""" + logging.info('Killing the Ceph primary site') + for application in ['ceph-rbd-mirror', 'ceph-mon', 'ceph-osd']: zaza.model.remove_application( application, model_name=self.site_a_model, forcefully_remove_machines=True) + + def test_100_forced_juju_failover(self): + """Validate Ceph failover via Juju when the primary site is down. + + * Kill the primary site + * Execute the forced failover via Juju actions + """ + # Get the site-b Ceph pools that need to be promoted + _, site_b_pools = self.get_failover_pools() + site_b_app_name = self.application_name + self.site_b_app_suffix + + # Simulate primary site unexpected shutdown + self.kill_primary_site() + + # Try and promote the site-b to primary. result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, + site_b_app_name, 'promote', model_name=self.site_b_model, - action_params=action_params) + action_params={ + 'pools': ','.join(site_b_pools), + }) + self.assertEqual(int(result.results['Code']), 0) + + # The site-b 'promote' Juju action is expected to fail, because the + # primary site is down. self.assertEqual(result.status, 'failed') - action_params['force'] = True + + # Retry to promote site-b using the 'force' Juju action parameter. result = zaza.model.run_action_on_leader( - 'ceph-rbd-mirror' + self.site_b_app_suffix, + site_b_app_name, 'promote', model_name=self.site_b_model, - action_params=action_params) + action_params={ + 'force': True, + 'pools': ','.join(site_b_pools), + }) + self.assertEqual(int(result.results['Code']), 0) + + # Validate successful Juju action execution self.assertEqual(result.status, 'completed') - if self.get_cinder_rbd_mirroring_mode() == 'image': - session = openstack.get_overcloud_keystone_session() - cinder = openstack.get_cinder_session_client(session) - self.forced_failover_cinder_volume_host(cinder) + + def test_200_forced_cinder_failover(self): + """Validate Ceph failover via Cinder when the primary site is down. + + This test only makes sense if Cinder RBD mirroring mode is 'image'. + It will return early, if this is not the case. + + This assumes that the primary site is already killed. + """ + cinder_rbd_mirroring_mode = get_cinder_rbd_mirroring_mode( + self.cinder_ceph_app_name) + if cinder_rbd_mirroring_mode != 'image': + logging.warning( + "Skipping 'test_200_cinder_failover_without_primary_site' " + "since Cinder RBD mirroring mode is {}.".format( + cinder_rbd_mirroring_mode)) + return + + # Make sure that the Cinder Ceph backend workaround is applied. + self.apply_cinder_ceph_workaround() + + session = openstack.get_overcloud_keystone_session() + cinder = openstack.get_cinder_session_client(session, version=3) + openstack.failover_cinder_volume_host( + cinder=cinder, + backend_name=self.cinder_ceph_app_name, + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over') + + # Check that the Cinder volumes are still available after forced + # failover. + for volume in cinder.volumes.list(): + self.assertEqual(volume.status, 'available') diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index d277656..4ed75b9 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2489,6 +2489,40 @@ def attach_volume(nova, volume_id, instance_id): device='/dev/vdx') +def failover_cinder_volume_host(cinder, backend_name='cinder-ceph', + target_backend_id='ceph', + target_status='disabled', + target_replication_status='failed-over'): + """Failover Cinder volume host with replication enabled. + + :param cinder: Authenticated cinderclient + :type cinder: cinder.Client + :param backend_name: Cinder volume backend name with + replication enabled. + :type backend_name: str + :param target_backend_id: Failover target Cinder backend id. + :type target_backend_id: str + :param target_status: Target Cinder volume status after failover. + :type target_status: str + :param target_replication_status: Target Cinder volume replication + status after failover. + :type target_replication_status: str + :raises: AssertionError + """ + host = 'cinder@{}'.format(backend_name) + logging.info('Failover Cinder volume host %s to backend_id %s', + host, target_backend_id) + cinder.services.failover_host(host=host, backend_id=target_backend_id) + for attempt in tenacity.Retrying( + retry=tenacity.retry_if_exception_type(AssertionError), + stop=tenacity.stop_after_attempt(10), + wait=tenacity.wait_exponential(multiplier=1, min=2, max=10)): + with attempt: + svc = cinder.services.list(host=host, binary='cinder-volume')[0] + assert svc.status == target_status + assert svc.replication_status == target_replication_status + + def create_volume_backup(cinder, volume_id, name=None): """Create cinder volume backup. From 73c78a0db3c2a1b89314405bf83540d7b0444509 Mon Sep 17 00:00:00 2001 From: Bartosz Woronicz Date: Tue, 2 Mar 2021 00:33:55 +0100 Subject: [PATCH 101/180] add reload and restart action tests for vault --- zaza/openstack/charm_tests/vault/tests.py | 54 ++++++++++++++++++++++- zaza/openstack/charm_tests/vault/utils.py | 14 ++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 68130a0..219291e 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -244,8 +244,6 @@ class VaultTest(BaseVaultTest): Pause service and check services are stopped, then resume and check they are started. """ - # Restarting vault process will set it as sealed so it's - # important to have the test executed at the end. vault_actions = zaza.model.get_actions( 'vault') if 'pause' not in vault_actions or 'resume' not in vault_actions: @@ -257,6 +255,58 @@ class VaultTest(BaseVaultTest): lead_client = vault_utils.extract_lead_unit_client(self.clients) self.assertTrue(lead_client.hvac_client.seal_status['sealed']) + def test_vault_reload(self): + """Run reload tests. + + Reload service and check services were restarted + by doing simple change in the running config by API. + Then confirm that service is not sealed + """ + vault_actions = zaza.model.get_actions( + 'vault') + if 'reload' not in vault_actions: + raise unittest.SkipTest("The version of charm-vault tested does " + "not have reload action") + + lead_client = vault_utils.extract_lead_unit_client(self.clients) + running_config = vault_utils.get_running_config(lead_client) + value_to_set = not running_config['data']['disable_mlock'] + + zaza.model.set_application_config( + 'vault', + {'disable-mlock': str(value_to_set)}) + + logging.info("Testing reload") + zaza.model.run_action_on_leader( + 'vault', + 'reload', + action_params={}) + + self.assertEqual( + value_to_set, + vault_utils.get_running_config(lead_client)[ + 'data']['disable_mlock']) + self.assertFalse(lead_client.hvac_client.seal_status['sealed']) + + def test_vault_restart(self): + """Run pause and resume tests. + + Restart service and check services are started. + """ + vault_actions = zaza.model.get_actions( + 'vault') + if 'restart' not in vault_actions: + raise unittest.SkipTest("The version of charm-vault tested does " + "not have restart action") + logging.info("Testing restart") + zaza.model.run_action_on_leader( + 'vault', + 'restart', + action_params={}) + + lead_client = vault_utils.extract_lead_unit_client(self.clients) + self.assertTrue(lead_client.hvac_client.seal_status['sealed']) + if __name__ == '__main__': unittest.main() diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 9814243..9098b12 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -137,6 +137,20 @@ def get_vip_client(cacert=None): return client +def get_running_config(client): + """Get Vault running config. + + :param client: Client to use for initiliasation + :type client: CharmVaultClient + The hvac library does not support getting info + from endpoint /v1/sys/config/state/sanitized + Therefore we implement it here + """ + return requests.get( + client.hvac_client.adapter.base_uri + '/v1/sys/config/state/sanitized', + headers={'X-Vault-Token': client.hvac_client.token}).json() + + def init_vault(client, shares=1, threshold=1): """Initialise vault. From 9901248f7982481c5434caaa93175e9b1f549fff Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Mon, 1 Mar 2021 12:20:06 +0200 Subject: [PATCH 102/180] Refactor Manila Ganesha test The current Manila Ganesha `test_manila_share` had a lot of functionality that can be re-used to any configured Manila backend. It's a good idea to have this functionality generalized into `zaza.manila.tests.ManilaBaseTest`, which can be reused for any tested Manila backend. --- zaza/openstack/charm_tests/manila/tests.py | 264 ++++++++++++++++++ .../charm_tests/manila_ganesha/setup.py | 5 +- .../charm_tests/manila_ganesha/tests.py | 118 +------- 3 files changed, 276 insertions(+), 111 deletions(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index c00b993..e7cba8f 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -16,12 +16,47 @@ """Encapsulate Manila testing.""" +import logging import tenacity from manilaclient import client as manilaclient import zaza.model +import zaza.openstack.configure.guest as guest +import zaza.openstack.utilities.openstack as openstack_utils import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.charm_tests.nova.utils as nova_utils +import zaza.openstack.charm_tests.neutron.tests as neutron_tests + + +def verify_status(stdin, stdout, stderr): + """Callable to verify the command output. + + It checks if the command successfully executed. + + This is meant to be given as parameter 'verify' to the helper function + 'openstack_utils.ssh_command'. + """ + status = stdout.channel.recv_exit_status() + if status: + logging.info("{}".format(stderr.readlines()[0].strip())) + assert status == 0 + + +def verify_manila_testing_file(stdin, stdout, stderr): + """Callable to verify the command output. + + It checks if the command successfully executed, and it validates the + testing file written on the Manila share. + + This is meant to be given as parameter 'verify' to the helper function + 'openstack_utils.ssh_command'. + """ + verify_status(stdin, stdout, stderr) + out = "" + for line in iter(stdout.readline, ""): + out += line + assert out == "test\n" class ManilaTests(test_utils.OpenStackBaseTest): @@ -49,3 +84,232 @@ class ManilaTests(test_utils.OpenStackBaseTest): wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) def _list_shares(self): return self.manila_client.shares.list() + + +class ManilaBaseTest(test_utils.OpenStackBaseTest): + """Encapsulate a Manila basic functionality test.""" + + RESOURCE_PREFIX = 'zaza-manilatests' + INSTANCE_KEY = 'bionic' + INSTANCE_USERDATA = """#cloud-config +packages: +- nfs-common +""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaBaseTest, cls).setUpClass() + cls.nova_client = openstack_utils.get_nova_session_client( + session=cls.keystone_session) + cls.manila_client = manilaclient.Client( + session=cls.keystone_session, client_version='2') + cls.share_name = 'test-manila-share' + cls.share_type_name = 'default_share_type' + cls.share_protocol = 'nfs' + cls.mount_dir = '/mnt/manila_share' + cls.share_network = None + + @classmethod + def tearDownClass(cls): + """Run class teardown after tests finished.""" + # Cleanup Nova servers + logging.info('Cleaning up test Nova servers') + fips_reservations = [] + for vm in cls.nova_client.servers.list(): + fips_reservations += neutron_tests.floating_ips_from_instance(vm) + vm.delete() + openstack_utils.resource_removed( + cls.nova_client.servers, + vm.id, + msg="Waiting for the Nova VM {} to be deleted".format(vm.name)) + + # Delete FiPs reservations + logging.info('Cleaning up test FiPs reservations') + neutron = openstack_utils.get_neutron_session_client( + session=cls.keystone_session) + for fip in neutron.list_floatingips()['floatingips']: + if fip['floating_ip_address'] in fips_reservations: + neutron.delete_floatingip(fip['id']) + + # Cleanup Manila shares + logging.info('Cleaning up test shares') + for share in cls.manila_client.shares.list(): + share.delete() + openstack_utils.resource_removed( + cls.manila_client.shares, + share.id, + msg="Waiting for the Manila share {} to be deleted".format( + share.name)) + + # Cleanup test Manila share servers (spawned by the driver when DHSS + # is enabled). + logging.info('Cleaning up test shares servers (if found)') + for server in cls.manila_client.share_servers.list(): + server.delete() + openstack_utils.resource_removed( + cls.manila_client.share_servers, + server.id, + msg="Waiting for the share server {} to be deleted".format( + server.id)) + + def _get_mount_options(self): + """Get the appropriate mount options used to mount the Manila share. + + :returns: The proper mount options flags for the share protocol. + :rtype: string + """ + if self.share_protocol == 'nfs': + return 'nfsvers=4.1,proto=tcp' + else: + raise NotImplementedError( + 'Share protocol not supported yet: {}'.format( + self.share_protocol)) + + def _mount_share_on_instance(self, instance_ip, ssh_user_name, + ssh_private_key, share_path): + """Mount a share into a Nova instance. + + The mount command is executed via SSH. + + :param instance_ip: IP of the Nova instance. + :type instance_ip: string + :param ssh_user_name: SSH user name. + :type ssh_user_name: string + :param ssh_private_key: SSH private key. + :type ssh_private_key: string + :param share_path: Share network path. + :type share_path: string + """ + ssh_cmd = ( + 'sudo mkdir -p {0} && ' + 'sudo mount -t {1} -o {2} {3} {0}'.format( + self.mount_dir, + self.share_protocol, + self._get_mount_options(), + share_path)) + + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)): + with attempt: + openstack_utils.ssh_command( + vm_name="instance-{}".format(instance_ip), + ip=instance_ip, + username=ssh_user_name, + privkey=ssh_private_key, + command=ssh_cmd, + verify=verify_status) + + @tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) + def _write_testing_file_on_instance(self, instance_ip, ssh_user_name, + ssh_private_key): + """Write a file on a Manila share mounted into a Nova instance. + + Mount the Manila share in the given Nova instance, and write a testing + file on it (which is meant to be validated from another instance). + These commands are executed via SSH. + + :param instance_ip: IP of the Nova instance. + :type instance_ip: string + :param ssh_user_name: SSH user name. + :type ssh_user_name: string + :param ssh_private_key: SSH private key. + :type ssh_private_key: string + """ + openstack_utils.ssh_command( + vm_name="instance-{}".format(instance_ip), + ip=instance_ip, + username=ssh_user_name, + privkey=ssh_private_key, + command='echo "test" | sudo tee {}/test'.format( + self.mount_dir), + verify=verify_status) + + @tenacity.retry( + stop=tenacity.stop_after_attempt(5), + wait=tenacity.wait_exponential(multiplier=3, min=2, max=10)) + def _validate_testing_file_from_instance(self, instance_ip, ssh_user_name, + ssh_private_key): + """Validate a file from the Manila share mounted into a Nova instance. + + This is meant to run after the testing file was already written into + another Nova instance. It validates the written file. The commands are + executed via SSH. + + :param instance_ip: IP of the Nova instance. + :type instance_ip: string + :param ssh_user_name: SSH user name. + :type ssh_user_name: string + :param ssh_private_key: SSH private key. + :type ssh_private_key: string + """ + openstack_utils.ssh_command( + vm_name="instance-{}".format(instance_ip), + ip=instance_ip, + username=ssh_user_name, + privkey=ssh_private_key, + command='sudo cat {}/test'.format(self.mount_dir), + verify=verify_manila_testing_file) + + def test_manila_share(self): + """Test that a Manila share can be accessed on two instances. + + 1. Create a share + 2. Spawn two servers + 3. Mount it on both + 4. Write a file on one + 5. Read it on the other + 6. Profit + """ + # Create a share + share = self.manila_client.shares.create( + share_type=self.share_type_name, + name=self.share_name, + share_proto=self.share_protocol, + share_network=self.share_network, + size=1) + + # Spawn Servers + instance_1 = self.launch_guest( + guest_name='ins-1', + userdata=self.INSTANCE_USERDATA, + instance_key=self.INSTANCE_KEY) + instance_2 = self.launch_guest( + guest_name='ins-2', + userdata=self.INSTANCE_USERDATA, + instance_key=self.INSTANCE_KEY) + + fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] + fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] + + # Wait for the created share to become available before it gets used. + openstack_utils.resource_reaches_status( + self.manila_client.shares, + share.id, + wait_iteration_max_time=120, + stop_after_attempt=2, + expected_status="available", + msg="Waiting for a share to become available") + + # Grant access to the Manila share for both Nova instances. + share.allow(access_type='ip', access=fip_1, access_level='rw') + share.allow(access_type='ip', access=fip_2, access_level='rw') + + ssh_user_name = guest.boot_tests[self.INSTANCE_KEY]['username'] + privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) + share_path = share.export_locations[0] + + # Write a testing file on instance #1 + self._mount_share_on_instance( + fip_1, ssh_user_name, privkey, share_path) + self._write_testing_file_on_instance( + fip_1, ssh_user_name, privkey) + + # Validate the testing file from instance #2 + self._mount_share_on_instance( + fip_2, ssh_user_name, privkey, share_path) + self._validate_testing_file_from_instance( + fip_2, ssh_user_name, privkey) diff --git a/zaza/openstack/charm_tests/manila_ganesha/setup.py b/zaza/openstack/charm_tests/manila_ganesha/setup.py index d2b694e..a804958 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/setup.py +++ b/zaza/openstack/charm_tests/manila_ganesha/setup.py @@ -23,6 +23,9 @@ import zaza.openstack.utilities.openstack as openstack_utils from manilaclient import client as manilaclient +MANILA_GANESHA_TYPE_NAME = "cephfsnfstype" + + def setup_ganesha_share_type(manila_client=None): """Create a share type for manila with Ganesha. @@ -35,7 +38,7 @@ def setup_ganesha_share_type(manila_client=None): session=keystone_session, client_version='2') manila_client.share_types.create( - name="cephfsnfstype", spec_driver_handles_share_servers=False, + name=MANILA_GANESHA_TYPE_NAME, spec_driver_handles_share_servers=False, extra_specs={ 'vendor_name': 'Ceph', 'storage_protocol': 'NFS', diff --git a/zaza/openstack/charm_tests/manila_ganesha/tests.py b/zaza/openstack/charm_tests/manila_ganesha/tests.py index 02ea825..f5d7b63 100644 --- a/zaza/openstack/charm_tests/manila_ganesha/tests.py +++ b/zaza/openstack/charm_tests/manila_ganesha/tests.py @@ -16,122 +16,20 @@ """Encapsulate Manila Ganesha testing.""" -import logging +from zaza.openstack.charm_tests.manila_ganesha.setup import ( + MANILA_GANESHA_TYPE_NAME, +) -from tenacity import Retrying, stop_after_attempt, wait_exponential - -from manilaclient import client as manilaclient - -import zaza.openstack.charm_tests.neutron.tests as neutron_tests -import zaza.openstack.charm_tests.nova.utils as nova_utils -import zaza.openstack.charm_tests.test_utils as test_utils -import zaza.openstack.configure.guest as guest -import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.manila.tests as manila_tests -class ManilaGaneshaTests(test_utils.OpenStackBaseTest): +class ManilaGaneshaTests(manila_tests.ManilaBaseTest): """Encapsulate Manila Ganesha tests.""" - RESOURCE_PREFIX = 'zaza-manilatests' - INSTANCE_USERDATA = """#cloud-config -packages: -- nfs-common -""" - @classmethod def setUpClass(cls): """Run class setup for running tests.""" super(ManilaGaneshaTests, cls).setUpClass() - cls.nova_client = ( - openstack_utils.get_nova_session_client(cls.keystone_session)) - cls.manila_client = manilaclient.Client( - session=cls.keystone_session, client_version='2') - - def test_manila_share(self): - """Test that Manila + Ganesha shares can be accessed on two instances. - - 1. create a share - 2. Spawn two servers - 3. mount it on both - 4. write a file on one - 5. read it on the other - 6. profit - """ - # Create a share - share = self.manila_client.shares.create( - share_type='cephfsnfstype', name='cephnfsshare1', - share_proto="nfs", size=1) - - # Spawn Servers - instance_1, instance_2 = self.launch_guests( - userdata=self.INSTANCE_USERDATA) - - fip_1 = neutron_tests.floating_ips_from_instance(instance_1)[0] - fip_2 = neutron_tests.floating_ips_from_instance(instance_2)[0] - - # Wait for the created share to become available before it gets used. - openstack_utils.resource_reaches_status( - self.manila_client.shares, - share.id, - wait_iteration_max_time=120, - stop_after_attempt=2, - expected_status="available", - msg="Waiting for a share to become available") - - share.allow(access_type='ip', access=fip_1, access_level='rw') - share.allow(access_type='ip', access=fip_2, access_level='rw') - - # Mount Share - username = guest.boot_tests['bionic']['username'] - password = guest.boot_tests['bionic'].get('password') - privkey = openstack_utils.get_private_key(nova_utils.KEYPAIR_NAME) - - # Write a file on instance_1 - def verify_setup(stdin, stdout, stderr): - status = stdout.channel.recv_exit_status() - if status: - logging.info("{}".format(stderr.readlines()[0].strip())) - self.assertEqual(status, 0) - - mount_path = share.export_locations[0] - - for attempt in Retrying( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10)): - with attempt: - openstack_utils.ssh_command( - username, fip_1, 'instance-1', - 'sudo mkdir -p /mnt/ceph && ' - 'sudo mount -t nfs -o nfsvers=4.1,proto=tcp ' - '{} /mnt/ceph && ' - 'echo "test" | sudo tee /mnt/ceph/test'.format( - mount_path), - password=password, privkey=privkey, verify=verify_setup) - - for attempt in Retrying( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10)): - with attempt: - # Setup that file on instance_2 - openstack_utils.ssh_command( - username, fip_2, 'instance-2', - 'sudo mkdir -p /mnt/ceph && ' - 'sudo /bin/mount -t nfs -o nfsvers=4.1,proto=tcp ' - '{} /mnt/ceph' - .format(mount_path), - password=password, privkey=privkey, verify=verify_setup) - - def verify(stdin, stdout, stderr): - status = stdout.channel.recv_exit_status() - if status: - logging.info("{}".format(stderr.readlines()[0].strip())) - self.assertEqual(status, 0) - out = "" - for line in iter(stdout.readline, ""): - out += line - self.assertEqual(out, "test\n") - - openstack_utils.ssh_command( - username, fip_2, 'instance-2', - 'sudo cat /mnt/ceph/test', - password=password, privkey=privkey, verify=verify) + cls.share_name = 'cephnfsshare1' + cls.share_type_name = MANILA_GANESHA_TYPE_NAME + cls.share_protocol = 'nfs' From 42d955209a422f6cccb887c369e10163d6484110 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 4 Mar 2021 13:55:09 +0000 Subject: [PATCH 103/180] Add setup code for s3 back Trilio Add code to prepare trilio applications to use an s3 backend. --- zaza/openstack/charm_tests/trilio/setup.py | 81 +++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/setup.py b/zaza/openstack/charm_tests/trilio/setup.py index 44f1d9d..1c8e97c 100644 --- a/zaza/openstack/charm_tests/trilio/setup.py +++ b/zaza/openstack/charm_tests/trilio/setup.py @@ -19,12 +19,16 @@ import logging import os +import boto3 + +import zaza.charm_lifecycle.utils as lifecycle_utils import zaza.model as zaza_model import zaza.openstack.utilities.juju as juju_utils import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.openstack as openstack_utils -def basic_setup(): +def nfs_setup(): """Run setup for testing Trilio. Setup for testing Trilio is currently part of functional @@ -54,6 +58,9 @@ def basic_setup(): target_status="active", ) + +def trust_setup(): + """Run setup Trilio trust setup.""" logging.info("Executing create-cloud-admin-trust") password = juju_utils.leader_get("keystone", "admin_passwd") @@ -66,6 +73,9 @@ def basic_setup(): ) ) + +def license_setup(): + """Run setup Trilio license setup.""" logging.info("Executing create-license") test_license = os.environ.get("TEST_TRILIO_LICENSE") if test_license and os.path.exists(test_license): @@ -83,6 +93,75 @@ def basic_setup(): logging.error("Unable to find Trilio License file") +def s3_setup(): + """Run setup of s3 options for Trilio.""" + session = openstack_utils.get_overcloud_keystone_session() + ks_client = openstack_utils.get_keystone_session_client( + session) + + # Get token data so we can glean our user_id and project_id + token_data = ks_client.tokens.get_token_data(session.get_token()) + project_id = token_data['token']['project']['id'] + user_id = token_data['token']['user']['id'] + + # Store URL to service providing S3 compatible API + for entry in token_data['token']['catalog']: + if entry['type'] == 's3': + for endpoint in entry['endpoints']: + if endpoint['interface'] == 'public': + s3_region = endpoint['region'] + s3_endpoint = endpoint['url'] + + # Create AWS compatible application credentials in Keystone + ec2_creds = ks_client.ec2.create(user_id, project_id) + cacert = openstack_utils.get_cacert() + kwargs = { + 'region_name': s3_region, + 'aws_access_key_id': ec2_creds.access, + 'aws_secret_access_key': ec2_creds.secret, + 'endpoint_url': s3_endpoint, + 'verify': cacert, + } + s3 = boto3.resource('s3', **kwargs) + + # Create bucket + bucket_name = 'zaza-trilio' + logging.info("Creating bucket: {}".format(bucket_name)) + bucket = s3.Bucket(bucket_name) + bucket.create() + + s3_config = { + 'tv-s3-secret-key': ec2_creds.secret, + 'tv-s3-access-key': ec2_creds.access, + 'tv-s3-region-name': s3_region, + 'tv-s3-bucket': bucket_name, + 'tv-s3-endpoint-url': s3_endpoint} + for app in ['trilio-wlm', 'trilio-data-mover']: + logging.info("Setting s3 config for {}".format(app)) + zaza_model.set_application_config(app, s3_config) + test_config = lifecycle_utils.get_charm_config(fatal=False) + states = test_config.get('target_deploy_status', {}) + states['trilio-wlm'] = { + 'workload-status': 'blocked', + 'workload-status-message': 'application not trusted'} + zaza_model.wait_for_application_states( + states=test_config.get('target_deploy_status', {}), + timeout=7200) + zaza_model.block_until_all_units_idle() + + +def basic_setup(): + """Run basic setup for Trilio apps.""" + backup_target_type = zaza_model.get_application_config( + 'trilio-wlm')['backup-target-type']['value'] + if backup_target_type == "nfs": + nfs_setup() + if backup_target_type in ["s3", "experimental-s3"]: + s3_setup() + trust_setup() + license_setup() + + def python2_workaround(): """Workaround for Bug #1915914. From 93a48133b35e20d79251922c321d97b1ddf03e4e Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 5 Mar 2021 12:12:00 +0100 Subject: [PATCH 104/180] Add assertions to HaclusterScaleBackAndForthTest --- zaza/openstack/charm_tests/hacluster/tests.py | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 66d4f45..53c7392 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -189,6 +189,10 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): # aren't guaranteed to be blocked, so we don't validate that here. zaza.model.block_until_all_units_idle() + # At this point the corosync ring hasn't been updated yet, so it should + # still remember the deleted unit: + self.__assert_some_corosync_nodes_are_offline(surviving_hacluster_unit) + logging.info('Updating corosync ring') hacluster_app_name = zaza.model.get_unit_from_name( surviving_hacluster_unit).application @@ -198,6 +202,10 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): action_params={'i-really-mean-it': True}, raise_on_failure=True) + # At this point if the corosync ring has been properly updated, there + # shouldn't be any trace of the deleted unit anymore: + self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit) + logging.info('Re-adding an hacluster unit') zaza.model.add_unit(self._principle_app_name, wait_appear=True) @@ -212,15 +220,30 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): 'active') zaza.model.block_until_all_units_idle() - # At this point if the corosync ring has been properly updated, there - # shouldn't be any trace of the deleted unit anymore: + # At this point the corosync ring should still not contain any offline + # node: + self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit) + + def __assert_some_corosync_nodes_are_offline(self, hacluster_unit): + logging.info('Checking that corosync considers at least one node to ' + 'be offline') + output = self._get_crm_status(hacluster_unit) + self.assertIn('OFFLINE', output, + "corosync should list at least one offline node") + + def __assert_all_corosync_nodes_are_online(self, hacluster_unit): logging.info('Checking that corosync considers all nodes to be online') + output = self._get_crm_status(hacluster_unit) + self.assertNotIn('OFFLINE', output, + "corosync shouldn't list any offline node") + + @staticmethod + def _get_crm_status(hacluster_unit): cmd = 'sudo crm status' - result = zaza.model.run_on_unit(surviving_hacluster_unit, cmd) + result = zaza.model.run_on_unit(hacluster_unit, cmd) code = result.get('Code') if code != '0': raise zaza.model.CommandRunFailed(cmd, result) output = result.get('Stdout').strip() - logging.debug('Output received: {}'.format(output)) - self.assertNotIn('OFFLINE', output, - "corosync shouldn't list any offline node") + logging.debug('crm output received: {}'.format(output)) + return output From aafdc4070f0984d87dea05008de9a86e0beed920 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 5 Mar 2021 18:08:54 +0100 Subject: [PATCH 105/180] Workaround for lp:1874719 --- zaza/openstack/charm_tests/hacluster/tests.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 53c7392..5f46565 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -220,8 +220,17 @@ class HaclusterScaleBackAndForthTest(HaclusterBaseTest): 'active') zaza.model.block_until_all_units_idle() - # At this point the corosync ring should still not contain any offline - # node: + # Because of lp:1874719 the corosync ring may show a mysterious offline + # 'node1' node. We clean up the ring by re-running the 'update-ring' + # action: + logging.info('Updating corosync ring - workaround for lp:1874719') + zaza.model.run_action_on_leader( + hacluster_app_name, + 'update-ring', + action_params={'i-really-mean-it': True}, + raise_on_failure=True) + + # At this point the corosync ring should not contain any offline node: self.__assert_all_corosync_nodes_are_online(surviving_hacluster_unit) def __assert_some_corosync_nodes_are_offline(self, hacluster_unit): From 79a78c2ca04424f39aa14ca6aa45cb7c240f74f5 Mon Sep 17 00:00:00 2001 From: "Bartosz \"mastier\" Woronicz" Date: Mon, 8 Mar 2021 08:59:23 +0100 Subject: [PATCH 106/180] fix reload test for vault (#517) The test was failing for xenial-ha-mysql bundle This fixes the issue for cluster vault by picking the right vault cluster leader to run the commands against. Co-authored-by: Bartosz Woronicz --- zaza/openstack/charm_tests/vault/tests.py | 17 +++++++++++------ zaza/openstack/charm_tests/vault/utils.py | 18 +++++++++++++++++- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 219291e..8c3daca 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -31,6 +31,8 @@ import zaza.openstack.charm_tests.vault.utils as vault_utils import zaza.openstack.utilities.cert import zaza.openstack.utilities.openstack import zaza.model +import zaza.utilities.juju as juju_utils + class BaseVaultTest(test_utils.OpenStackBaseTest): @@ -268,7 +270,7 @@ class VaultTest(BaseVaultTest): raise unittest.SkipTest("The version of charm-vault tested does " "not have reload action") - lead_client = vault_utils.extract_lead_unit_client(self.clients) + lead_client = vault_utils.get_cluster_leader(self.clients) running_config = vault_utils.get_running_config(lead_client) value_to_set = not running_config['data']['disable_mlock'] @@ -277,15 +279,18 @@ class VaultTest(BaseVaultTest): {'disable-mlock': str(value_to_set)}) logging.info("Testing reload") - zaza.model.run_action_on_leader( - 'vault', + zaza.model.run_action( + juju_utils.get_unit_name_from_ip_address( + lead_client.addr, 'vault'), 'reload', - action_params={}) + model_name=self.model_name) + new_value = vault_utils.get_running_config(lead_client)[ + 'data']['disable_mlock'] + logging.info(new_value) self.assertEqual( value_to_set, - vault_utils.get_running_config(lead_client)[ - 'data']['disable_mlock']) + new_value) self.assertFalse(lead_client.hvac_client.seal_status['sealed']) def test_vault_restart(self): diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 9098b12..97fd060 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -137,10 +137,26 @@ def get_vip_client(cacert=None): return client +def get_cluster_leader(clients): + """Get Vault cluster leader + :param clients: Clients list to get leader + :type clients: List of CharmVaultClient + We have to make sure we run api calls + against the actual leader + """ + if len(clients) == 1: + return clients[0] + + for client in clients: + if client.hvac_client.ha_status['is_self']: + return client + return None + + def get_running_config(client): """Get Vault running config. - :param client: Client to use for initiliasation + :param client: Client used to get config :type client: CharmVaultClient The hvac library does not support getting info from endpoint /v1/sys/config/state/sanitized From 975f5058f44d3a2c1bb4b1e28e635f30482e02d1 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 8 Mar 2021 11:46:15 +0100 Subject: [PATCH 107/180] Add github actions support with linting and unit tests --- .github/workflows/tox.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/tox.yaml diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml new file mode 100644 index 0000000..880048a --- /dev/null +++ b/.github/workflows/tox.yaml @@ -0,0 +1,27 @@ +name: Python package + +on: + - push + - pull_request + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.4, 3.5, 3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + - name: Lint with tox + run: tox -e pep8 + - name: Test with tox + run: tox -e py${{ matrix.python-version }} \ No newline at end of file From 2d753a32bababc6b31aba116b098188816f4497a Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Mon, 8 Mar 2021 15:45:46 +0100 Subject: [PATCH 108/180] remove python 3.4 support --- .github/workflows/tox.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tox.yaml b/.github/workflows/tox.yaml index 880048a..754ce29 100644 --- a/.github/workflows/tox.yaml +++ b/.github/workflows/tox.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.4, 3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v1 From d7ec4fd0b8e70f6f96323c7dddcde4b30169a124 Mon Sep 17 00:00:00 2001 From: Bartosz Woronicz Date: Mon, 8 Mar 2021 16:18:52 +0100 Subject: [PATCH 109/180] fix formatting issues and docstring That fixes the issues introduced in PR #512 #517 Now flake8 should not complain --- zaza/openstack/charm_tests/vault/tests.py | 1 - zaza/openstack/charm_tests/vault/utils.py | 7 ++++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 8c3daca..c5211e0 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -34,7 +34,6 @@ import zaza.model import zaza.utilities.juju as juju_utils - class BaseVaultTest(test_utils.OpenStackBaseTest): """Base class for vault tests.""" diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 97fd060..60ac429 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -138,9 +138,12 @@ def get_vip_client(cacert=None): def get_cluster_leader(clients): - """Get Vault cluster leader + """Get Vault cluster leader. + :param clients: Clients list to get leader :type clients: List of CharmVaultClient + :returns: CharmVaultClient + :rtype: CharmVaultClient or None We have to make sure we run api calls against the actual leader """ @@ -158,6 +161,8 @@ def get_running_config(client): :param client: Client used to get config :type client: CharmVaultClient + :returns: dict from Vault api response + :rtype: dict The hvac library does not support getting info from endpoint /v1/sys/config/state/sanitized Therefore we implement it here From 4551f665059542bac1320f3dba067209d30dd71f Mon Sep 17 00:00:00 2001 From: Alex Kavanagh <567675+ajkavanagh@users.noreply.github.com> Date: Tue, 9 Mar 2021 17:59:33 +0000 Subject: [PATCH 110/180] Fix the kerberos set-up by loosening ubuntu check (#521) * Fix the kerberos set-up by loosening ubuntu check The cs:ubuntu charm's workload-status-message has changed recently (as the charm was re-written). This PR loosens the application states check to ignore the workload-status-message and just wait for the charm to be active workload-status. This is almost certainly enough, as it's a very simple charm. * Fix tox.ini to pass tests (maybe) --- tox.ini | 20 ++++++++++++++++++++ zaza/openstack/charm_tests/kerberos/setup.py | 10 +++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index dc09948..aea173f 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,26 @@ commands = nosetests --with-coverage --cover-package=zaza.openstack {posargs} {t basepython = python3 deps = -r{toxinidir}/requirements.txt +[testenv:py3.5] +basepython = python3.5 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.6] +basepython = python3.6 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.7] +basepython = python3.7 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.8] +basepython = python3.8 +deps = -r{toxinidir}/requirements.txt + +[testenv:py3.9] +basepython = python3.9 +deps = -r{toxinidir}/requirements.txt + [testenv:pep8] basepython = python3 deps = -r{toxinidir}/requirements.txt diff --git a/zaza/openstack/charm_tests/kerberos/setup.py b/zaza/openstack/charm_tests/kerberos/setup.py index d441f8c..11f4afa 100644 --- a/zaza/openstack/charm_tests/kerberos/setup.py +++ b/zaza/openstack/charm_tests/kerberos/setup.py @@ -113,7 +113,15 @@ def retrieve_and_attach_keytab(): 'keystone_keytab', tmp_file) - zaza.model.wait_for_application_states() + # cs:ubuntu charm has changed behaviour and we can't rely on the workload + # staus message. Thus, ignore it. + states = { + "ubuntu-test-host": { + "workload-status": "active", + "workload-status-message": "", + } + } + zaza.model.wait_for_application_states(states=states) zaza.model.block_until_all_units_idle() From c6e54dd50526efea014324173877d84ac396dbc9 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Tue, 9 Mar 2021 18:09:37 +0000 Subject: [PATCH 111/180] Clean-up/correct docstrings in vault/utils.py --- zaza/openstack/charm_tests/vault/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/utils.py b/zaza/openstack/charm_tests/vault/utils.py index 60ac429..40e33ac 100644 --- a/zaza/openstack/charm_tests/vault/utils.py +++ b/zaza/openstack/charm_tests/vault/utils.py @@ -140,12 +140,12 @@ def get_vip_client(cacert=None): def get_cluster_leader(clients): """Get Vault cluster leader. + We have to make sure we run api calls against the actual leader. + :param clients: Clients list to get leader :type clients: List of CharmVaultClient :returns: CharmVaultClient :rtype: CharmVaultClient or None - We have to make sure we run api calls - against the actual leader """ if len(clients) == 1: return clients[0] @@ -159,13 +159,13 @@ def get_cluster_leader(clients): def get_running_config(client): """Get Vault running config. + The hvac library does not support getting info from endpoint + /v1/sys/config/state/sanitized Therefore we implement it here + :param client: Client used to get config :type client: CharmVaultClient :returns: dict from Vault api response :rtype: dict - The hvac library does not support getting info - from endpoint /v1/sys/config/state/sanitized - Therefore we implement it here """ return requests.get( client.hvac_client.adapter.base_uri + '/v1/sys/config/state/sanitized', From 45146b6c45d306cb0312fc97509b5eca4c075c95 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 3 Mar 2021 14:54:47 +0000 Subject: [PATCH 112/180] Add ObjectRetrier to perform retries on openstack client calls This adds a wrapper class that detects if a callable object in any of the descendent objects raises an Exception. If so, then it retries that exception. This is to attempt to make the zaza tests a little more robust in the face of small network failures or strange restarts. This is a test, and robust logging a reporting should be used to determine whether it is covering up actual bugs rather than CI system issues. Related Bug: (zot repo)#348 --- unit_tests/utilities/test_utilities.py | 166 +++++++++++++++++++++++++ zaza/openstack/utilities/__init__.py | 128 +++++++++++++++++++ 2 files changed, 294 insertions(+) create mode 100644 unit_tests/utilities/test_utilities.py diff --git a/unit_tests/utilities/test_utilities.py b/unit_tests/utilities/test_utilities.py new file mode 100644 index 0000000..e961de1 --- /dev/null +++ b/unit_tests/utilities/test_utilities.py @@ -0,0 +1,166 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import unit_tests.utils as ut_utils + +import zaza.openstack.utilities as utilities + + +class SomeException(Exception): + pass + + +class SomeException2(Exception): + pass + + +class SomeException3(Exception): + pass + + +class TestObjectRetrier(ut_utils.BaseTestCase): + + def test_object_wrap(self): + + class A: + + def func(self, a, b=1): + return a + b + + a = A() + wrapped_a = utilities.ObjectRetrier(a) + self.assertEqual(wrapped_a.func(3), 4) + + def test_object_multilevel_wrap(self): + + class A: + + def f1(self, a, b): + return a * b + + class B: + + @property + def f2(self): + + return A() + + b = B() + wrapped_b = utilities.ObjectRetrier(b) + self.assertEqual(wrapped_b.f2.f1(5, 6), 30) + + def test_object_wrap_number(self): + + class A: + + class_a = 5 + + def __init__(self): + self.instance_a = 10 + + def f1(self, a, b): + return a * b + + a = A() + wrapped_a = utilities.ObjectRetrier(a) + self.assertEqual(wrapped_a.class_a, 5) + self.assertEqual(wrapped_a.instance_a, 10) + + @mock.patch("time.sleep") + def test_object_wrap_exception(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + # retry on a specific exception + wrapped_a = utilities.ObjectRetrier(a, num_retries=1, + retry_exceptions=[SomeException]) + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_called_once_with(5) + + # also retry on any exception if none specified + wrapped_a = utilities.ObjectRetrier(a, num_retries=1) + mock_sleep.reset_mock() + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_called_once_with(5) + + # no retry if exception isn't listed. + wrapped_a = utilities.ObjectRetrier(a, num_retries=1, + retry_exceptions=[SomeException2]) + mock_sleep.reset_mock() + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_not_called() + + @mock.patch("time.sleep") + def test_log_called(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + mock_log = mock.Mock() + wrapped_a = utilities.ObjectRetrier(a, num_retries=1, log=mock_log) + with self.assertRaises(SomeException): + wrapped_a.func() + + # there should be two calls; one for the single retry and one for the + # failure. + self.assertEqual(mock_log.call_count, 2) + + @mock.patch("time.sleep") + def test_back_off_maximum(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + wrapped_a = utilities.ObjectRetrier(a, num_retries=3, backoff=2) + with self.assertRaises(SomeException): + wrapped_a.func() + # Note third call hits maximum wait time of 15. + mock_sleep.assert_has_calls([mock.call(5), + mock.call(10), + mock.call(15)]) + + @mock.patch("time.sleep") + def test_total_wait(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + wrapped_a = utilities.ObjectRetrier(a, num_retries=3, total_wait=9) + with self.assertRaises(SomeException): + wrapped_a.func() + # Note only two calls, as total wait is 9, so a 3rd retry would exceed + # that. + mock_sleep.assert_has_calls([mock.call(5), + mock.call(5)]) diff --git a/zaza/openstack/utilities/__init__.py b/zaza/openstack/utilities/__init__.py index 35b5a14..5011689 100644 --- a/zaza/openstack/utilities/__init__.py +++ b/zaza/openstack/utilities/__init__.py @@ -13,3 +13,131 @@ # limitations under the License. """Collection of utilities to support zaza tests etc.""" + + +import time + + +class ObjectRetrier(object): + """An automatic retrier for an object. + + This is designed to be used with an instance of an object. Basically, it + wraps the object and any attributes that are fetched. Essentially, it is + used to provide retries on method calls on openstack client objects in + tests to increase robustness of tests. + + Although, technically this is bad, retries can be logged with the optional + log method. + + Usage: + + # get a client that does 3 retries, waits 5 seconds between retries and + # retries on any error. + some_client = ObjectRetrier(get_some_client) + # this gets retried up to 3 times. + things = some_client.list_things() + + Note, it is quite simple. It wraps the object and on a getattr(obj, name) + it finds the name and then returns a wrapped version of that name. On a + call, it returns the value of that call. It only wraps objects in the + chain that are either callable or have a __getattr__() method. i.e. one + that can then be retried or further fetched. This means that if a.b.c() is + a chain of objects, and we just wrap 'a', then 'b' and 'c' will both be + wrapped that the 'c' object __call__() method will be the one that is + actually retried. + + Note: this means that properties that do method calls won't be retried. + This is a limitation that may be addressed in the future, if it is needed. + """ + + def __init__(self, obj, num_retries=3, initial_interval=5.0, backoff=1.0, + max_interval=15.0, total_wait=30.0, retry_exceptions=None, + log=None): + """Initialise the retrier object. + + :param obj: The object to wrap. Ought to be an instance of something + that you want to get methods on to call or be called itself. + :type obj: Any + :param num_retries: The (maximum) number of retries. May not be hit if + the total_wait time is exceeded. + :type num_retries: int + :param initial_interval: The initial or starting interval between + retries. + :type initial_interval: float + :param backoff: The exponential backoff multiple. 1 is linear. + :type backoff: float + :param max_interval: The maximum interval between retries. + If backoff is >1 then the initial_interval will never grow larger + than max_interval. + :type max_interval: float + :param retry_exceptions: The list of exceptions to retry on, or None. + If a list, then it will only retry if the exception is one of the + ones in the list. + :type retry_exceptions: List[Exception] + """ + # Note we use semi-private variable names that shouldn't clash with any + # on the actual object. + self.__obj = obj + self.__kwargs = { + 'num_retries': num_retries, + 'initial_interval': initial_interval, + 'backoff': backoff, + 'max_interval': max_interval, + 'total_wait': total_wait, + 'retry_exceptions': retry_exceptions, + 'log': log or (lambda x: None), + } + + def __getattr__(self, name): + """Get attribute; delegates to wrapped object.""" + # Note the above may generate an attribute error; we expect this and + # will fail with an attribute error. + attr = getattr(self.__obj, name) + if callable(attr) or hasattr(attr, "__getattr__"): + return ObjectRetrier(attr, **self.__kwargs) + else: + return attr + # TODO(ajkavanagh): Note detecting a property is a bit trickier. we + # can do isinstance(attr, property), but then the act of accessing it + # is what calls it. i.e. it would fail at the getattr(self.__obj, + # name) stage. The solution is to check first, and if it's a property, + # then treat it like the retrier. However, I think this is too + # complex for the first go, and to use manual retries in that instance. + + def __call__(self, *args, **kwargs): + """Call the object; delegates to the wrapped object.""" + obj = self.__obj + retry = 0 + wait = self.__kwargs['initial_interval'] + max_interval = self.__kwargs['max_interval'] + log = self.__kwargs['log'] + backoff = self.__kwargs['backoff'] + total_wait = self.__kwargs['total_wait'] + num_retries = self.__kwargs['num_retries'] + retry_exceptions = self.__kwargs['retry_exceptions'] + wait_so_far = 0 + while True: + try: + return obj(*args, **kwargs) + except Exception as e: + # if retry_exceptions is None, or the type of the exception is + # not in the list of retries, then raise an exception + # immediately. This means that if retry_exceptions is None, + # then the method is always retried. + if (retry_exceptions is not None and + type(e) not in retry_exceptions): + raise + retry += 1 + if retry > num_retries: + log("{}: exceeded number of retries, so erroring out" + .format(str(obj))) + raise e + log("{}: call failed: retrying in {} seconds" + .format(str(obj), wait)) + time.sleep(wait) + wait_so_far += wait + if wait_so_far >= total_wait: + raise e + wait = wait * backoff + if wait > max_interval: + wait = max_interval From ed8528b76a842d5729ed6f5700d4cd4270de1c8a Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 3 Mar 2021 14:58:27 +0000 Subject: [PATCH 113/180] Add object retrier to the LB tests for octavia --- zaza/openstack/charm_tests/octavia/tests.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 26941a2..d6bae64 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -25,6 +25,8 @@ import osc_lib.exceptions import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils +from zaza.openstack import ObjectRetrier + class CharmOperationTest(test_utils.OpenStackBaseTest): """Charm operation tests.""" @@ -63,12 +65,12 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run class setup for running LBaaSv2 service tests.""" super(LBAASv2Test, cls).setUpClass() - cls.keystone_client = openstack_utils.get_keystone_session_client( - cls.keystone_session) - cls.neutron_client = openstack_utils.get_neutron_session_client( - cls.keystone_session) - cls.octavia_client = openstack_utils.get_octavia_session_client( - cls.keystone_session) + cls.keystone_client = ObjectRetrier( + openstack_utils.get_keystone_session_client(cls.keystone_session)) + cls.neutron_client = ObjectRetrier( + openstack_utils.get_neutron_session_client(cls.keystone_session)) + cls.octavia_client = ObjectRetrier( + openstack_utils.get_octavia_session_client(cls.keystone_session)) cls.RESOURCE_PREFIX = 'zaza-octavia' # NOTE(fnordahl): in the event of a test failure we do not want to run From c041042fe220fa5de33f8006734c66acb8255f11 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Mon, 8 Mar 2021 10:50:41 +0000 Subject: [PATCH 114/180] Add function to just retry on keystone ConnectFailure The main failure that seems to occur with clients is the ConnectFailure, which according to the docs, is retry-able. Thus provide a function that adds that exception condition automatically. Also fix the import problem in octavia test for the object retrier that is being used to validate the ObjectRetrier feature. --- unit_tests/utilities/test_utilities.py | 20 ++++++++++++++++++ zaza/openstack/charm_tests/octavia/tests.py | 2 +- zaza/openstack/utilities/__init__.py | 23 +++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/unit_tests/utilities/test_utilities.py b/unit_tests/utilities/test_utilities.py index e961de1..a78600c 100644 --- a/unit_tests/utilities/test_utilities.py +++ b/unit_tests/utilities/test_utilities.py @@ -164,3 +164,23 @@ class TestObjectRetrier(ut_utils.BaseTestCase): # that. mock_sleep.assert_has_calls([mock.call(5), mock.call(5)]) + + @mock.patch("time.sleep") + def test_retry_on_connect_failure(self, mock_sleep): + + class A: + + def func1(self): + raise SomeException() + + def func2(self): + raise utilities.ConnectFailure() + + a = A() + wrapped_a = utilities.retry_on_connect_failure(a, num_retries=2) + with self.assertRaises(SomeException): + wrapped_a.func1() + mock_sleep.assert_not_called() + with self.assertRaises(utilities.ConnectFailure): + wrapped_a.func2() + mock_sleep.assert_has_calls([mock.call(5)]) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index d6bae64..9a71b49 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -25,7 +25,7 @@ import osc_lib.exceptions import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils -from zaza.openstack import ObjectRetrier +from zaza.openstack.utilities import ObjectRetrier class CharmOperationTest(test_utils.OpenStackBaseTest): diff --git a/zaza/openstack/utilities/__init__.py b/zaza/openstack/utilities/__init__.py index 5011689..6db3ce3 100644 --- a/zaza/openstack/utilities/__init__.py +++ b/zaza/openstack/utilities/__init__.py @@ -17,6 +17,8 @@ import time +from keystoneauth1.exceptions.connection import ConnectFailure + class ObjectRetrier(object): """An automatic retrier for an object. @@ -141,3 +143,24 @@ class ObjectRetrier(object): wait = wait * backoff if wait > max_interval: wait = max_interval + + +def retry_on_connect_failure(client, **kwargs): + """Retry an object that eventually gets resolved to a call. + + Specifically, this uses ObjectRetrier but only against the + keystoneauth1.exceptions.connection.ConnectFailure exeception. + + :params client: the object that may throw and exception when called. + :type client: Any + :params **kwargs: the arguments supplied to the ObjectRetrier init method + :type **kwargs: Dict[Any] + :returns: client wrapped in an ObjectRetrier instance + :rtype: ObjectRetrier[client] + """ + kwcopy = kwargs.copy() + if 'retry_exceptions' not in kwcopy: + kwcopy['retry_exceptions'] = [] + if ConnectFailure not in kwcopy['retry_exceptions']: + kwcopy['retry_exceptions'].append(ConnectFailure) + return ObjectRetrier(client, **kwcopy) From 74c5c9d69c979a51b48bf8bd0eedd34dd0eb8883 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Fri, 12 Mar 2021 14:53:21 +0200 Subject: [PATCH 115/180] Fix _write_testing_file_on_instance docstring --- zaza/openstack/charm_tests/manila/tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/charm_tests/manila/tests.py b/zaza/openstack/charm_tests/manila/tests.py index e7cba8f..1b58f65 100644 --- a/zaza/openstack/charm_tests/manila/tests.py +++ b/zaza/openstack/charm_tests/manila/tests.py @@ -208,9 +208,9 @@ packages: ssh_private_key): """Write a file on a Manila share mounted into a Nova instance. - Mount the Manila share in the given Nova instance, and write a testing - file on it (which is meant to be validated from another instance). - These commands are executed via SSH. + Write a testing file into the already mounted Manila share from the + given Nova instance (which is meant to be validated from another + instance). These commands are executed via SSH. :param instance_ip: IP of the Nova instance. :type instance_ip: string From 7e568138857307ee445d9552a2717088b550f6f0 Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Sun, 28 Feb 2021 22:39:29 +0200 Subject: [PATCH 116/180] Add Manila NetApp tests These Zaza tests are meant to be used with the new `manila-netapp` charm. --- .../charm_tests/manila_netapp/__init__.py | 17 ++++ .../charm_tests/manila_netapp/setup.py | 96 +++++++++++++++++++ .../charm_tests/manila_netapp/tests.py | 51 ++++++++++ zaza/openstack/utilities/openstack.py | 14 +++ 4 files changed, 178 insertions(+) create mode 100644 zaza/openstack/charm_tests/manila_netapp/__init__.py create mode 100644 zaza/openstack/charm_tests/manila_netapp/setup.py create mode 100644 zaza/openstack/charm_tests/manila_netapp/tests.py diff --git a/zaza/openstack/charm_tests/manila_netapp/__init__.py b/zaza/openstack/charm_tests/manila_netapp/__init__.py new file mode 100644 index 0000000..f6e2865 --- /dev/null +++ b/zaza/openstack/charm_tests/manila_netapp/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila NetApp setup and testing.""" diff --git a/zaza/openstack/charm_tests/manila_netapp/setup.py b/zaza/openstack/charm_tests/manila_netapp/setup.py new file mode 100644 index 0000000..fa1a671 --- /dev/null +++ b/zaza/openstack/charm_tests/manila_netapp/setup.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila NetApp setup.""" + +import zaza.openstack.utilities.openstack as openstack_utils +import zaza.openstack.charm_tests.neutron.setup as neutron_setup + + +MANILA_NETAPP_TYPE_NAME = "netapp-ontap" +MANILA_NETAPP_BACKEND_NAME = "netapp-ontap" + +MANILA_NETAPP_DHSS_TYPE_NAME = "netapp-ontap-dhss" +MANILA_NETAPP_DHSS_BACKEND_NAME = "netapp-ontap-dhss" + +MANILA_NETAPP_SHARE_NET_NAME = "netapp-ontap-share-network" + + +def create_netapp_share_type(manila_client=None): + """Create a share type for Manila with NetApp Data ONTAP driver. + + :param manila_client: Authenticated manilaclient + :type manila_client: manilaclient.Client + """ + if manila_client is None: + manila_client = openstack_utils.get_manila_session_client( + openstack_utils.get_overcloud_keystone_session()) + + manila_client.share_types.create( + name=MANILA_NETAPP_TYPE_NAME, + spec_driver_handles_share_servers=False, + extra_specs={ + 'vendor_name': 'NetApp', + 'share_backend_name': MANILA_NETAPP_BACKEND_NAME, + 'storage_protocol': 'NFS_CIFS', + }) + + +def create_netapp_dhss_share_type(manila_client=None): + """Create a DHSS share type for Manila with NetApp Data ONTAP driver. + + :param manila_client: Authenticated manilaclient + :type manila_client: manilaclient.Client + """ + if manila_client is None: + manila_client = openstack_utils.get_manila_session_client( + openstack_utils.get_overcloud_keystone_session()) + + manila_client.share_types.create( + name=MANILA_NETAPP_DHSS_TYPE_NAME, + spec_driver_handles_share_servers=True, + extra_specs={ + 'vendor_name': 'NetApp', + 'share_backend_name': MANILA_NETAPP_DHSS_BACKEND_NAME, + 'storage_protocol': 'NFS_CIFS', + }) + + +def create_netapp_share_network(manila_client=None): + """Create a Manila share network from the existing provider network. + + This setup function assumes that 'neutron.setup.basic_overcloud_network' + is called to have the proper tenant networks setup. + + The share network will be bound to the provider network configured by + 'neutron.setup.basic_overcloud_network'. + """ + session = openstack_utils.get_overcloud_keystone_session() + if manila_client is None: + manila_client = openstack_utils.get_manila_session_client(session) + + neutron = openstack_utils.get_neutron_session_client(session) + external_net = neutron.find_resource( + 'network', + neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_net_name']) + external_subnet = neutron.find_resource( + 'subnet', + neutron_setup.OVERCLOUD_NETWORK_CONFIG['external_subnet_name']) + + manila_client.share_networks.create( + name=MANILA_NETAPP_SHARE_NET_NAME, + neutron_net_id=external_net['id'], + neutron_subnet_id=external_subnet['id']) diff --git a/zaza/openstack/charm_tests/manila_netapp/tests.py b/zaza/openstack/charm_tests/manila_netapp/tests.py new file mode 100644 index 0000000..f9178ba --- /dev/null +++ b/zaza/openstack/charm_tests/manila_netapp/tests.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulate Manila NetApp testing.""" + +from zaza.openstack.charm_tests.manila_netapp.setup import ( + MANILA_NETAPP_TYPE_NAME, + MANILA_NETAPP_DHSS_TYPE_NAME, + MANILA_NETAPP_SHARE_NET_NAME, +) + +import zaza.openstack.charm_tests.manila.tests as manila_tests + + +class ManilaNetAppNFSTest(manila_tests.ManilaBaseTest): + """Encapsulate Manila NetApp NFS test.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaNetAppNFSTest, cls).setUpClass() + cls.share_name = 'netapp-ontap-share' + cls.share_type_name = MANILA_NETAPP_TYPE_NAME + cls.share_protocol = 'nfs' + + +class ManilaNetAppDHSSNFSTest(manila_tests.ManilaBaseTest): + """Encapsulate Manila NetApp NFS test.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running tests.""" + super(ManilaNetAppDHSSNFSTest, cls).setUpClass() + cls.share_name = 'netapp-ontap-dhss-share' + cls.share_type_name = MANILA_NETAPP_DHSS_TYPE_NAME + cls.share_protocol = 'nfs' + cls.share_network = cls.manila_client.share_networks.find( + name=MANILA_NETAPP_SHARE_NET_NAME) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 4ed75b9..230a3f6 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -69,6 +69,7 @@ from neutronclient.v2_0 import client as neutronclient from neutronclient.common import exceptions as neutronexceptions from octaviaclient.api.v2 import octavia as octaviaclient from swiftclient import client as swiftclient +from manilaclient import client as manilaclient from juju.errors import JujuError @@ -449,6 +450,19 @@ def get_aodh_session_client(session): return aodh_client.Client(session=session) +def get_manila_session_client(session, version='2'): + """Return Manila client authenticated by keystone session. + + :param session: Keystone session object + :type session: keystoneauth1.session.Session object + :param version: Manila API version + :type version: str + :returns: Authenticated manilaclient + :rtype: manilaclient.Client + """ + return manilaclient.Client(session=session, client_version=version) + + def get_keystone_scope(model_name=None): """Return Keystone scope based on OpenStack release of the overcloud. From fae2f6319e730285866c77c98b86282511b8f432 Mon Sep 17 00:00:00 2001 From: Peter Matulis Date: Mon, 15 Mar 2021 13:17:22 -0400 Subject: [PATCH 117/180] Add upgrades doc note (#526) * Add upgrades doc note Add a note so that upgrade testing (charms and payload) remains in sync with the end-user documentation (CDG) --- zaza/openstack/utilities/upgrade_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/upgrade_utils.py b/zaza/openstack/utilities/upgrade_utils.py index 8e3a063..4f00592 100644 --- a/zaza/openstack/utilities/upgrade_utils.py +++ b/zaza/openstack/utilities/upgrade_utils.py @@ -26,7 +26,13 @@ from zaza.openstack.utilities.os_versions import ( OPENSTACK_RELEASES_PAIRS, ) - +""" +The below upgrade order is surfaced in end-user documentation. Any change to +it should be accompanied by an update to the OpenStack Charms Deployment Guide +for both charm upgrades and payload upgrades: +- source/upgrade-charms.rst#upgrade-order +- source/upgrade-openstack.rst#openstack_upgrade_order +""" SERVICE_GROUPS = ( ('Database Services', ['percona-cluster', 'mysql-innodb-cluster']), ('Stateful Services', ['rabbitmq-server', 'ceph-mon']), From e3fab555ccc84f886cd8c5078fbf58bf6b47bbb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Lal?= Date: Wed, 17 Mar 2021 18:52:10 +0100 Subject: [PATCH 118/180] Add tests for neutron-openvswitch and neutron-gateway charms (#523) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add support for running NeutronOvsVsctlTest against any charm and not only charm-neutron-gateway. * Add new test case for charm-neutron-gateway and charm-neutron-openvswitch to verify correct handling of conflicting ext-port and data-port configurations. Signed-off-by: Przemysław Lal --- zaza/openstack/charm_tests/neutron/tests.py | 55 ++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index bebe3b6..70def89 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -592,6 +592,58 @@ class NeutronOpenvSwitchTest(NeutronPluginApiSharedTests): logging.info('Testing pause resume') +class NeutronBridgePortMappingTest(NeutronPluginApiSharedTests): + """Test correct handling of network-bridge-port mapping functionality.""" + + def test_600_conflict_data_ext_ports(self): + """Verify proper handling of conflict between data-port and ext-port. + + Configuring ext-port and data-port at the same time should make the + charm to enter "blocked" state. After unsetting ext-port charm should + be active again. + """ + if self.application_name not in ["neutron-gateway", + "neutron-openvswitch"]: + logging.debug("Skipping test, charm under test is not " + "neutron-gateway or neutron-openvswitch") + return + + current_data_port = zaza.model.get_application_config( + self.application_name).get("data-port").get("value", "") + current_ext_port = zaza.model.get_application_config( + self.application_name).get("ext-port").get("value", "") + logging.debug("Current data-port: '{}'".format(current_data_port)) + logging.debug("Current data-port: '{}'".format(current_ext_port)) + + test_config = zaza.charm_lifecycle.utils.get_charm_config( + fatal=False) + current_state = test_config.get("target_deploy_status", {}) + blocked_state = copy.deepcopy(current_state) + blocked_state[self.application_name] = { + "workload-status": "blocked", + "workload-status-message": + "ext-port set when data-port set: see config.yaml"} + + logging.info("Setting conflicting ext-port and data-port options") + zaza.model.set_application_config( + self.application_name, {"data-port": "br-phynet43:eth43", + "ext-port": "br-phynet43:eth43"}) + zaza.model.wait_for_application_states(states=blocked_state) + + # unset ext-port and wait for app state to return to active + logging.info("Unsetting conflicting ext-port option") + zaza.model.set_application_config( + self.application_name, {"ext-port": ""}) + zaza.model.wait_for_application_states(states=current_state) + + # restore original config + zaza.model.set_application_config( + self.application_name, {'data-port': current_data_port, + 'ext-port': current_ext_port}) + zaza.model.wait_for_application_states(states=current_state) + logging.info('OK') + + class NeutronOvsVsctlTest(NeutronPluginApiSharedTests): """Test 'ovs-vsctl'-related functionality on Neutron charms.""" @@ -610,7 +662,8 @@ class NeutronOvsVsctlTest(NeutronPluginApiSharedTests): unit.name, bridge_name ) + ' is marked as managed by us' ) - expected_external_id = 'charm-neutron-gateway=managed' + expected_external_id = 'charm-{}=managed'.format( + self.application_name) actual_external_id = zaza.model.run_on_unit( unit.entity_id, 'ovs-vsctl br-get-external-id {}'.format(bridge_name), From b428b6cf155184541ac7cf0ddd0a05d8418e34fc Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 19 Mar 2021 15:37:03 +0100 Subject: [PATCH 119/180] Remove obsolete HaclusterScalebackTest It has been superseded by HaclusterScaleBackAndForthTest and was used in the following charm gates: hacluster and mysql-router. --- zaza/openstack/charm_tests/hacluster/tests.py | 64 +------------------ 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/zaza/openstack/charm_tests/hacluster/tests.py b/zaza/openstack/charm_tests/hacluster/tests.py index 5f46565..9414145 100644 --- a/zaza/openstack/charm_tests/hacluster/tests.py +++ b/zaza/openstack/charm_tests/hacluster/tests.py @@ -76,70 +76,8 @@ class HaclusterTest(HaclusterBaseTest): self._toggle_maintenance_and_wait('false') -class HaclusterScalebackTest(HaclusterBaseTest): - """hacluster scaleback tests. - - Use for testing older releases where lp:1400481 wasn't fixed yet. - Superseded by HaclusterScaleBackAndForthTest. - """ - - @classmethod - def setUpClass(cls): - """Run class setup for running hacluster scaleback tests.""" - super(HaclusterScalebackTest, cls).setUpClass() - test_config = cls.test_config['tests_options']['hacluster'] - cls._principle_app_name = test_config['principle-app-name'] - cls._hacluster_charm_name = test_config['hacluster-charm-name'] - - def test_930_scaleback(self): - """Remove a unit and add a new one.""" - principle_units = sorted(zaza.model.get_status().applications[ - self._principle_app_name]['units'].keys()) - self.assertEqual(len(principle_units), 3) - doomed_principle_unit = principle_units[0] - other_principle_unit = principle_units[1] - doomed_hacluster_unit = juju_utils.get_subordinate_units( - [doomed_principle_unit], charm_name=self._hacluster_charm_name)[0] - other_hacluster_unit = juju_utils.get_subordinate_units( - [other_principle_unit], charm_name=self._hacluster_charm_name)[0] - - logging.info('Pausing unit {}'.format(doomed_hacluster_unit)) - zaza.model.run_action( - doomed_hacluster_unit, - 'pause', - raise_on_failure=True) - logging.info('OK') - - logging.info('Removing {}'.format(doomed_principle_unit)) - zaza.model.destroy_unit( - self._principle_app_name, - doomed_principle_unit, - wait_disappear=True) - logging.info('OK') - - logging.info('Waiting for model to settle') - zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'blocked') - zaza.model.block_until_unit_wl_status(other_principle_unit, 'blocked') - zaza.model.block_until_all_units_idle() - logging.info('OK') - - logging.info('Adding an hacluster unit') - zaza.model.add_unit(self._principle_app_name, wait_appear=True) - logging.info('OK') - - logging.info('Waiting for model to settle') - zaza.model.block_until_unit_wl_status(other_hacluster_unit, 'active') - # NOTE(lourot): the principle application sometimes remain blocked - # after scaling back up. - zaza.model.block_until_all_units_idle() - logging.debug('OK') - - class HaclusterScaleBackAndForthTest(HaclusterBaseTest): - """hacluster tests scaling back and forth. - - Supersedes HaclusterScalebackTest. - """ + """hacluster tests scaling back and forth.""" @classmethod def setUpClass(cls): From 1f47ef0bd769e3e7976130d9c81071d84bacd60f Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 22 Mar 2021 18:43:13 +0000 Subject: [PATCH 120/180] Update package restart message afer ch change --- zaza/openstack/charm_tests/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 3aff816..ea134d3 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -893,7 +893,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): self.check_show_deferred_restarts_wlm(self.restart_package_service) self.check_show_deferred_restarts_action( self.restart_package_service, - 'Pkg Update') + 'Package update') logging.info("Running restart action to clear deferred restarts") self.check_clear_restarts() From 2a6517f6fd1fca128c414d29604706e54b3cfe29 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 23 Mar 2021 08:32:01 -0700 Subject: [PATCH 121/180] Add restart on change test (#527) Add the restart on change test for mysql-router. --- zaza/openstack/charm_tests/mysql/tests.py | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 15256e7..45dbf71 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -861,3 +861,38 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): assert action.data.get("results") is not None, ( "Remove instance action failed: No results: {}" .format(action.data)) + + +class MySQLRouterTests(test_utils.OpenStackBaseTest): + """MySQL Router Tests.""" + + @classmethod + def setUpClass(cls, application_name="keystone-mysql-router"): + """Run class setup for running mysql-router tests.""" + super().setUpClass(application_name=application_name) + cls.application = application_name + cls.services = ["mysqlrouter"] + # Config file affected by juju set config change + cls.conf_file = ( + "/var/lib/mysql/{}-mysql-router/mysqlrouter.conf" + .format(application_name)) + + def test_910_restart_on_config_change(self): + """Checking restart happens on config change. + + Change max connections and assert that change propagates to the correct + file and that services are restarted as a result + """ + # Expected default and alternate values + set_default = {"ttl": ".5"} + set_alternate = {"ttl": "7"} + + # Make config change, check for service restarts + logging.info("Setting TTL ...") + self.restart_on_changed( + self.conf_file, + set_default, + set_alternate, + {}, {}, + self.services) + logging.info("Passed restart on changed test.") From 085d5a1513afbf17828ae2bbf3e2c60bd8f0eb02 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 23 Mar 2021 15:57:21 +0000 Subject: [PATCH 122/180] Add RabbitMQDeferredRestartTest + fixes --- .../charm_tests/rabbitmq_server/tests.py | 42 +++++++++++++++++++ zaza/openstack/charm_tests/test_utils.py | 3 +- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index d4a3111..40e227c 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -428,3 +428,45 @@ class RmqTests(test_utils.OpenStackBaseTest): check_units(all_units) logging.info('OK') + + +class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass( + restart_config_file='/etc/rabbitmq/rabbitmq.config', + test_service='rabbitmq-server', + restart_package='rabbitmq-server', + restart_package_service='rabbitmq-server', + application_name='rabbitmq-server') + + def trigger_deferred_restart_via_charm(self): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be renderred in + applications config file. + + NOTE: The implementation assumes the charm has a `debug` option and + self.restart_config_file in an oslo config file where that + debug option is renderred. If that is not true the specaliasation + class should override this method. + """ + app_config = zaza.model.get_application_config(self.application_name) + logging.info("Triggering deferred restart via config change") + new_debug_value = str(int(app_config['connection-backlog'].get('value', 100) + 1)) + logging.info("Setting connection-backlog: {}".format(new_debug_value)) + zaza.model.set_application_config( + self.application_name, + {'connection-backlog': new_debug_value}) + logging.info("Waiting for connection-backlog to be {} in {}".format( + new_debug_value, + self.restart_config_file)) + zaza.model.block_until_file_matches_re( + self.application_name, + self.restart_config_file, + '{{backlog, {}}}'.format(new_debug_value)) + logging.info("Waiting for units to be idle") + zaza.model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index ea134d3..26a9917 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -770,7 +770,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): # Check workload status no longer shows deferred restarts. for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + assert 'Services queued' not in unit.workload_status_message def check_show_deferred_restarts_action(self, test_service, restart_reason): @@ -796,6 +796,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): "show-deferred-restarts action on {}").format( test_service, unit.entity_id)) + assert action.data['status'] == 'completed' for event in yaml.safe_load(action.data['results']['output']): if test_service in event and restart_reason in event: break From 0315ca680e8f84304f9f2aa0ddc0c25b65eca25a Mon Sep 17 00:00:00 2001 From: Ionut Balutoiu Date: Thu, 11 Feb 2021 17:27:42 +0200 Subject: [PATCH 123/180] Add test for Glance with Ceph RGW backend This test shall be used for Glance testing bundles with Ceph deployed, and multi-backend enabled. It validates that images can be stored using the Ceph RGW (via the OpenStack Swift API). --- zaza/openstack/charm_tests/glance/tests.py | 50 ++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index 363032d..5d45338 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -92,3 +92,53 @@ class GlanceTest(test_utils.OpenStackBaseTest): they are started """ self.pause_resume(['glance-api']) + + +class GlanceCephRGWBackendTest(test_utils.OpenStackBaseTest): + """Encapsulate glance tests using the Ceph RGW backend. + + It validates the Ceph RGW backend in glance, which uses the Swift API. + """ + + @classmethod + def setUpClass(cls): + """Run class setup for running glance tests.""" + super(GlanceCephRGWBackendTest, cls).setUpClass() + + swift_session = openstack_utils.get_keystone_session_from_relation( + 'ceph-radosgw') + cls.swift = openstack_utils.get_swift_session_client( + swift_session) + cls.glance_client = openstack_utils.get_glance_session_client( + cls.keystone_session) + + def test_100_create_image(self): + """Create an image and do a simple validation of it. + + The OpenStack Swift API is used to do the validation, since the Ceph + Rados Gateway serves an API which is compatible with that. + """ + image_name = 'zaza-ceph-rgw-image' + openstack_utils.create_image( + glance=self.glance_client, + image_url=openstack_utils.find_cirros_image(arch='x86_64'), + image_name=image_name, + backend='swift') + headers, containers = self.swift.get_account() + self.assertEqual(len(containers), 1) + container_name = containers[0].get('name') + headers, objects = self.swift.get_container(container_name) + images = openstack_utils.get_images_by_name( + self.glance_client, + image_name) + self.assertEqual(len(images), 1) + image = images[0] + total_bytes = 0 + for ob in objects: + if '{}-'.format(image['id']) in ob['name']: + total_bytes = total_bytes + int(ob['bytes']) + logging.info( + 'Checking glance image size {} matches swift ' + 'image size {}'.format(image['size'], total_bytes)) + self.assertEqual(image['size'], total_bytes) + openstack_utils.delete_image(self.glance_client, image['id']) From 77cb2d0bd84f1a095ac034be6fa428a8242fe0d8 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 24 Mar 2021 13:52:34 +0000 Subject: [PATCH 124/180] Fix doc string and lint --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 40e227c..14fadc4 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -449,14 +449,13 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): Set the charm debug option and wait for that change to be renderred in applications config file. - NOTE: The implementation assumes the charm has a `debug` option and - self.restart_config_file in an oslo config file where that - debug option is renderred. If that is not true the specaliasation - class should override this method. + This overrides the base class version as the rabbit charm does not + have a debug option and the config file is not in oslo config format. """ app_config = zaza.model.get_application_config(self.application_name) logging.info("Triggering deferred restart via config change") - new_debug_value = str(int(app_config['connection-backlog'].get('value', 100) + 1)) + new_debug_value = str( + int(app_config['connection-backlog'].get('value', 100) + 1)) logging.info("Setting connection-backlog: {}".format(new_debug_value)) zaza.model.set_application_config( self.application_name, From 8af30a5275ad6af342696bae642b03937f9a7fd1 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 25 Mar 2021 11:34:43 +0000 Subject: [PATCH 125/180] Pin lxml to 4.6.2 See https://github.com/openstack-charmers/zaza-openstack-tests/issues/530 Issue: Python 3.5. is deprecated/EOL Py35 is used on Xenial Xenial is used as the test platform to run tests. lxml is used by zaza-openstack-tests for the SAML assertions (because they are XML) lxml > 4.6.2 releases are after py3.5 has been deprecated/EOL. Therefore, no wheels are now built for lxml for py35. Thus there is a binary dependency to build lxml for testing --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index 78a41f0..adec5bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +# pin lxml < 4.6.3 for py35 as no wheels exist for 4.6.3 (deprecated platform) +# This is necessary for Xenial builders +# BUG: https://github.com/openstack-charmers/zaza-openstack-tests/issues/530 +lxml<4.6.3 aiounittest async_generator boto3 From e9f11da5df5c536eb953f4e4d6f99a68fd840079 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Thu, 25 Mar 2021 14:47:42 +0000 Subject: [PATCH 126/180] Fix typo and rename class for better understanding Fixes a typo (None -> not None) and renames the class from ObjectRetrier -> ObjectRetrierWraps to make it clearer that the class instantiation is to wrap and object with the retrier code rather than do retries at that moment. --- unit_tests/utilities/test_utilities.py | 26 +++++++++++---------- zaza/openstack/charm_tests/octavia/tests.py | 8 +++---- zaza/openstack/utilities/__init__.py | 21 +++++++++-------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/unit_tests/utilities/test_utilities.py b/unit_tests/utilities/test_utilities.py index a78600c..6af6089 100644 --- a/unit_tests/utilities/test_utilities.py +++ b/unit_tests/utilities/test_utilities.py @@ -31,7 +31,7 @@ class SomeException3(Exception): pass -class TestObjectRetrier(ut_utils.BaseTestCase): +class TestObjectRetrierWraps(ut_utils.BaseTestCase): def test_object_wrap(self): @@ -41,7 +41,7 @@ class TestObjectRetrier(ut_utils.BaseTestCase): return a + b a = A() - wrapped_a = utilities.ObjectRetrier(a) + wrapped_a = utilities.ObjectRetrierWraps(a) self.assertEqual(wrapped_a.func(3), 4) def test_object_multilevel_wrap(self): @@ -59,7 +59,7 @@ class TestObjectRetrier(ut_utils.BaseTestCase): return A() b = B() - wrapped_b = utilities.ObjectRetrier(b) + wrapped_b = utilities.ObjectRetrierWraps(b) self.assertEqual(wrapped_b.f2.f1(5, 6), 30) def test_object_wrap_number(self): @@ -75,7 +75,7 @@ class TestObjectRetrier(ut_utils.BaseTestCase): return a * b a = A() - wrapped_a = utilities.ObjectRetrier(a) + wrapped_a = utilities.ObjectRetrierWraps(a) self.assertEqual(wrapped_a.class_a, 5) self.assertEqual(wrapped_a.instance_a, 10) @@ -89,15 +89,15 @@ class TestObjectRetrier(ut_utils.BaseTestCase): a = A() # retry on a specific exception - wrapped_a = utilities.ObjectRetrier(a, num_retries=1, - retry_exceptions=[SomeException]) + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=1, retry_exceptions=[SomeException]) with self.assertRaises(SomeException): wrapped_a.func() mock_sleep.assert_called_once_with(5) # also retry on any exception if none specified - wrapped_a = utilities.ObjectRetrier(a, num_retries=1) + wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=1) mock_sleep.reset_mock() with self.assertRaises(SomeException): wrapped_a.func() @@ -105,8 +105,8 @@ class TestObjectRetrier(ut_utils.BaseTestCase): mock_sleep.assert_called_once_with(5) # no retry if exception isn't listed. - wrapped_a = utilities.ObjectRetrier(a, num_retries=1, - retry_exceptions=[SomeException2]) + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=1, retry_exceptions=[SomeException2]) mock_sleep.reset_mock() with self.assertRaises(SomeException): wrapped_a.func() @@ -123,7 +123,8 @@ class TestObjectRetrier(ut_utils.BaseTestCase): a = A() mock_log = mock.Mock() - wrapped_a = utilities.ObjectRetrier(a, num_retries=1, log=mock_log) + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=1, log=mock_log) with self.assertRaises(SomeException): wrapped_a.func() @@ -140,7 +141,7 @@ class TestObjectRetrier(ut_utils.BaseTestCase): raise SomeException() a = A() - wrapped_a = utilities.ObjectRetrier(a, num_retries=3, backoff=2) + wrapped_a = utilities.ObjectRetrierWraps(a, num_retries=3, backoff=2) with self.assertRaises(SomeException): wrapped_a.func() # Note third call hits maximum wait time of 15. @@ -157,7 +158,8 @@ class TestObjectRetrier(ut_utils.BaseTestCase): raise SomeException() a = A() - wrapped_a = utilities.ObjectRetrier(a, num_retries=3, total_wait=9) + wrapped_a = utilities.ObjectRetrierWraps( + a, num_retries=3, total_wait=9) with self.assertRaises(SomeException): wrapped_a.func() # Note only two calls, as total wait is 9, so a 3rd retry would exceed diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 9a71b49..535d7aa 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -25,7 +25,7 @@ import osc_lib.exceptions import zaza.openstack.charm_tests.test_utils as test_utils import zaza.openstack.utilities.openstack as openstack_utils -from zaza.openstack.utilities import ObjectRetrier +from zaza.openstack.utilities import ObjectRetrierWraps class CharmOperationTest(test_utils.OpenStackBaseTest): @@ -65,11 +65,11 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run class setup for running LBaaSv2 service tests.""" super(LBAASv2Test, cls).setUpClass() - cls.keystone_client = ObjectRetrier( + cls.keystone_client = ObjectRetrierWraps( openstack_utils.get_keystone_session_client(cls.keystone_session)) - cls.neutron_client = ObjectRetrier( + cls.neutron_client = ObjectRetrierWraps( openstack_utils.get_neutron_session_client(cls.keystone_session)) - cls.octavia_client = ObjectRetrier( + cls.octavia_client = ObjectRetrierWraps( openstack_utils.get_octavia_session_client(cls.keystone_session)) cls.RESOURCE_PREFIX = 'zaza-octavia' diff --git a/zaza/openstack/utilities/__init__.py b/zaza/openstack/utilities/__init__.py index 6db3ce3..5798f26 100644 --- a/zaza/openstack/utilities/__init__.py +++ b/zaza/openstack/utilities/__init__.py @@ -20,7 +20,7 @@ import time from keystoneauth1.exceptions.connection import ConnectFailure -class ObjectRetrier(object): +class ObjectRetrierWraps(object): """An automatic retrier for an object. This is designed to be used with an instance of an object. Basically, it @@ -35,7 +35,7 @@ class ObjectRetrier(object): # get a client that does 3 retries, waits 5 seconds between retries and # retries on any error. - some_client = ObjectRetrier(get_some_client) + some_client = ObjectRetrierWraps(get_some_client) # this gets retried up to 3 times. things = some_client.list_things() @@ -96,7 +96,7 @@ class ObjectRetrier(object): # will fail with an attribute error. attr = getattr(self.__obj, name) if callable(attr) or hasattr(attr, "__getattr__"): - return ObjectRetrier(attr, **self.__kwargs) + return ObjectRetrierWraps(attr, **self.__kwargs) else: return attr # TODO(ajkavanagh): Note detecting a property is a bit trickier. we @@ -122,8 +122,8 @@ class ObjectRetrier(object): try: return obj(*args, **kwargs) except Exception as e: - # if retry_exceptions is None, or the type of the exception is - # not in the list of retries, then raise an exception + # if retry_exceptions is not None, or the type of the exception + # is not in the list of retries, then raise an exception # immediately. This means that if retry_exceptions is None, # then the method is always retried. if (retry_exceptions is not None and @@ -148,19 +148,20 @@ class ObjectRetrier(object): def retry_on_connect_failure(client, **kwargs): """Retry an object that eventually gets resolved to a call. - Specifically, this uses ObjectRetrier but only against the + Specifically, this uses ObjectRetrierWraps but only against the keystoneauth1.exceptions.connection.ConnectFailure exeception. :params client: the object that may throw and exception when called. :type client: Any - :params **kwargs: the arguments supplied to the ObjectRetrier init method + :params **kwargs: the arguments supplied to the ObjectRetrierWraps init + method :type **kwargs: Dict[Any] - :returns: client wrapped in an ObjectRetrier instance - :rtype: ObjectRetrier[client] + :returns: client wrapped in an ObjectRetrierWraps instance + :rtype: ObjectRetrierWraps[client] """ kwcopy = kwargs.copy() if 'retry_exceptions' not in kwcopy: kwcopy['retry_exceptions'] = [] if ConnectFailure not in kwcopy['retry_exceptions']: kwcopy['retry_exceptions'].append(ConnectFailure) - return ObjectRetrier(client, **kwcopy) + return ObjectRetrierWraps(client, **kwcopy) From e5d2acc88c399fb549ac5d3971c8bff98775c534 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 26 Mar 2021 16:58:08 +0000 Subject: [PATCH 127/180] Fix test_vault_reload race condition The test_vault_reload test was checking for running config without giving the config-change time to complete. Simply adding an idle wait resolves the issue. Closes Issue: #528 --- zaza/openstack/charm_tests/vault/tests.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index c5211e0..29719d5 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -273,23 +273,33 @@ class VaultTest(BaseVaultTest): running_config = vault_utils.get_running_config(lead_client) value_to_set = not running_config['data']['disable_mlock'] + logging.info("Setting disable-mlock to {}".format(str(value_to_set))) zaza.model.set_application_config( 'vault', {'disable-mlock': str(value_to_set)}) - logging.info("Testing reload") + logging.info("Waiting for modle to be idle ...") + zaza.model.block_until_all_units_idle(model_name=self.model_name) + + logging.info("Testing action reload on {}".format(lead_client)) zaza.model.run_action( juju_utils.get_unit_name_from_ip_address( lead_client.addr, 'vault'), 'reload', model_name=self.model_name) + logging.info("Getting new value ...") new_value = vault_utils.get_running_config(lead_client)[ 'data']['disable_mlock'] - logging.info(new_value) + + logging.info( + "Asserting new value {} is equal to set value {}" + .format(new_value, value_to_set)) self.assertEqual( value_to_set, new_value) + + logging.info("Asserting not sealed") self.assertFalse(lead_client.hvac_client.seal_status['sealed']) def test_vault_restart(self): From 88fe0af9cb2f39ea804a958ccf5c0e1a09ccd2ae Mon Sep 17 00:00:00 2001 From: mkalcok <69471063+mkalcok@users.noreply.github.com> Date: Sat, 27 Mar 2021 09:27:02 +0100 Subject: [PATCH 128/180] Functional tests for nova-compute 'cloud' actions (#470) * Functional test for 'enable' and 'disable' actions on nova-compute * lint fixes * Put tests of new functionality into separate class * Added tests for actions `remove-from-cloud` and `register-to-cloud` * Set longer timeout for action verification * Add logging to help with CI debugging * Raise error if action fails * Cleanup running VMs before test starts * Add cleanup to tests that create VMs --- zaza/openstack/charm_tests/nova/tests.py | 143 +++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index edad683..f457ea9 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -19,6 +19,8 @@ import json import logging import unittest +from configparser import ConfigParser +from time import sleep import zaza.model import zaza.openstack.charm_tests.glance.setup as glance_setup @@ -47,6 +49,10 @@ class CirrosGuestCreateTest(test_utils.OpenStackBaseTest): self.launch_guest( 'cirros', instance_key=glance_setup.CIRROS_IMAGE_NAME) + def tearDown(self): + """Cleanup of VM guests.""" + self.resource_cleanup() + class LTSGuestCreateTest(test_utils.OpenStackBaseTest): """Tests to launch a LTS image.""" @@ -57,6 +63,10 @@ class LTSGuestCreateTest(test_utils.OpenStackBaseTest): self.launch_guest( 'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME) + def tearDown(self): + """Cleanup of VM guests.""" + self.resource_cleanup() + class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): """Tests to launch a LTS image.""" @@ -68,6 +78,139 @@ class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): 'volume-backed-ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME, use_boot_volume=True) + def tearDown(self): + """Cleanup of VM guests.""" + self.resource_cleanup() + + +class CloudActions(test_utils.OpenStackBaseTest): + """Test actions from actions/cloud.py.""" + + def fetch_nova_service_hostname(self, unit_name): + """ + Fetch hostname used to register with nova-cloud-controller. + + When nova-compute registers with nova-cloud-controller it uses either + config variable from '/etc/nova/nova.conf` or host's hostname to + identify itself. We need to fetch this value directly from the unit, + otherwise it's not possible to correlate entries from + `nova service-list` with nova-compute units. + + :param unit_name: nova-compute unit name. + :return: hostname used when registering to cloud-controller + """ + nova_cfg = ConfigParser() + + result = zaza.model.run_on_unit(unit_name, + 'cat /etc/nova/nova.conf') + nova_cfg.read_string(result['Stdout']) + + try: + nova_service_name = nova_cfg['DEFAULT']['host'] + except KeyError: + # Fallback to hostname if 'host' variable is not present in the + # config + result = zaza.model.run_on_unit(unit_name, 'hostname') + nova_service_name = result['Stdout'].rstrip('\n') + + if not nova_service_name: + self.fail("Failed to fetch nova service name from" + " nova-compute unit.") + return nova_service_name + + def test_940_enable_disable_actions(self): + """Test disable/enable actions on nova-compute units.""" + nova_units = zaza.model.get_units('nova-compute', + model_name=self.model_name) + + # Check that nova-compute services are enabled before testing + for service in self.nova_client.services.list(binary='nova-compute'): + self.assertEqual(service.status, 'enabled') + + # Run 'disable' action on units + zaza.model.run_action_on_units([unit.name for unit in nova_units], + 'disable') + + # Check action results via nova API + for service in self.nova_client.services.list(binary='nova-compute'): + self.assertEqual(service.status, 'disabled') + + # Run 'enable' action on units + zaza.model.run_action_on_units([unit.name for unit in nova_units], + 'enable') + + # Check action results via nova API + for service in self.nova_client.services.list(binary='nova-compute'): + self.assertEqual(service.status, 'enabled') + + def test_950_remove_from_cloud_actions(self): + """Test actions remove-from-cloud and register-to-cloud. + + Note (martin-kalcok): This test requires that nova-compute unit is not + running any VMs. If there are any leftover VMs from previous tests, + action `remove-from-cloud` will fail. + """ + all_units = zaza.model.get_units('nova-compute', + model_name=self.model_name) + + unit_to_remove = all_units[0] + + service_name = self.fetch_nova_service_hostname(unit_to_remove.name) + + registered_nova_services = self.nova_client.services.list( + host=service_name, binary='nova-compute') + + service_count = len(registered_nova_services) + if service_count < 1: + self.fail("Unit '{}' has no nova-compute services registered in" + " nova-cloud-controller".format(unit_to_remove.name)) + elif service_count > 1: + self.fail("Unexpected number of nova-compute services registered" + " in nova-cloud controller. Expecting: 1, found: " + "{}".format(service_count)) + + # run action remove-from-cloud and wait for the results in + # nova-cloud-controller + zaza.model.run_action_on_units([unit_to_remove.name], + 'remove-from-cloud', + raise_on_failure=True) + + # Wait for nova-compute service to be removed from the + # nova-cloud-controller + sleep_timeout = 1 # don't waste 10 seconds on the first run + + for _ in range(31): + sleep(sleep_timeout) + service_list = self.nova_client.services.list( + host=service_name, binary='nova-compute') + if len(service_list) == 0: + break + sleep_timeout = 10 + else: + self.fail("nova-compute service was not unregistered from the " + "nova-cloud-controller as expected.") + + # run action register-to-cloud to revert previous action + # and wait for the results in nova-cloud-controller + zaza.model.run_action_on_units([unit_to_remove.name], + 'register-to-cloud', + raise_on_failure=True) + + # Wait for nova-compute service to be registered to the + # nova-cloud-controller + sleep_timeout = 1 # don't waste 10 seconds on the first run + + for _ in range(31): + sleep(sleep_timeout) + service_list = self.nova_client.services.list( + host=service_name, binary='nova-compute') + if len(service_list) == 1: + break + sleep_timeout = 10 + else: + self.fail("nova-compute service was not re-registered to the " + "nova-cloud-controller as expected.") + class NovaCompute(test_utils.OpenStackBaseTest): """Run nova-compute specific tests.""" From 0fdd876213079cfd1adfdea2d29ee23eb87bc2a0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 29 Mar 2021 10:41:46 +0000 Subject: [PATCH 129/180] Update inline with recent charm changes --- .../charm_tests/rabbitmq_server/tests.py | 13 +- zaza/openstack/charm_tests/test_utils.py | 129 ++++++++++++------ 2 files changed, 93 insertions(+), 49 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 14fadc4..13d0e52 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -443,7 +443,7 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): restart_package_service='rabbitmq-server', application_name='rabbitmq-server') - def trigger_deferred_restart_via_charm(self): + def trigger_deferred_hook_via_charm(self): """Set charm config option which requires a service start. Set the charm debug option and wait for that change to be renderred in @@ -460,12 +460,9 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): zaza.model.set_application_config( self.application_name, {'connection-backlog': new_debug_value}) - logging.info("Waiting for connection-backlog to be {} in {}".format( - new_debug_value, - self.restart_config_file)) - zaza.model.block_until_file_matches_re( - self.application_name, - self.restart_config_file, - '{{backlog, {}}}'.format(new_debug_value)) logging.info("Waiting for units to be idle") + test_unit = zaza.model.get_units(self.application_name)[0] + zaza.model.block_until_unit_wl_message_match( + test_unit.entity_id, + status_pattern='.*config-changed.*') zaza.model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 26a9917..94ac235 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -723,7 +723,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): NOTE: The test has been broken into various class methods which may require specialisation if the charm being tested is not a standard OpenStack - charm e.g. `trigger_deferred_restart_via_charm` if the charm is not + charm e.g. `trigger_deferred_hook_via_charm` if the charm is not an oslo config or does not have a debug option. """ @@ -770,10 +770,36 @@ class BaseDeferredRestartTest(OpenStackBaseTest): # Check workload status no longer shows deferred restarts. for unit in model.get_units(self.application_name): - assert 'Services queued' not in unit.workload_status_message + assert unit.workload_status_message == 'Unit is ready' - def check_show_deferred_restarts_action(self, test_service, - restart_reason): + def check_clear_hooks(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in model.get_units(self.application_name): + model.run_action( + unit.entity_id, + 'run-deferred-hooks') + + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + assert unit.workload_status_message == 'Unit is ready' + + def run_show_deferred_events_action(self): + """Run show-deferred-events and return results. + + :returns: Data from action run + :rtype: Dict + """ + unit = model.get_units(self.application_name)[0] + action = model.run_action(unit.entity_id, 'show-deferred-events') + return yaml.safe_load(action.data['results']['output']) + + def check_show_deferred_events_action_restart(self, test_service, + restart_reason): """Check the output from the action to list deferred restarts. Run the action to list any deferred restarts and check it has entry for @@ -787,24 +813,44 @@ class BaseDeferredRestartTest(OpenStackBaseTest): :type restart_reason: str """ # Ensure that the deferred restart and cause are listed via action - for unit in model.get_units(self.application_name): - action = model.run_action( - unit.entity_id, - 'show-deferred-restarts') - logging.info( - ("Checking {} is marked as needing restart in " - "show-deferred-restarts action on {}").format( - test_service, - unit.entity_id)) - assert action.data['status'] == 'completed' - for event in yaml.safe_load(action.data['results']['output']): - if test_service in event and restart_reason in event: - break - else: - msg = 'No entry for restart of {} for reason {} found'.format( - test_service, - restart_reason) - raise Exception(msg) + logging.info( + ("Checking {} is marked as needing restart in " + "show-deferred-events action").format( + test_service)) + for event in self.run_show_deferred_events_action()['restarts']: + logging.info("{} in {} and {} in {}".format( + test_service, + event, + restart_reason, + event)) + if test_service in event and restart_reason in event: + break + else: + msg = 'No entry for restart of {} for reason {} found'.format( + test_service, + restart_reason) + raise Exception(msg) + + def check_show_deferred_events_action_hook(self, hook): + """Check the output from the action to list deferred eveents. + + Run the action to list any deferred events and check it has entry for + the given hook. + + :param hook: Hook name + :type hook: str + """ + # Ensure that the deferred restart and cause are listed via action + logging.info( + ("Checking {} is marked as skipped in " + "show-deferred-events action").format(hook)) + for event in self.run_show_deferred_events_action()['hooks']: + logging.info("{} in {}".format(hook, event)) + if hook in event: + break + else: + msg = '{} not found in {}'.format(hook, event) + raise Exception(msg) def check_show_deferred_restarts_wlm(self, test_service): """Check the workload status message lists deferred restart. @@ -820,7 +866,16 @@ class BaseDeferredRestartTest(OpenStackBaseTest): "message of {}".format(test_service, unit.entity_id))) assert test_service in unit.workload_status_message - def trigger_deferred_restart_via_charm(self): + def check_deferred_hook_wlm(self): + """Check the workload status message lists deferred event.""" + # Ensure that the deferred restarts are visible in Juju status + for unit in model.get_units(self.application_name): + logging.info( + ("Checking {} is marked as having deferred hook in workload " + "message".format(unit.entity_id))) + assert 'config-changed' in unit.workload_status_message + + def trigger_deferred_hook_via_charm(self): """Set charm config option which requires a service start. Set the charm debug option and wait for that change to be renderred in @@ -838,17 +893,11 @@ class BaseDeferredRestartTest(OpenStackBaseTest): model.set_application_config( self.application_name, {'debug': new_debug_value}) - expected_contents = { - 'DEFAULT': { - 'debug': [new_debug_value]}} - logging.info("Waiting for debug to be {} in {}".format( - new_debug_value, - self.restart_config_file)) - model.block_until_oslo_config_entries_match( - self.application_name, - self.restart_config_file, - expected_contents) logging.info("Waiting for units to be idle") + test_unit = model.get_units(self.application_name)[0] + model.block_until_unit_wl_message_match( + test_unit.entity_id, + status_pattern='.*config-changed.*') model.block_until_all_units_idle() def trigger_deferred_restart_via_package(self): @@ -870,16 +919,14 @@ class BaseDeferredRestartTest(OpenStackBaseTest): NOTE: If this test is not relevant for the target charm then override this method and raise unittest.SkipTest """ - self.trigger_deferred_restart_via_charm() + self.trigger_deferred_hook_via_charm() - self.check_show_deferred_restarts_wlm(self.test_service) - self.check_show_deferred_restarts_action( - self.test_service, - self.restart_config_file) + self.check_deferred_hook_wlm() + self.check_show_deferred_events_action_hook('config-changed') # Rerunning to flip config option back to previous value. - self.trigger_deferred_restart_via_charm() + self.trigger_deferred_hook_via_charm() logging.info("Running restart action to clear deferred restarts") - self.check_clear_restarts() + self.check_clear_hooks() def run_package_change_test(self): """Trigger a deferred restart by updating a package. @@ -892,7 +939,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): self.trigger_deferred_restart_via_package() self.check_show_deferred_restarts_wlm(self.restart_package_service) - self.check_show_deferred_restarts_action( + self.check_show_deferred_events_action_restart( self.restart_package_service, 'Package update') logging.info("Running restart action to clear deferred restarts") From f78105a6460f9338d756bab160ddf13b295627a0 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 29 Mar 2021 14:46:04 +0000 Subject: [PATCH 130/180] Allow status message to be overridden --- zaza/openstack/charm_tests/rabbitmq_server/tests.py | 7 +++++++ zaza/openstack/charm_tests/test_utils.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index 13d0e52..c3f03fb 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -443,6 +443,13 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): restart_package_service='rabbitmq-server', application_name='rabbitmq-server') + def check_status_message_is_clear(self): + """Check each units status message show no defeerred events.""" + for unit in zaza.model.get_units(self.application_name): + assert unit.workload_status_message in [ + 'Unit is ready', + 'Unit is ready and clustered'] + def trigger_deferred_hook_via_charm(self): """Set charm config option which requires a service start. diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 94ac235..dec0cd0 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -755,6 +755,12 @@ class BaseDeferredRestartTest(OpenStackBaseTest): super(BaseDeferredRestartTest, cls).setUpClass( application_name=cls.application_name) + def check_status_message_is_clear(self): + """Check each units status message show no defeerred events.""" + # Check workload status no longer shows deferred restarts. + for unit in model.get_units(self.application_name): + assert unit.workload_status_message == 'Unit is ready' + def check_clear_restarts(self): """Clear and deferred restarts and check status. @@ -769,8 +775,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): action_params={'deferred': True}) # Check workload status no longer shows deferred restarts. - for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + self.check_status_message_is_clear() def check_clear_hooks(self): """Clear and deferred restarts and check status. @@ -785,8 +790,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): 'run-deferred-hooks') # Check workload status no longer shows deferred restarts. - for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + self.check_status_message_is_clear() def run_show_deferred_events_action(self): """Run show-deferred-events and return results. From 50c781fb53455badbc8fa724ba85bf641882be0b Mon Sep 17 00:00:00 2001 From: mkalcok <69471063+mkalcok@users.noreply.github.com> Date: Tue, 30 Mar 2021 08:44:46 +0200 Subject: [PATCH 131/180] Remove repetitive code. (#534) --- zaza/openstack/charm_tests/nova/tests.py | 42 +++++++++++------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index f457ea9..f2c68e0 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -150,6 +150,24 @@ class CloudActions(test_utils.OpenStackBaseTest): running any VMs. If there are any leftover VMs from previous tests, action `remove-from-cloud` will fail. """ + def wait_for_nova_compute_count(expected_count): + """Wait for expected number of nova compute services to be present. + + Returns True or False based on whether the expected number of nova + compute services was reached within the timeout. Checks are + performed every 10 second in the span of maximum 5 minutes. + """ + sleep_timeout = 1 # don't waste 10 seconds on the first run + + for _ in range(31): + sleep(sleep_timeout) + service_list = self.nova_client.services.list( + host=service_name, binary='nova-compute') + if len(service_list) == expected_count: + return True + sleep_timeout = 10 + return False + all_units = zaza.model.get_units('nova-compute', model_name=self.model_name) @@ -177,16 +195,7 @@ class CloudActions(test_utils.OpenStackBaseTest): # Wait for nova-compute service to be removed from the # nova-cloud-controller - sleep_timeout = 1 # don't waste 10 seconds on the first run - - for _ in range(31): - sleep(sleep_timeout) - service_list = self.nova_client.services.list( - host=service_name, binary='nova-compute') - if len(service_list) == 0: - break - sleep_timeout = 10 - else: + if not wait_for_nova_compute_count(0): self.fail("nova-compute service was not unregistered from the " "nova-cloud-controller as expected.") @@ -196,18 +205,7 @@ class CloudActions(test_utils.OpenStackBaseTest): 'register-to-cloud', raise_on_failure=True) - # Wait for nova-compute service to be registered to the - # nova-cloud-controller - sleep_timeout = 1 # don't waste 10 seconds on the first run - - for _ in range(31): - sleep(sleep_timeout) - service_list = self.nova_client.services.list( - host=service_name, binary='nova-compute') - if len(service_list) == 1: - break - sleep_timeout = 10 - else: + if not wait_for_nova_compute_count(1): self.fail("nova-compute service was not re-registered to the " "nova-cloud-controller as expected.") From fe8b78b7c6640ec40b1566aa8fd2d512c2da7476 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 14:08:54 -0700 Subject: [PATCH 132/180] Functional testing for mysql-innodb-cluster This is a change with functional test for mysql-innodb-cluster implementing `force-quorum-using-partition-of` feature. It is addressing bug: https://bugs.launchpad.net/charm-mysql-innodb-cluster/+bug/1917332 --- zaza/openstack/charm_tests/mysql/tests.py | 71 +++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 45dbf71..7af9c75 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -617,6 +617,77 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) + def test_110_force_quorum_using_partition_of(self): + """Force quorum using partition of given address. + + After outage, cluster can end up without quorum. Force it. + """ + _machines = sorted( + juju_utils.get_machine_uuids_for_application(self.application)) + # Wait until update-status hooks have completed + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # Block all traffic across mysql instances: 0<-1, 1<-2 and 2<-0 + mysql_units = [unit for unit in zaza.model.get_units(self.application)] + no_of_units = len(mysql_units) + for index, unit in enumerate(mysql_units): + next_unit = mysql_units[(index+1)%no_of_units] + ip_address = next_unit.public_address + cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) + zaza.model.async_run_on_unit(unit, cmd) + + logging.info( + "Wait till all {} units are in state 'blocked' ..." + .format(self.application)) + for unit in zaza.model.get_units(self.application): + zaza.model.block_until_unit_wl_status( + unit.entity_id, + 'blocked', + negate_match=True) + + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + + # Unblock all traffic across mysql instances + for unit in zaza.model.get_units(self.application): + cmd = "sudo iptables -F" + zaza.model.async_run_on_unit(unit, cmd) + + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + + logging.info("Execute force-quorum-using-partition-of action ...") + + # Select "quorum leader" unit + leader_unit, other_units = mysql_units[0], mysq_units[1:] + action = zaza.model.run_action( + leader_unit.entity_id, + "force-quorum-using-partition-of", + action_params={"address": leader_unit.public_ip}) + + assert "Success" in action.data["results"]["outcome"], ( + "Force quorum using partition of action failed: {}" + .format(action.data)) + + # Rejoin other units to cluster + for unit in other_units: + zaza.model.run_action( + leader_unit.entity_id, + "rejoin-instance", + action_params={"address": unit.public_ip}) + + for unit in zaza.model.get_units(self.application): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" From a891ff28d5131c0e6bc02f38b3336bfe4eb61a0d Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 16:38:21 -0700 Subject: [PATCH 133/180] Added 'i-really-mean-it' tick to force-quorum --- zaza/openstack/charm_tests/mysql/tests.py | 29 +++++++---------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 7af9c75..22a6b5c 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -618,7 +618,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): states=test_config.get("target_deploy_status", {})) def test_110_force_quorum_using_partition_of(self): - """Force quorum using partition of given address. + """Force quorum using partition of instance with given address. After outage, cluster can end up without quorum. Force it. """ @@ -653,18 +653,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): self.resolve_update_status_errors() zaza.model.block_until_all_units_idle() - # Unblock all traffic across mysql instances - for unit in zaza.model.get_units(self.application): - cmd = "sudo iptables -F" - zaza.model.async_run_on_unit(unit, cmd) - - logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() - logging.info("Execute force-quorum-using-partition-of action ...") # Select "quorum leader" unit @@ -672,18 +660,19 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): action = zaza.model.run_action( leader_unit.entity_id, "force-quorum-using-partition-of", - action_params={"address": leader_unit.public_ip}) + action_params={ + "address": leader_unit.public_ip, + 'i-really-mean-it': True + }) assert "Success" in action.data["results"]["outcome"], ( "Force quorum using partition of action failed: {}" .format(action.data)) - # Rejoin other units to cluster - for unit in other_units: - zaza.model.run_action( - leader_unit.entity_id, - "rejoin-instance", - action_params={"address": unit.public_ip}) + # Unblock all traffic across mysql instances + for unit in zaza.model.get_units(self.application): + cmd = "sudo iptables -F" + zaza.model.async_run_on_unit(unit, cmd) for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") From e8157f0df8fab193de6ac70e2ddcd5e21145206f Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 20:44:23 -0700 Subject: [PATCH 134/180] Addressed flake8 violations --- zaza/openstack/charm_tests/mysql/tests.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 22a6b5c..4e9da67 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -622,9 +622,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): After outage, cluster can end up without quorum. Force it. """ - _machines = sorted( - juju_utils.get_machine_uuids_for_application(self.application)) - # Wait until update-status hooks have completed logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() @@ -632,7 +629,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): mysql_units = [unit for unit in zaza.model.get_units(self.application)] no_of_units = len(mysql_units) for index, unit in enumerate(mysql_units): - next_unit = mysql_units[(index+1)%no_of_units] + next_unit = mysql_units[(index+1) % no_of_units] ip_address = next_unit.public_address cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) zaza.model.async_run_on_unit(unit, cmd) @@ -656,7 +653,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): logging.info("Execute force-quorum-using-partition-of action ...") # Select "quorum leader" unit - leader_unit, other_units = mysql_units[0], mysq_units[1:] + leader_unit = mysql_units[0] action = zaza.model.run_action( leader_unit.entity_id, "force-quorum-using-partition-of", From 66fd299f16eaf5f57f6db855370faa146b59daa2 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Thu, 1 Apr 2021 21:30:13 -0700 Subject: [PATCH 135/180] Fixed typo --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 4e9da67..6923bc0 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -658,7 +658,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): leader_unit.entity_id, "force-quorum-using-partition-of", action_params={ - "address": leader_unit.public_ip, + "address": leader_unit.public_address, 'i-really-mean-it': True }) From 51a53d4e864ed558f4ffafc033b683d8ca5f56a7 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 07:12:06 -0700 Subject: [PATCH 136/180] Verify at the end that cluster is in operable state --- zaza/openstack/charm_tests/mysql/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 6923bc0..6e0de25 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -671,8 +671,12 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): cmd = "sudo iptables -F" zaza.model.async_run_on_unit(unit, cmd) + logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {})) class MySQL8MigrationTests(MySQLBaseTest): From 8b47c8a37f4e9a94f0db1d6755988ccdb507f8ea Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 07:40:00 -0700 Subject: [PATCH 137/180] Change the way how I'm checking state --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 6e0de25..2226893 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -662,7 +662,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): 'i-really-mean-it': True }) - assert "Success" in action.data["results"]["outcome"], ( + assert action.data.get("results")["outcome"] is not Noe, ( "Force quorum using partition of action failed: {}" .format(action.data)) From 795272269f3f1548a9692404ea7ac5760276dd97 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 07:52:18 -0700 Subject: [PATCH 138/180] Fixed typo --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 2226893..311158a 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -662,7 +662,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): 'i-really-mean-it': True }) - assert action.data.get("results")["outcome"] is not Noe, ( + assert action.data.get("results") is not None, ( "Force quorum using partition of action failed: {}" .format(action.data)) From 9a5e334b444f1ab626aa8225f1884f6185bf7822 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 11:05:52 -0700 Subject: [PATCH 139/180] Addressed review comments --- zaza/openstack/charm_tests/mysql/tests.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 311158a..546fef3 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -644,11 +644,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): negate_match=True) logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() + zaza.model.block_until_all_units_idle() logging.info("Execute force-quorum-using-partition-of action ...") @@ -665,6 +661,9 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): assert action.data.get("results") is not None, ( "Force quorum using partition of action failed: {}" .format(action.data)) + logging.debug( + "Results from running 'force-quorum' command ...\n{}".format( + action.data) # Unblock all traffic across mysql instances for unit in zaza.model.get_units(self.application): @@ -678,6 +677,13 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" From 0858f016cdf42d9a076e9f47fe9e8d720801c261 Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 11:29:44 -0700 Subject: [PATCH 140/180] Fixed TokenError --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 546fef3..219d25b 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -663,7 +663,7 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): .format(action.data)) logging.debug( "Results from running 'force-quorum' command ...\n{}".format( - action.data) + action.data)) # Unblock all traffic across mysql instances for unit in zaza.model.get_units(self.application): From 4df0cbee8ce6a70926d1fa953c0859409000437c Mon Sep 17 00:00:00 2001 From: Dariusz Smigiel <1412617+dasm@users.noreply.github.com> Date: Fri, 2 Apr 2021 11:40:07 -0700 Subject: [PATCH 141/180] Moved block_until_idle immediately after force-quorum cmd --- zaza/openstack/charm_tests/mysql/tests.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 219d25b..5cc563d 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -665,6 +665,13 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): "Results from running 'force-quorum' command ...\n{}".format( action.data)) + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + # Unblock all traffic across mysql instances for unit in zaza.model.get_units(self.application): cmd = "sudo iptables -F" @@ -677,13 +684,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) - logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() - class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" From a6cd0ea62a0124c45039e1a151dbbd4dfdecf8db Mon Sep 17 00:00:00 2001 From: Billy Olsen Date: Thu, 8 Apr 2021 13:51:33 -0700 Subject: [PATCH 142/180] Fix Ceph service action tests The ServiceTest.test_start_specific under the ceph osd tests stops all services using ceph-osd.target. This will stop all the OSDs, but the restarting of any single OSD will restart all the OSDs. This behavior causes the charm test to fail. Unfortunately, this wasn't caught in the initial review because the test is skipped when there are fewer the 2 OSDs on a unit and the final test run skipped the test due to this condition. Fixes #542 Signed-off-by: Billy Olsen --- zaza/openstack/charm_tests/ceph/osd/tests.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/osd/tests.py b/zaza/openstack/charm_tests/ceph/osd/tests.py index 63fb128..e598a5d 100644 --- a/zaza/openstack/charm_tests/ceph/osd/tests.py +++ b/zaza/openstack/charm_tests/ceph/osd/tests.py @@ -258,8 +258,11 @@ class ServiceTest(unittest.TestCase): to_start = should_stop.pop() should_stop = [service.name for service in should_stop] + # Note: can't stop ceph-osd.target as restarting a single OSD will + # cause this to start all of the OSDs when a single one starts. logging.info("Stopping all running ceph-osd services") - service_stop_cmd = 'systemctl stop ceph-osd.target' + service_stop_cmd = '; '.join(['systemctl stop {}'.format(service) + for service in service_names]) zaza_model.run_on_unit(self.TESTED_UNIT, service_stop_cmd) wait_for_service(unit_name=self.TESTED_UNIT, From 01544d22ed8af9a779b2aeb697dbae08044dc682 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 9 Apr 2021 10:48:52 +0100 Subject: [PATCH 143/180] Deferred restart test fixes (#540) * Add ovn-chassis test * Yet another refactor to reduce the amount that implementations need to override. * Add ovn dedicated chassis support * Fix race with checking wlm --- zaza/openstack/charm_tests/neutron/tests.py | 38 ++- zaza/openstack/charm_tests/ovn/tests.py | 58 +++++ .../charm_tests/rabbitmq_server/tests.py | 68 ++--- zaza/openstack/charm_tests/test_utils.py | 235 ++++++++++++------ 4 files changed, 291 insertions(+), 108 deletions(-) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index 8a1f959..def9cda 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -1013,9 +1013,35 @@ class NeutronOVSDeferredRestartTest(test_utils.BaseDeferredRestartTest): @classmethod def setUpClass(cls): """Run setup for deferred restart tests.""" - super(NeutronOVSDeferredRestartTest, cls).setUpClass( - restart_config_file='/etc/neutron/neutron.conf', - test_service='neutron-openvswitch-agent', - restart_package='openvswitch-switch', - restart_package_service='openvswitch-switch', - application_name='neutron-openvswitch') + super().setUpClass(application_name='neutron-openvswitch') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + +class NeutronGatewayDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='neutron-gateway') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which requires a restart + self.run_charm_change_restart_test( + 'neutron-l3-agent', + '/etc/neutron/neutron.conf') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 9bea951..6b43abe 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -431,3 +431,61 @@ class OVSOVNMigrationTest(test_utils.BaseCharmTest): except KeyError: # One of the applications is not in the model, which is fine pass + + +class OVNChassisDeferredRestartTest(test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='ovn-chassis') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('configure_ovs') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. + + :returns: Config key and new value + :rtype: (str, bool) + """ + app_config = zaza.model.get_application_config(self.application_name) + return 'enable-sriov', str(not app_config['enable-sriov']['value']) + + +class OVNDedicatedChassisDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + """Deferred restart tests.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='ovn-dedicated-chassis') + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('configure_ovs') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. + + :returns: Config key and new value + :rtype: (str, bool) + """ + app_config = zaza.model.get_application_config(self.application_name) + new_value = str(not app_config['disable-mlockall']['value']) + return 'disable-mlockall', new_value diff --git a/zaza/openstack/charm_tests/rabbitmq_server/tests.py b/zaza/openstack/charm_tests/rabbitmq_server/tests.py index c3f03fb..fed72df 100644 --- a/zaza/openstack/charm_tests/rabbitmq_server/tests.py +++ b/zaza/openstack/charm_tests/rabbitmq_server/tests.py @@ -436,40 +436,50 @@ class RabbitMQDeferredRestartTest(test_utils.BaseDeferredRestartTest): @classmethod def setUpClass(cls): """Run setup for deferred restart tests.""" - super().setUpClass( - restart_config_file='/etc/rabbitmq/rabbitmq.config', - test_service='rabbitmq-server', - restart_package='rabbitmq-server', - restart_package_service='rabbitmq-server', - application_name='rabbitmq-server') + super().setUpClass(application_name='rabbitmq-server') def check_status_message_is_clear(self): """Check each units status message show no defeerred events.""" + pattern = '(Unit is ready|Unit is ready and clustered)$' for unit in zaza.model.get_units(self.application_name): - assert unit.workload_status_message in [ - 'Unit is ready', - 'Unit is ready and clustered'] + zaza.model.block_until_unit_wl_message_match( + unit.entity_id, + pattern) + zaza.model.block_until_all_units_idle() - def trigger_deferred_hook_via_charm(self): - """Set charm config option which requires a service start. + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. - Set the charm debug option and wait for that change to be renderred in - applications config file. - - This overrides the base class version as the rabbit charm does not - have a debug option and the config file is not in oslo config format. + :returns: Config key and new value + :rtype: (str, bool) """ app_config = zaza.model.get_application_config(self.application_name) - logging.info("Triggering deferred restart via config change") - new_debug_value = str( - int(app_config['connection-backlog'].get('value', 100) + 1)) - logging.info("Setting connection-backlog: {}".format(new_debug_value)) - zaza.model.set_application_config( - self.application_name, - {'connection-backlog': new_debug_value}) - logging.info("Waiting for units to be idle") - test_unit = zaza.model.get_units(self.application_name)[0] - zaza.model.block_until_unit_wl_message_match( - test_unit.entity_id, - status_pattern='.*config-changed.*') - zaza.model.block_until_all_units_idle() + new_value = str(int( + app_config['connection-backlog'].get('value', 100) + 1)) + return 'connection-backlog', new_value + + def run_tests(self): + """Run deferred restart tests.""" + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'rabbitmq-server', + 'rabbitmq-server') + + def check_clear_restarts(self): + """Clear and deferred restarts and check status. + + Clear and deferred restarts and then check the workload status message + for each unit. + """ + # Use action to run any deferred restarts + for unit in zaza.model.get_units(self.application_name): + zaza.model.run_action( + unit.entity_id, + 'restart-services', + action_params={'services': 'rabbitmq-server'}) + + # Check workload status no longer shows deferred restarts. + self.check_status_message_is_clear() diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index dec0cd0..a4fbc87 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -704,7 +704,7 @@ class OpenStackBaseTest(BaseCharmTest): return instance_1, instance_2 -class BaseDeferredRestartTest(OpenStackBaseTest): +class BaseDeferredRestartTest(BaseCharmTest): """Check deferred restarts. Example of adding a deferred restart test:: @@ -714,12 +714,17 @@ class BaseDeferredRestartTest(OpenStackBaseTest): @classmethod def setUpClass(cls): - super(NeutronOVSDeferredRestartTest, cls).setUpClass( - restart_config_file = '/etc/neutron/neutron.conf', - test_service = 'neutron-openvswitch-agent', - restart_package = 'openvswitch-switch', - restart_package_service = 'openvswitch-switch', - application_name = 'neutron-openvswitch') + super().setUpClass(application_name='neutron-openvswitch') + + def run_tests(self): + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + NOTE: The test has been broken into various class methods which may require specialisation if the charm being tested is not a standard OpenStack @@ -728,38 +733,23 @@ class BaseDeferredRestartTest(OpenStackBaseTest): """ @classmethod - def setUpClass(cls, restart_config_file, test_service, restart_package, - restart_package_service, application_name): + def setUpClass(cls, application_name): """Run test setup. - :param restart_config_file: Config file that will be changed to trigger - a service restart. - :type restart_config_file: str - :param test_service: Service that will require a restart after - restart_config_file has changed. - :type test_service: str - :param restart_package: Package that will be changed to trigger a - service restart. - :type restart_package: str - :param restart_package_service: Service that will require a restart - after restart_package has changed. - :type restart_package_service: str :param application_name: Name of application to run tests against. :type application_name: str """ - cls.restart_config_file = restart_config_file - cls.test_service = test_service - cls.restart_package = restart_package - cls.restart_package_service = restart_package_service cls.application_name = application_name - super(BaseDeferredRestartTest, cls).setUpClass( - application_name=cls.application_name) + super().setUpClass(application_name=cls.application_name) def check_status_message_is_clear(self): """Check each units status message show no defeerred events.""" # Check workload status no longer shows deferred restarts. for unit in model.get_units(self.application_name): - assert unit.workload_status_message == 'Unit is ready' + model.block_until_unit_wl_message_match( + unit.entity_id, + 'Unit is ready') + model.block_until_all_units_idle() def check_clear_restarts(self): """Clear and deferred restarts and check status. @@ -772,7 +762,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): model.run_action( unit.entity_id, 'restart-services', - action_params={'deferred': True}) + action_params={'deferred-only': True}) # Check workload status no longer shows deferred restarts. self.check_status_message_is_clear() @@ -841,7 +831,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): Run the action to list any deferred events and check it has entry for the given hook. - :param hook: Hook name + :param hook: Hook or method name :type hook: str """ # Ensure that the deferred restart and cause are listed via action @@ -870,85 +860,188 @@ class BaseDeferredRestartTest(OpenStackBaseTest): "message of {}".format(test_service, unit.entity_id))) assert test_service in unit.workload_status_message - def check_deferred_hook_wlm(self): - """Check the workload status message lists deferred event.""" + def check_deferred_hook_wlm(self, deferred_hook): + """Check the workload status message lists deferred event. + + :param deferred_hook: Hook or method name which should be showing as + deferred. + :type deferred_hook: str + """ # Ensure that the deferred restarts are visible in Juju status for unit in model.get_units(self.application_name): logging.info( ("Checking {} is marked as having deferred hook in workload " "message".format(unit.entity_id))) - assert 'config-changed' in unit.workload_status_message + assert deferred_hook in unit.workload_status_message - def trigger_deferred_hook_via_charm(self): + def get_new_config(self): + """Return the config key and new value to trigger a hook execution. + + NOTE: The implementation assumes the charm has a `debug` option and + If that is not true the derived class should override this + method. + :returns: Config key and new value + :rtype: (str, bool) + """ + app_config = model.get_application_config(self.application_name) + return 'debug', str(not app_config['debug']['value']) + + def set_new_config(self): + """Change applications charm config.""" + logging.info("Triggering deferred restart via config change") + config_key, new_value = self.get_new_config() + logging.info("Setting {}: {}".format(config_key, new_value)) + model.set_application_config( + self.application_name, + {config_key: new_value}) + return new_value + + def trigger_deferred_restart_via_charm(self, restart_config_file): """Set charm config option which requires a service start. Set the charm debug option and wait for that change to be renderred in applications config file. - NOTE: The implementation assumes the charm has a `debug` option and - self.restart_config_file in an oslo config file where that - debug option is renderred. If that is not true the specaliasation - class should override this method. + NOTE: The implementation assumes the restart_config_file in an oslo + config file. If that is not true the derived class should + override this method. + + :param restart_config_file: Config file that updated value is expected + in. + :type restart_config_file: str """ - app_config = model.get_application_config(self.application_name) - logging.info("Triggering deferred restart via config change") - new_debug_value = str(not app_config['debug']['value']) - logging.info("Setting debug: {}".format(new_debug_value)) - model.set_application_config( + new_debug_value = self.set_new_config() + expected_contents = { + 'DEFAULT': { + 'debug': [new_debug_value]}} + logging.info("Waiting for debug to be {} in {}".format( + new_debug_value, + restart_config_file)) + model.block_until_oslo_config_entries_match( self.application_name, - {'debug': new_debug_value}) + restart_config_file, + expected_contents) logging.info("Waiting for units to be idle") - test_unit = model.get_units(self.application_name)[0] - model.block_until_unit_wl_message_match( - test_unit.entity_id, - status_pattern='.*config-changed.*') model.block_until_all_units_idle() - def trigger_deferred_restart_via_package(self): - """Update a package which requires a service restart.""" + def trigger_deferred_hook_via_charm(self, deferred_hook): + """Set charm config option which requires a service start. + + Set the charm debug option and wait for that change to be rendered in + applications config file. + + :param deferred_hook: Hook or method name which should be showing as + deferred. + :type deferred_hook: str + :returns: New config value + :rtype: Union[str, int, float] + """ + new_debug_value = self.set_new_config() + for unit in model.get_units(self.application_name): + logging.info('Waiting for {} to show deferred hook'.format( + unit.entity_id)) + model.block_until_unit_wl_message_match( + unit.entity_id, + status_pattern='.*{}.*'.format(deferred_hook)) + logging.info("Waiting for units to be idle") + model.block_until_all_units_idle() + return new_debug_value + + def trigger_deferred_restart_via_package(self, restart_package): + """Update a package which requires a service restart. + + :param restart_package: Package that will be changed to trigger a + service restart. + :type restart_package: str + """ logging.info("Triggering deferred restart via package change") # Test restart requested by package for unit in model.get_units(self.application_name): model.run_on_unit( unit.entity_id, - 'dpkg-reconfigure {}; ./hooks/update-status'.format( - self.restart_package)) + ('dpkg-reconfigure {}; ' + 'JUJU_HOOK_NAME=update-status ./hooks/update-status').format( + restart_package)) - def run_charm_change_test(self): + def run_charm_change_restart_test(self, test_service, restart_config_file): """Trigger a deferred restart by updating a config file via the charm. - Trigger a config-changed hook in the charm which will update a config - file and add a deferred restart. + Trigger a hook in the charm which the charm will defer. - NOTE: If this test is not relevant for the target charm then override - this method and raise unittest.SkipTest + :param test_service: Service that should need a restart + :type test_service: str + :param restart_config_file: Config file that updated value is expected + in. + :type restart_config_file: str """ - self.trigger_deferred_hook_via_charm() + self.trigger_deferred_restart_via_charm(restart_config_file) - self.check_deferred_hook_wlm() - self.check_show_deferred_events_action_hook('config-changed') - # Rerunning to flip config option back to previous value. - self.trigger_deferred_hook_via_charm() + self.check_show_deferred_restarts_wlm(test_service) + self.check_show_deferred_events_action_restart( + test_service, + restart_config_file) logging.info("Running restart action to clear deferred restarts") + self.check_clear_restarts() + + def run_charm_change_hook_test(self, deferred_hook): + """Trigger a deferred restart by updating a config file via the charm. + + :param deferred_hook: Hook or method name which should be showing as + defeerred. + :type deferred_hook: str + """ + self.trigger_deferred_hook_via_charm(deferred_hook) + + self.check_deferred_hook_wlm(deferred_hook) + self.check_show_deferred_events_action_hook(deferred_hook) + # Rerunning to flip config option back to previous value. + self.trigger_deferred_hook_via_charm(deferred_hook) + logging.info("Running restart action to clear deferred hooks") self.check_clear_hooks() - def run_package_change_test(self): + def run_package_change_test(self, restart_package, restart_package_svc): """Trigger a deferred restart by updating a package. Update a package which requires will add a deferred restart. - NOTE: If this test is not relevant for the target charm then override - this method and raise unittest.SkipTest + :param restart_package: Package that will be changed to trigger a + service restart. + :type restart_package: str + :param restart_package_service: Service that will require a restart + after restart_package has changed. + :type restart_package_service: str """ - self.trigger_deferred_restart_via_package() + self.trigger_deferred_restart_via_package(restart_package) - self.check_show_deferred_restarts_wlm(self.restart_package_service) + self.check_show_deferred_restarts_wlm(restart_package_svc) self.check_show_deferred_events_action_restart( - self.restart_package_service, + restart_package_svc, 'Package update') logging.info("Running restart action to clear deferred restarts") self.check_clear_restarts() + def run_tests(self): + """Run charm tests. should specify which tests to run. + + The charm test that implements this test should specify which tests to + run, for example: + + def run_tests(self): + # Trigger a config change which triggers a deferred hook. + self.run_charm_change_hook_test('config-changed') + + # Trigger a config change which requires a restart + self.run_charm_change_restart_test( + 'neutron-l3-agent', + '/etc/neutron/neutron.conf') + + # Trigger a package change which requires a restart + self.run_package_change_test( + 'openvswitch-switch', + 'openvswitch-switch') + """ + raise NotImplementedError + def test_deferred_restarts(self): """Run deferred restart tests.""" app_config = model.get_application_config(self.application_name) @@ -977,11 +1070,7 @@ class BaseDeferredRestartTest(OpenStackBaseTest): else: logging.info("Auto restarts already disabled") - # Trigger a config change which requires a restart - self.run_charm_change_test() - - # Trigger a package change which requires a restart - self.run_package_change_test() + self.run_tests() # Finished so turn auto-restarts back on. logging.info("Turning on auto restarts") From 7d02173914be51b9fc8cb6d171fe5a918329c44d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 9 Apr 2021 17:10:09 +0000 Subject: [PATCH 144/180] Fix bug caused by disable-mlockall being unseet --- zaza/openstack/charm_tests/ovn/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 6b43abe..1f2f890 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -487,5 +487,5 @@ class OVNDedicatedChassisDeferredRestartTest( :rtype: (str, bool) """ app_config = zaza.model.get_application_config(self.application_name) - new_value = str(not app_config['disable-mlockall']['value']) + new_value = str(not app_config['disable-mlockall'].get('value', False)) return 'disable-mlockall', new_value From 8a04739912db7127866a99830076893466baa191 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Fri, 9 Apr 2021 16:31:35 +0000 Subject: [PATCH 145/180] Fix race exposed by rabbit charm. If there are multiple charms of a given application then a hook like leader-settings-changed can trigger a deferred hook. So a race can occur where the deferred hooks of unit1 are cleared then in the process of clearing unit2 a deferred hook is triggerred in unit1. To fix this do not block on the wl status message after the hook test completes. Also, make sure that an action failure is fatal to surface any issues. --- zaza/openstack/charm_tests/test_utils.py | 35 ++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index a4fbc87..15e90f8 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -759,26 +759,38 @@ class BaseDeferredRestartTest(BaseCharmTest): """ # Use action to run any deferred restarts for unit in model.get_units(self.application_name): + logging.info("Running restart-services on {}".format( + unit.entity_id)) model.run_action( unit.entity_id, 'restart-services', - action_params={'deferred-only': True}) + action_params={'deferred-only': True}, + raise_on_failure=True) # Check workload status no longer shows deferred restarts. self.check_status_message_is_clear() - def check_clear_hooks(self): - """Clear and deferred restarts and check status. + def clear_hooks(self): + """Clear and deferred hooks. - Clear and deferred restarts and then check the workload status message - for each unit. + Run any deferred hooks. """ # Use action to run any deferred restarts for unit in model.get_units(self.application_name): + logging.info("Running run-deferred-hooks on {}".format( + unit.entity_id)) model.run_action( unit.entity_id, - 'run-deferred-hooks') + 'run-deferred-hooks', + raise_on_failure=True) + def check_clear_hooks(self): + """Clear deferred hooks and check status. + + Clear deferred hooks and then check the workload status message + for each unit. + """ + self.clear_hooks() # Check workload status no longer shows deferred restarts. self.check_status_message_is_clear() @@ -789,7 +801,10 @@ class BaseDeferredRestartTest(BaseCharmTest): :rtype: Dict """ unit = model.get_units(self.application_name)[0] - action = model.run_action(unit.entity_id, 'show-deferred-events') + action = model.run_action( + unit.entity_id, + 'show-deferred-events', + raise_on_failure=True) return yaml.safe_load(action.data['results']['output']) def check_show_deferred_events_action_restart(self, test_service, @@ -997,7 +1012,10 @@ class BaseDeferredRestartTest(BaseCharmTest): # Rerunning to flip config option back to previous value. self.trigger_deferred_hook_via_charm(deferred_hook) logging.info("Running restart action to clear deferred hooks") - self.check_clear_hooks() + # If there are a number of units in the application and restarts take + # time then another deferred hook can occur so do not block on a + # clear status message. + self.clear_hooks() def run_package_change_test(self, restart_package, restart_package_svc): """Trigger a deferred restart by updating a package. @@ -1080,3 +1098,4 @@ class BaseDeferredRestartTest(BaseCharmTest): self.application_name, policy_file) model.block_until_all_units_idle() + self.check_clear_hooks() From d89a1651d2d8cefbe5e3a18fd725eadd486d0711 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 12 Apr 2021 09:28:31 +0000 Subject: [PATCH 146/180] Fix Neutron Gateway deferred restart test Do not run the deferred hooks action for neutron gateway as it is not needed and will end in an action error. Tested locally here: https://paste.ubuntu.com/p/cSmgxmrTr4/ This closes issue #553 --- zaza/openstack/charm_tests/neutron/tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/charm_tests/neutron/tests.py b/zaza/openstack/charm_tests/neutron/tests.py index def9cda..54420e8 100644 --- a/zaza/openstack/charm_tests/neutron/tests.py +++ b/zaza/openstack/charm_tests/neutron/tests.py @@ -1045,3 +1045,7 @@ class NeutronGatewayDeferredRestartTest(test_utils.BaseDeferredRestartTest): self.run_package_change_test( 'openvswitch-switch', 'openvswitch-switch') + + def check_clear_hooks(self): + """Gateway does not defer hooks so noop.""" + return From 669b57574b7a36e890215ffcd01049a171fff354 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 13 Apr 2021 10:24:15 +0000 Subject: [PATCH 147/180] Add deferred restart test for ovn central --- zaza/openstack/charm_tests/ovn/tests.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/zaza/openstack/charm_tests/ovn/tests.py b/zaza/openstack/charm_tests/ovn/tests.py index 1f2f890..bfa0f21 100644 --- a/zaza/openstack/charm_tests/ovn/tests.py +++ b/zaza/openstack/charm_tests/ovn/tests.py @@ -489,3 +489,21 @@ class OVNDedicatedChassisDeferredRestartTest( app_config = zaza.model.get_application_config(self.application_name) new_value = str(not app_config['disable-mlockall'].get('value', False)) return 'disable-mlockall', new_value + + +class OVNCentralDeferredRestartTest( + test_utils.BaseDeferredRestartTest): + """Deferred restart tests for OVN Central.""" + + @classmethod + def setUpClass(cls): + """Run setup for deferred restart tests.""" + super().setUpClass(application_name='ovn-central') + + def run_tests(self): + """Run deferred restart tests.""" + # Charm does not defer hooks so that test is not included. + # Trigger a package change which requires a restart + self.run_package_change_test( + 'ovn-central', + 'ovn-central') From 1c15672b61cd054d3f47e7d3ad2604f711ed5d55 Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 19 Apr 2021 08:00:38 -0700 Subject: [PATCH 148/180] MySQL Partition Test class Currently the new partition test_110 is breaking on master due to the test landing before the charm PR which implemented the change [0] has landed. Move the partition test to its own class which can be added as a target in the charm's tests.yaml. This will not affect master and can be trivially updated in [0]. [0] https://review.opendev.org/c/openstack/charm-mysql-innodb-cluster/+/779427 --- zaza/openstack/charm_tests/mysql/tests.py | 138 +++++++++++----------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 5cc563d..3438932 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -617,73 +617,6 @@ class MySQLInnoDBClusterColdStartTest(MySQLBaseTest): zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {})) - def test_110_force_quorum_using_partition_of(self): - """Force quorum using partition of instance with given address. - - After outage, cluster can end up without quorum. Force it. - """ - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() - - # Block all traffic across mysql instances: 0<-1, 1<-2 and 2<-0 - mysql_units = [unit for unit in zaza.model.get_units(self.application)] - no_of_units = len(mysql_units) - for index, unit in enumerate(mysql_units): - next_unit = mysql_units[(index+1) % no_of_units] - ip_address = next_unit.public_address - cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) - zaza.model.async_run_on_unit(unit, cmd) - - logging.info( - "Wait till all {} units are in state 'blocked' ..." - .format(self.application)) - for unit in zaza.model.get_units(self.application): - zaza.model.block_until_unit_wl_status( - unit.entity_id, - 'blocked', - negate_match=True) - - logging.info("Wait till model is idle ...") - zaza.model.block_until_all_units_idle() - - logging.info("Execute force-quorum-using-partition-of action ...") - - # Select "quorum leader" unit - leader_unit = mysql_units[0] - action = zaza.model.run_action( - leader_unit.entity_id, - "force-quorum-using-partition-of", - action_params={ - "address": leader_unit.public_address, - 'i-really-mean-it': True - }) - - assert action.data.get("results") is not None, ( - "Force quorum using partition of action failed: {}" - .format(action.data)) - logging.debug( - "Results from running 'force-quorum' command ...\n{}".format( - action.data)) - - logging.info("Wait till model is idle ...") - try: - zaza.model.block_until_all_units_idle() - except zaza.model.UnitError: - self.resolve_update_status_errors() - zaza.model.block_until_all_units_idle() - - # Unblock all traffic across mysql instances - for unit in zaza.model.get_units(self.application): - cmd = "sudo iptables -F" - zaza.model.async_run_on_unit(unit, cmd) - - logging.info("Wait for application states ...") - for unit in zaza.model.get_units(self.application): - zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") - test_config = lifecycle_utils.get_charm_config(fatal=False) - zaza.model.wait_for_application_states( - states=test_config.get("target_deploy_status", {})) - class MySQL8MigrationTests(MySQLBaseTest): """Percona Cluster to MySQL InnoDB Cluster Tests.""" @@ -930,6 +863,77 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): .format(action.data)) +class MySQLInnoDBClusterPartitionTest(MySQLBaseTest): + """MySQL parition handling.""" + + def test_850_force_quorum_using_partition_of(self): + """Force quorum using partition of instance with given address. + + After outage, cluster can end up without quorum. Force it. + """ + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # Block all traffic across mysql instances: 0<-1, 1<-2 and 2<-0 + mysql_units = [unit for unit in zaza.model.get_units(self.application)] + no_of_units = len(mysql_units) + for index, unit in enumerate(mysql_units): + next_unit = mysql_units[(index+1) % no_of_units] + ip_address = next_unit.public_address + cmd = "sudo iptables -A INPUT -s {} -j DROP".format(ip_address) + zaza.model.async_run_on_unit(unit, cmd) + + logging.info( + "Wait till all {} units are in state 'blocked' ..." + .format(self.application)) + for unit in zaza.model.get_units(self.application): + zaza.model.block_until_unit_wl_status( + unit.entity_id, + 'blocked', + negate_match=True) + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + logging.info("Execute force-quorum-using-partition-of action ...") + + # Select "quorum leader" unit + leader_unit = mysql_units[0] + action = zaza.model.run_action( + leader_unit.entity_id, + "force-quorum-using-partition-of", + action_params={ + "address": leader_unit.public_address, + 'i-really-mean-it': True + }) + + assert action.data.get("results") is not None, ( + "Force quorum using partition of action failed: {}" + .format(action.data)) + logging.debug( + "Results from running 'force-quorum' command ...\n{}".format( + action.data)) + + logging.info("Wait till model is idle ...") + try: + zaza.model.block_until_all_units_idle() + except zaza.model.UnitError: + self.resolve_update_status_errors() + zaza.model.block_until_all_units_idle() + + # Unblock all traffic across mysql instances + for unit in zaza.model.get_units(self.application): + cmd = "sudo iptables -F" + zaza.model.async_run_on_unit(unit, cmd) + + logging.info("Wait for application states ...") + for unit in zaza.model.get_units(self.application): + zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") + test_config = lifecycle_utils.get_charm_config(fatal=False) + zaza.model.wait_for_application_states( + states=test_config.get("target_deploy_status", {})) + + class MySQLRouterTests(test_utils.OpenStackBaseTest): """MySQL Router Tests.""" From 49929ef01885666123918b12c8a624b5294ef0b2 Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 19 Apr 2021 09:05:45 -0700 Subject: [PATCH 149/180] Fix typo --- zaza/openstack/charm_tests/mysql/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index 3438932..bd054e2 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -864,7 +864,7 @@ class MySQLInnoDBClusterScaleTest(MySQLBaseTest): class MySQLInnoDBClusterPartitionTest(MySQLBaseTest): - """MySQL parition handling.""" + """MySQL partition handling.""" def test_850_force_quorum_using_partition_of(self): """Force quorum using partition of instance with given address. From 0b401de17f1e2814f46184e5df0ec49d140902c8 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 19 Apr 2021 17:03:05 -0400 Subject: [PATCH 150/180] Get OpenStack codename from /etc/openstack-release In newer versions of Ubuntu and the Cloud Archive (currently Hirsute, Wallaby, and above), there is an openstack-release package that, if installed, specifies the current OpenStack release in /etc/openstack-release. Also adds wallaby definitions to non-version definitions in os_versions.py. Fixes #560 --- .../test_zaza_utilities_openstack.py | 36 ++++++++++++- zaza/openstack/utilities/openstack.py | 50 +++++++++++++++++-- zaza/openstack/utilities/os_versions.py | 4 +- 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index c42fc9d..b7f2eeb 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -884,7 +884,24 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): result = openstack_utils.get_current_os_release_pair() self.assertEqual(expected, result) - def test_get_openstack_release(self): + def test_get_current_os_versions(self): + self.patch_object(openstack_utils, "get_openstack_release") + self.patch_object(openstack_utils.generic_utils, "get_pkg_version") + + # Pre-Wallaby scenario where openstack-release package isn't installed + self.get_openstack_release.return_value = None + self.get_pkg_version.return_value = '18.0.0' + expected = {'keystone': 'victoria'} + result = openstack_utils.get_current_os_versions('keystone') + self.assertEqual(expected, result) + + # Wallaby+ scenario where openstack-release package is installed + self.get_openstack_release.return_value = 'wallaby' + expected = {'keystone': 'wallaby'} + result = openstack_utils.get_current_os_versions('keystone') + self.assertEqual(expected, result) + + def test_get_os_release(self): self.patch( 'zaza.openstack.utilities.openstack.get_current_os_release_pair', new_callable=mock.MagicMock(), @@ -936,6 +953,23 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): self.get_application_config_option.return_value = None self.assertEqual(openstack_utils.get_keystone_api_version(), 3) + def test_get_openstack_release(self): + self.patch_object(openstack_utils.model, "get_units") + self.patch_object(openstack_utils.juju_utils, "remote_run") + + # Test pre-Wallaby behavior where openstack-release pkg isn't installed + self.get_units.return_value = [] + self.remote_run.return_value = "OPENSTACK_CODENAME=wallaby " + + # Test Wallaby+ behavior where openstack-release package is installed + unit1 = mock.MagicMock() + unit1.entity_id = 1 + self.get_units.return_value = [unit1] + self.remote_run.return_value = "OPENSTACK_CODENAME=wallaby " + + result = openstack_utils.get_openstack_release("application", "model") + self.assertEqual(result, "wallaby") + def test_get_project_id(self): # No domain self.patch_object(openstack_utils, "get_keystone_api_version") diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 230a3f6..75ab9d7 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1753,9 +1753,44 @@ def get_os_code_info(package, pkg_version): return OPENSTACK_CODENAMES[vers] +def get_openstack_release(application, model_name=None): + """Return the openstack release codename based on /etc/openstack-release. + + This will only return a codename if the openstack-release package is + installed on the unit. + + :param application: Application name + :type application: string + :param model_name: Name of model to query. + :type model_name: str + :returns: OpenStack release codename for application + :rtype: string + """ + versions = [] + units = model.get_units(application, model_name=model_name) + for unit in units: + cmd = 'cat /etc/openstack-release | grep OPENSTACK_CODENAME' + out = juju_utils.remote_run(unit.entity_id, cmd, model_name=model_name) + codename = out.split('=')[1].strip() + versions.append(codename) + if len(set(versions)) == 0: + return None + elif len(set(versions)) > 1: + raise Exception('Unexpected mix of OpenStack releases for {}: {}', + application, versions) + return versions[0] + + def get_current_os_versions(deployed_applications, model_name=None): """Determine OpenStack codename of deployed applications. + Initially, see if the openstack-release pkg is available and use it + instead. + + If it isn't then it falls back to the existing method of checking the + version of the package passed and then resolving the version from that + using lookup tables. + :param deployed_applications: List of deployed applications :type deployed_applications: list :param model_name: Name of model to query. @@ -1769,11 +1804,16 @@ def get_current_os_versions(deployed_applications, model_name=None): continue logging.info("looking at application: {}".format(application)) - version = generic_utils.get_pkg_version(application['name'], - application['type']['pkg'], - model_name=model_name) - versions[application['name']] = ( - get_os_code_info(application['type']['pkg'], version)) + codename = get_openstack_release(application['name'], + model_name=model_name) + if codename: + versions[application['name']] = codename + else: + version = generic_utils.get_pkg_version(application['name'], + application['type']['pkg'], + model_name=model_name) + versions[application['name']] = ( + get_os_code_info(application['type']['pkg'], version)) return versions diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index 420253c..fdb6f0f 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -36,6 +36,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([ ('eoan', 'train'), ('focal', 'ussuri'), ('groovy', 'victoria'), + ('hirsute', 'wallaby'), ]) @@ -69,7 +70,8 @@ OPENSTACK_RELEASES_PAIRS = [ 'bionic_queens', 'bionic_rocky', 'cosmic_rocky', 'bionic_stein', 'disco_stein', 'bionic_train', 'eoan_train', 'bionic_ussuri', 'focal_ussuri', - 'focal_victoria', 'groovy_victoria'] + 'focal_victoria', 'groovy_victoria', + 'focal_wallaby', 'hirsute_wallaby'] SWIFT_CODENAMES = OrderedDict([ ('diablo', From 9a48757d5e214c4c0c4e14e8528a1f895a54901c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 20 Apr 2021 12:24:23 -0400 Subject: [PATCH 151/180] Fix version fallback path when /etc/openstack-release doesn't exist --- zaza/openstack/utilities/openstack.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 75ab9d7..846cca4 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1770,9 +1770,14 @@ def get_openstack_release(application, model_name=None): units = model.get_units(application, model_name=model_name) for unit in units: cmd = 'cat /etc/openstack-release | grep OPENSTACK_CODENAME' - out = juju_utils.remote_run(unit.entity_id, cmd, model_name=model_name) - codename = out.split('=')[1].strip() - versions.append(codename) + try: + out = juju_utils.remote_run(unit.entity_id, cmd, + model_name=model_name) + except model.CommandRunFailed: + logging.info('Fall back to version check for OpenStack codename') + else: + codename = out.split('=')[1].strip() + versions.append(codename) if len(set(versions)) == 0: return None elif len(set(versions)) > 1: From d70ff79b1e5e6bfc7fcffac98ef1e80f92da4c28 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 20 Apr 2021 13:05:18 -0400 Subject: [PATCH 152/180] Change log to debug to create less noise --- zaza/openstack/utilities/openstack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 846cca4..79e1c54 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1774,7 +1774,7 @@ def get_openstack_release(application, model_name=None): out = juju_utils.remote_run(unit.entity_id, cmd, model_name=model_name) except model.CommandRunFailed: - logging.info('Fall back to version check for OpenStack codename') + logging.debug('Fall back to version check for OpenStack codename') else: codename = out.split('=')[1].strip() versions.append(codename) From a05263f111ef79e134200936815bb0a8a389c2d2 Mon Sep 17 00:00:00 2001 From: Billy Olsen Date: Thu, 22 Apr 2021 04:13:40 -0700 Subject: [PATCH 153/180] Update nova tests (#563) Add test for the new nova_cc charm action syncing AZs The new action is meant to sync the availability zones from the underlying Juju provider with the OpenStack availability zones. It is useful within MAAS environments to map the MAAZ AZs to OpenStack AZs. Also fix output check logic for sync az test. The validation logic for the sync az test checks the specific output of the command text, but this is dependent on ordering. The code also validates that the hypervisors are added to the appropriate host aggregates, so this changes the test to verify the action was successful and has some output. Also use the availability-zone based on the configured settings. In the attempt to sync the az test, the availability zone is assumed to be the JUJU_AVAILABILITY_ZONE environment setting. In various scenarios, this is not what is used for the availability zone of a compute node. This change mimics the logic that's contained within the nova-compute charm to determine which availability zone should be used for the compute node. Signed-off-by: Billy Olsen Co-authored-by: Ionut Balutoiu --- zaza/openstack/charm_tests/nova/tests.py | 56 ++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index f2c68e0..1e44e31 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -319,6 +319,62 @@ class NovaComputeActionTest(test_utils.OpenStackBaseTest): "The action failed: {}".format(action.data["message"])) +class NovaCloudControllerActionTest(test_utils.OpenStackBaseTest): + """Run nova-cloud-controller specific tests. + + Add this test class for new nova-cloud-controller action + to avoid breaking older version. + """ + + def test_sync_compute_az_action(self): + """Test sync-compute-availability-zones action.""" + juju_units_az_map = {} + compute_config = zaza.model.get_application_config('nova-compute') + default_az = compute_config['default-availability-zone']['value'] + use_juju_az = compute_config['customize-failure-domain']['value'] + + for unit in zaza.model.get_units('nova-compute', + model_name=self.model_name): + zone = default_az + if use_juju_az: + result = zaza.model.run_on_unit(unit.name, + 'echo $JUJU_AVAILABILITY_ZONE', + model_name=self.model_name, + timeout=60) + self.assertEqual(int(result['Code']), 0) + juju_az = result['Stdout'].strip() + if juju_az: + zone = juju_az + + juju_units_az_map[unit.public_address] = zone + continue + + session = openstack_utils.get_overcloud_keystone_session() + nova = openstack_utils.get_nova_session_client(session) + + result = zaza.model.run_action_on_leader( + 'nova-cloud-controller', + 'sync-compute-availability-zones', + model_name=self.model_name) + + # For validating the action results, we simply want to validate that + # the action was completed and we have something in the output. The + # functional validation really occurs below, in that the hosts are + # checked to be in the appropriate host aggregates. + self.assertEqual(result.status, 'completed') + self.assertNotEqual('', result.results['output']) + + unique_az_list = list(set(juju_units_az_map.values())) + aggregates = nova.aggregates.list() + self.assertEqual(len(aggregates), len(unique_az_list)) + for unit_address in juju_units_az_map: + az = juju_units_az_map[unit_address] + aggregate = nova.aggregates.find( + name='{}_az'.format(az), availability_zone=az) + hypervisor = nova.hypervisors.find(host_ip=unit_address) + self.assertIn(hypervisor.hypervisor_hostname, aggregate.hosts) + + class NovaCloudController(test_utils.OpenStackBaseTest): """Run nova-cloud-controller specific tests.""" From 5d533fba6af93395eff006e9912fea3b3cae4893 Mon Sep 17 00:00:00 2001 From: David Ames Date: Fri, 23 Apr 2021 09:26:52 -0700 Subject: [PATCH 154/180] Restart blocked mysql routers (#565) LP Bug #1918953 [0] was resolved on the mysql-innodb-cluster side using coordinated delayed action. Since then we have seen a similar issue in CI [1] after the pause and resume test. Mysql-router hangs with: 2021-04-21 20:42:05 metadata_cache WARNING [7f3f968d5700] Instance '192.168.254.18:3306' [72b4ac2c-a2dd-11eb-82a5-fa163e5a4b7e] of replicaset 'default' is unreachable. Increasing metadata cache refresh frequency. This cannot be fixed from the cluster side. I will be looking into solutions on the mysql-rotuer side. But in the meantime, to unblock the mysql-innodb-cluster gate this change restarts blocked MySQL routers. [0] https://bugs.launchpad.net/charm-mysql-router/+bug/1918953 [1] https://openstack-ci-reports.ubuntu.com/artifacts/test_charm_pipeline_func_full/openstack/charm-mysql-innodb-cluster/786514/3/8479/consoleText.test_charm_func_full_11494.txt --- zaza/openstack/charm_tests/mysql/tests.py | 59 +++++++++++++++++++++++ zaza/openstack/charm_tests/test_utils.py | 26 ++++++++++ 2 files changed, 85 insertions(+) diff --git a/zaza/openstack/charm_tests/mysql/tests.py b/zaza/openstack/charm_tests/mysql/tests.py index bd054e2..4044c55 100644 --- a/zaza/openstack/charm_tests/mysql/tests.py +++ b/zaza/openstack/charm_tests/mysql/tests.py @@ -113,6 +113,56 @@ class MySQLBaseTest(test_utils.OpenStackBaseTest): if _primary_ip in unit.public_address: return unit + def get_blocked_mysql_routers(self): + """Get blocked mysql routers. + + :returns: List of blocked mysql-router unit names + :rtype: List[str] + """ + # Make sure mysql-router units are up to date + # We cannot assume they are as there is up to a five minute delay + mysql_router_units = [] + for application in self.get_applications_with_substring_in_name( + "mysql-router"): + for unit in zaza.model.get_units(application): + mysql_router_units.append(unit.entity_id) + self.run_update_status_hooks(mysql_router_units) + + # Get up to date status + status = zaza.model.get_status().applications + blocked_mysql_routers = [] + # Check if the units are blocked + for application in self.get_applications_with_substring_in_name( + "mysql-router"): + # Subordinate dance with primary + # There is no satus[applicatoin]["units"] for subordinates + _subordinate_to = status[application].subordinate_to[0] + for appunit in status[_subordinate_to].units: + for subunit in ( + status[_subordinate_to]. + units[appunit].subordinates.keys()): + if "blocked" in ( + status[_subordinate_to].units[appunit]. + subordinates[subunit].workload_status.status): + blocked_mysql_routers.append(subunit) + return blocked_mysql_routers + + def restart_blocked_mysql_routers(self): + """Restart blocked mysql routers. + + :returns: None + :rtype: None + """ + # Check for blocked mysql-router units + blocked_mysql_routers = self.get_blocked_mysql_routers() + for unit in blocked_mysql_routers: + logging.warning( + "Restarting blocked mysql-router unit {}" + .format(unit)) + zaza.model.run_on_unit( + unit, + "systemctl restart {}".format(unit.rpartition("/")[0])) + class MySQLCommonTests(MySQLBaseTest): """Common mysql charm tests.""" @@ -169,6 +219,15 @@ class MySQLCommonTests(MySQLBaseTest): """ with self.pause_resume(self.services): logging.info("Testing pause resume") + + logging.info("Wait till model is idle ...") + zaza.model.block_until_all_units_idle() + + # If there are any blocekd mysql routers restart them. + self.restart_blocked_mysql_routers() + assert not self.get_blocked_mysql_routers(), ( + "Should no longer be blocked mysql-router units") + logging.info("Passed pause and resume test.") diff --git a/zaza/openstack/charm_tests/test_utils.py b/zaza/openstack/charm_tests/test_utils.py index 15e90f8..bc67933 100644 --- a/zaza/openstack/charm_tests/test_utils.py +++ b/zaza/openstack/charm_tests/test_utils.py @@ -573,6 +573,32 @@ class BaseCharmTest(unittest.TestCase): return self.test_config.get('tests_options', {}).get( '.'.join(caller_path + [key]), default) + def get_applications_with_substring_in_name(self, substring): + """Get applications with substring in name. + + :param substring: String to search for in application names + :type substring: str + :returns: List of matching applictions + :rtype: List + """ + status = model.get_status().applications + applications = [] + for application in status.keys(): + if substring in application: + applications.append(application) + return applications + + def run_update_status_hooks(self, units): + """Run update status hooks on units. + + :param units: List of unit names or unit.entity_id + :type units: List[str] + :returns: None + :rtype: None + """ + for unit in units: + model.run_on_unit(unit, "hooks/update-status") + class OpenStackBaseTest(BaseCharmTest): """Generic helpers for testing OpenStack API charms.""" From e631b9354117cae33f49f1343be0776cd4c0ca4e Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Tue, 4 May 2021 11:46:27 +0200 Subject: [PATCH 155/180] Add proper support for Octavia's tempest tests (#498) * Update tempest config for Octavia * ensure the test_server is executable * remove reference to accounts.yaml --- zaza/openstack/charm_tests/tempest/setup.py | 31 +++++++++++++++++-- .../tempest/templates/tempest_v3.j2 | 9 +++++- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index 5c85d9c..b22ccb7 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -17,6 +17,7 @@ import jinja2 import urllib.parse import os +import subprocess import zaza.utilities.deployment_env as deployment_env import zaza.openstack.utilities.juju as juju_utils @@ -170,6 +171,27 @@ def add_keystone_config(ctxt, keystone_session): ctxt['default_domain_id'] = domain.id +def add_octavia_config(ctxt): + """Add octavia config to context. + + :param ctxt: Context dictionary + :type ctxt: dict + :returns: None + :rtype: None + :raises: subprocess.CalledProcessError + """ + subprocess.check_call([ + 'curl', + "{}:80/swift/v1/fixtures/test_server.bin".format( + ctxt['test_swift_ip']), + '-o', "{}/test_server.bin".format(ctxt['workspace_path']) + ]) + subprocess.check_call([ + 'chmod', '+x', + "{}/test_server.bin".format(ctxt['workspace_path']) + ]) + + def get_service_list(keystone_session): """Retrieve list of services from keystone. @@ -227,7 +249,7 @@ def add_auth_config(ctxt): overcloud_auth['OS_PROJECT_DOMAIN_NAME']) -def get_tempest_context(): +def get_tempest_context(workspace_path): """Generate the tempest config context. :returns: Context dictionary @@ -235,6 +257,7 @@ def get_tempest_context(): """ keystone_session = openstack_utils.get_overcloud_keystone_session() ctxt = {} + ctxt['workspace_path'] = workspace_path ctxt_funcs = { 'nova': add_nova_config, 'neutron': add_neutron_config, @@ -253,6 +276,7 @@ def get_tempest_context(): ctxt_func(ctxt, keystone_session) add_environment_var_config(ctxt, ctxt['enabled_services']) add_auth_config(ctxt) + add_octavia_config(ctxt) return ctxt @@ -289,13 +313,14 @@ def setup_tempest(tempest_template, accounts_template): workspace_name, workspace_path = tempest_utils.get_workspace() tempest_utils.destroy_workspace(workspace_name, workspace_path) tempest_utils.init_workspace(workspace_path) + context = get_tempest_context(workspace_path) render_tempest_config( os.path.join(workspace_path, 'etc/tempest.conf'), - get_tempest_context(), + context, tempest_template) render_tempest_config( os.path.join(workspace_path, 'etc/accounts.yaml'), - get_tempest_context(), + context, accounts_template) diff --git a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 index b441f2d..5cb65ed 100644 --- a/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 +++ b/zaza/openstack/charm_tests/tempest/templates/tempest_v3.j2 @@ -4,7 +4,6 @@ use_stderr = false log_file = tempest.log [auth] -test_accounts_file = accounts.yaml default_credentials_domain_name = {{ default_credentials_domain_name }} admin_username = {{ admin_username }} admin_project_name = {{ admin_project_name }} @@ -65,6 +64,7 @@ floating_network_name = {{ ext_net }} [network-feature-enabled] ipv6 = false api_extensions = {{ neutron_api_extensions }} +port_security = true {% endif %} {% if 'heat' in enabled_services %} @@ -104,3 +104,10 @@ catalog_type = {{ catalog_type }} [volume-feature-enabled] backup = false {% endif %} + +{% if 'octavia' in enabled_services %} +[load_balancer] +enable_security_groups = true +test_with_ipv6 = false +test_server_path = {{ workspace_path }}/test_server.bin +{% endif %} \ No newline at end of file From cd2df7534c33424d4881823f8cb154f2ead62765 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 5 May 2021 11:39:53 +0000 Subject: [PATCH 156/180] Seperate TrilioGhostNFSShareTest from TrilioDataMoverTest as its not needed for s3 --- zaza/openstack/charm_tests/trilio/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 581c395..0bf0cd8 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -463,7 +463,7 @@ class TrilioDMAPITest(TrilioBaseTest): services = ["dmapi-api"] -class TrilioDataMoverTest(TrilioGhostNFSShareTest): +class TrilioDataMoverTest(TrilioBaseTest): """Tests for Trilio Data Mover charm.""" conf_file = "/etc/tvault-contego/tvault-contego.conf" From 8e998c2d02b2f66522e844e83bba451d376f4e50 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 6 May 2021 08:22:10 +0000 Subject: [PATCH 157/180] More seperation --- zaza/openstack/charm_tests/trilio/tests.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 0bf0cd8..5072b29 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -463,10 +463,18 @@ class TrilioDMAPITest(TrilioBaseTest): services = ["dmapi-api"] -class TrilioDataMoverTest(TrilioBaseTest): +class TrilioDataMoverBaseTest(TrilioBaseTest): """Tests for Trilio Data Mover charm.""" conf_file = "/etc/tvault-contego/tvault-contego.conf" application_name = "trilio-data-mover" services = ["tvault-contego"] + + +class TrilioDataMoverNFSTest(TrilioDataMoverBaseTest, TrilioGhostNFSShareTest): + """Tests for Trilio Data Mover charm backed by NFS.""" + + +class TrilioDataMoverS3Test(TrilioDataMoverBaseTest): + """Tests for Trilio Data Mover charm backed by S3.""" From 779137f60fbcda8197a63d1d93380d5cad962223 Mon Sep 17 00:00:00 2001 From: Chris MacNaughton Date: Thu, 6 May 2021 14:12:33 +0200 Subject: [PATCH 158/180] Restrict adding Octavia config Only add Octavia's config when Octavia is in the enabled services --- zaza/openstack/charm_tests/tempest/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/tempest/setup.py b/zaza/openstack/charm_tests/tempest/setup.py index b22ccb7..7ae0ba3 100644 --- a/zaza/openstack/charm_tests/tempest/setup.py +++ b/zaza/openstack/charm_tests/tempest/setup.py @@ -276,7 +276,8 @@ def get_tempest_context(workspace_path): ctxt_func(ctxt, keystone_session) add_environment_var_config(ctxt, ctxt['enabled_services']) add_auth_config(ctxt) - add_octavia_config(ctxt) + if 'octavia' in ctxt['enabled_services']: + add_octavia_config(ctxt) return ctxt From 03ddaf0079a31277b2ca0c635f1f29454cd56612 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 10 May 2021 13:44:24 +0200 Subject: [PATCH 159/180] Add more logging around downloading/caching test images --- zaza/openstack/utilities/openstack.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 79e1c54..237e02d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2478,7 +2478,11 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], local_path = os.path.join(image_cache_dir, img_name) if not os.path.exists(local_path): + logging.info('Downloading {} ...'.format(image_url)) download_image(image_url, local_path) + else: + logging.info('Cached image found at {} - Skipping download'.format( + local_path)) image = upload_image_to_glance( glance, local_path, image_name, backend=backend, From 7dd96c8fb1ddfaf85539fa77809eef670b084fd4 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Tue, 11 May 2021 10:51:18 +0200 Subject: [PATCH 160/180] Revert "Skip OVN provider octavia test on Victoria" This reverts commit c4691ef1c74365bf2a33f4922f852247e205e8bb. --- zaza/openstack/charm_tests/octavia/tests.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/tests.py b/zaza/openstack/charm_tests/octavia/tests.py index 535d7aa..52083ab 100644 --- a/zaza/openstack/charm_tests/octavia/tests.py +++ b/zaza/openstack/charm_tests/octavia/tests.py @@ -180,14 +180,6 @@ class LBAASv2Test(test_utils.OpenStackBaseTest): for provider in octavia_client.provider_list().get('providers', []) if provider['name'] != 'octavia' # alias for `amphora`, skip } - if (openstack_utils.get_os_release() in [ - openstack_utils.get_os_release('focal_victoria'), - openstack_utils.get_os_release('groovy_victoria')]): - logging.info("Skipping tests of ovn backed lb (Bug #1896603)") - try: - del providers['ovn'] - except KeyError: - pass return providers def _create_lb_resources(self, octavia_client, provider, vip_subnet_id, From 3aacd5c89c5ab66403036a60e02421b3207de020 Mon Sep 17 00:00:00 2001 From: mkalcok <69471063+mkalcok@users.noreply.github.com> Date: Thu, 13 May 2021 12:48:06 +0200 Subject: [PATCH 161/180] Functional tests for `instance-count` action on nova-compute charm. (#535) Launchpad: #1911011 Gerrit review: https://review.opendev.org/c/openstack/charm-nova-compute/+/783371 * Functional tests for `instance-count` action on nova-compute charm. Test for `remove_from_cloud_actions` waits at the end until the nova-compute service is available again. However there's a few seconds lag until that host shows up also in `hypervisor-list` which can cause race conditions with other tests. --- zaza/openstack/charm_tests/nova/tests.py | 34 +++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 1e44e31..5f6709a 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -143,7 +143,39 @@ class CloudActions(test_utils.OpenStackBaseTest): for service in self.nova_client.services.list(binary='nova-compute'): self.assertEqual(service.status, 'enabled') - def test_950_remove_from_cloud_actions(self): + def test_950_instance_count_action(self): + """Test that action 'instance-count' returns expected values.""" + def check_instance_count(expect_count, unit_name): + """Assert that unit with 'unit_name' has 'expect_count' of VMs. + + :param expect_count: How many VMs are expected to be running + :param unit_name: Name of the target nova-compute unit + :return: None + :raises AssertionError: If result of the 'instance-count' action + does not match 'expect_count'. + """ + logging.debug('Running "instance-count" action on unit "{}".' + 'Expecting result: {}'.format(unit_name, + expect_count)) + result = zaza.model.run_action(unit_name, 'instance-count') + self.assertEqual(result.status, 'completed') + instances = result.data.get('results', {}).get('instance-count') + self.assertEqual(instances, str(expect_count)) + + nova_unit = zaza.model.get_units('nova-compute', + model_name=self.model_name)[0] + + check_instance_count(0, nova_unit.entity_id) + + self.RESOURCE_PREFIX = 'zaza-nova' + self.launch_guest( + 'ubuntu', instance_key=glance_setup.LTS_IMAGE_NAME) + + check_instance_count(1, nova_unit.entity_id) + + self.resource_cleanup() + + def test_960_remove_from_cloud_actions(self): """Test actions remove-from-cloud and register-to-cloud. Note (martin-kalcok): This test requires that nova-compute unit is not From 1c4a117c3aa8280e0c1cd057fb09bf412c3ee257 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Mon, 17 May 2021 11:12:13 +0000 Subject: [PATCH 162/180] Add ovn codename wallaby (20.12) --- zaza/openstack/utilities/os_versions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/utilities/os_versions.py b/zaza/openstack/utilities/os_versions.py index fdb6f0f..beaedec 100644 --- a/zaza/openstack/utilities/os_versions.py +++ b/zaza/openstack/utilities/os_versions.py @@ -121,6 +121,8 @@ OVN_CODENAMES = OrderedDict([ ['20.03']), ('victoria', ['20.06']), + ('wallaby', + ['20.12']), ]) # >= Liberty version->codename mapping From 620a098e8d4d714324269f94a436d3fe02556db3 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 18 May 2021 07:56:34 +0000 Subject: [PATCH 163/180] Add debugging --- zaza/openstack/charm_tests/trilio/setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zaza/openstack/charm_tests/trilio/setup.py b/zaza/openstack/charm_tests/trilio/setup.py index 1c8e97c..11fe187 100644 --- a/zaza/openstack/charm_tests/trilio/setup.py +++ b/zaza/openstack/charm_tests/trilio/setup.py @@ -39,12 +39,14 @@ def nfs_setup(): trilio_wlm_unit = zaza_model.get_first_unit_name("trilio-wlm") nfs_shares_conf = {"nfs-shares": "{}:/srv/testing".format(nfs_server_ip)} + logging.info("NFS share config: {}".format(nfs_shares_conf)) _trilio_services = ["trilio-wlm", "trilio-data-mover"] conf_changed = False for juju_service in _trilio_services: app_config = zaza_model.get_application_config(juju_service) if app_config["nfs-shares"] != nfs_shares_conf["nfs-shares"]: + logging.info("Updating nfs-shares config option") zaza_model.set_application_config(juju_service, nfs_shares_conf) conf_changed = True From d51a28dc0435f05a323a8fb426ab09f921d5139e Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 13 May 2021 06:15:46 +0200 Subject: [PATCH 164/180] Re-order change of expected workload status After recent changes to zaza workload status code the present ordering does no longer work. --- zaza/openstack/charm_tests/octavia/setup.py | 23 ++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/zaza/openstack/charm_tests/octavia/setup.py b/zaza/openstack/charm_tests/octavia/setup.py index 4b09aa0..b04b696 100644 --- a/zaza/openstack/charm_tests/octavia/setup.py +++ b/zaza/openstack/charm_tests/octavia/setup.py @@ -107,23 +107,36 @@ def configure_octavia(): 'amp-ssh-pub-key': base64.b64encode( bytes(ssh_public_key, 'utf-8')).decode('utf-8'), } - logging.info('Configuring certificates for mandatory Octavia ' - 'client/server authentication ' - '(client being the ``Amphorae`` load balancer instances)') + + # Tell Octavia charm it is safe to create cloud resources, we do this now + # because the workload status will be checked on config-change and it gets + # a bit complicated to augment test config to accept 'blocked' vs. 'active' + # in the various stages. + logging.info('Running `configure-resources` action on Octavia leader unit') + zaza.model.run_action_on_leader( + 'octavia', + 'configure-resources', + action_params={}) # Our expected workload status will change after we have configured the # certificates test_config = zaza.charm_lifecycle.utils.get_charm_config() del test_config['target_deploy_status']['octavia'] + logging.info('Configuring certificates for mandatory Octavia ' + 'client/server authentication ' + '(client being the ``Amphorae`` load balancer instances)') + _singleton = zaza.openstack.charm_tests.test_utils.OpenStackBaseTest() _singleton.setUpClass(application_name='octavia') with _singleton.config_change(charm_config, charm_config): # wait for configuration to be applied then return pass - # Tell Octavia charm it is safe to create cloud resources - logging.info('Running `configure-resources` action on Octavia leader unit') + # Should we consider making the charm attempt to create this key on + # config-change? + logging.info('Running `configure-resources` action again to ensure ' + 'Octavia Nova SSH key pair is created after config change.') zaza.model.run_action_on_leader( 'octavia', 'configure-resources', From be1706cb76d6cc2f7d7b20bbf3d69e63f21e8cd1 Mon Sep 17 00:00:00 2001 From: Nobuto Murata Date: Fri, 21 May 2021 15:19:28 +0900 Subject: [PATCH 165/180] Functional testing for glance external S3 backend Set up Ceph-Radosgw in the same model as an external S3 backend for Glance: - create an S3 user in Radosgw - create an S3 bucket for Glance - inject those parameters into Glance charm configurations as an external S3 backend - upload an image to the S3 backend via Glance, and check the size of the image in the S3 bucket directly Co-authored-by: Connor Chamberlain --- zaza/openstack/charm_tests/glance/setup.py | 59 ++++++++++++++++++++++ zaza/openstack/charm_tests/glance/tests.py | 56 +++++++++++++++++++- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index 6686d70..32f7d4e 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -14,7 +14,11 @@ """Code for configuring glance.""" +import json import logging + +import boto3 +import zaza.model as model import zaza.openstack.utilities.openstack as openstack_utils import zaza.utilities.deployment_env as deployment_env @@ -161,3 +165,58 @@ def add_lts_image(glance_client=None, image_name=None, release=None, glance_client=glance_client, image_name=image_name, properties=properties) + + +def configure_external_s3_backend(): + """Set up Ceph-radosgw as an external S3 backend for Glance.""" + logging.info("Creating a test S3 user and credentials for Glance") + username, displayname = "zaza-glance-test", "Zaza Glance Test User" + cmd = "radosgw-admin user create --uid='{}' --display-name='{}'".format( + username, displayname + ) + results = model.run_on_leader("ceph-mon", cmd) + stdout = json.loads(results["stdout"]) + keys = stdout["keys"][0] + access_key, secret_key = keys["access_key"], keys["secret_key"] + + logging.info("Getting S3 endpoint URL of Radosgw from Keystone") + keystone_auth = openstack_utils.get_overcloud_auth() + keystone_client = openstack_utils.get_keystone_client(keystone_auth) + endpoint_url = keystone_client.session.get_endpoint( + service_type="s3", + interface="public", + region="RegionOne", + ) + + logging.info("Creating a test S3 bucketfor Glance") + bucket_name = "zaza-glance-s3-test" + s3_client = boto3.client( + "s3", + endpoint_url=endpoint_url, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + ) + s3_client.create_bucket(Bucket=bucket_name) + + logging.info("Updating Glance configs with S3 endpoint infomation") + model.set_application_config( + "glance", + { + "s3-store-host": endpoint_url, + "s3-store-access-key": access_key, + "s3-store-secret-key": secret_key, + "s3-store-bucket": bucket_name, + }, + ) + model.wait_for_agent_status() + + logging.info("Waiting for units to reach target states") + model.wait_for_application_states( + states={ + "glance": { + "workload-status": "active", + "workload-status-message": "Unit is ready", + } + } + ) + model.block_until_all_units_idle() diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index 5d45338..799e69a 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -18,8 +18,10 @@ import logging -import zaza.openstack.utilities.openstack as openstack_utils +import boto3 +import zaza.model as model import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.openstack.utilities.openstack as openstack_utils class GlanceTest(test_utils.OpenStackBaseTest): @@ -142,3 +144,55 @@ class GlanceCephRGWBackendTest(test_utils.OpenStackBaseTest): 'image size {}'.format(image['size'], total_bytes)) self.assertEqual(image['size'], total_bytes) openstack_utils.delete_image(self.glance_client, image['id']) + + +class GlanceExternalS3Test(test_utils.OpenStackBaseTest): + """Encapsulate glance tests using an external S3 backend.""" + + @classmethod + def setUpClass(cls): + """Run class setup for running glance tests with S3 backend.""" + super(GlanceExternalS3Test, cls).setUpClass() + cls.glance_client = openstack_utils.get_glance_session_client( + cls.keystone_session + ) + + configs = model.get_application_config("glance") + cls.s3_store_host = configs["s3-store-host"]["value"] + cls.s3_store_access_key = configs["s3-store-access-key"]["value"] + cls.s3_store_secret_key = configs["s3-store-secret-key"]["value"] + cls.s3_store_bucket = configs["s3-store-bucket"]["value"] + + def test_100_create_delete_image(self): + """Create an image and do a simple validation of it. + + Validate the size of the image in both Glance API and actual S3 bucket. + """ + image_name = "zaza-s3-test-image" + openstack_utils.create_image( + glance=self.glance_client, + image_url=openstack_utils.find_cirros_image(arch="x86_64"), + image_name=image_name, + backend="s3", + ) + images = openstack_utils.get_images_by_name( + self.glance_client, image_name + ) + self.assertEqual(len(images), 1) + image = images[0] + + s3_client = boto3.client( + "s3", + endpoint_url=self.s3_store_host, + aws_access_key_id=self.s3_store_access_key, + aws_secret_access_key=self.s3_store_secret_key, + ) + response = s3_client.head_object( + Bucket=self.s3_store_bucket, Key=image["id"] + ) + logging.info( + "Checking glance image size {} matches S3 object's ContentLength " + "{}".format(image["size"], response["ContentLength"]) + ) + self.assertEqual(image["size"], response["ContentLength"]) + openstack_utils.delete_image(self.glance_client, image["id"]) From 6a688d29bb242eee8874e3c9d4c95cb438f91d4a Mon Sep 17 00:00:00 2001 From: Liam Young Date: Tue, 25 May 2021 13:18:11 +0000 Subject: [PATCH 166/180] Add classes to collect NFS and S3 Trilio tests --- zaza/openstack/charm_tests/trilio/tests.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 5072b29..64239ae 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -440,7 +440,7 @@ class TrilioGhostNFSShareTest(TrilioBaseTest): ) -class TrilioWLMTest(TrilioGhostNFSShareTest): +class TrilioWLMBaseTest(TrilioBaseTest): """Tests for Trilio Workload Manager charm.""" conf_file = "/etc/workloadmgr/workloadmgr.conf" @@ -478,3 +478,11 @@ class TrilioDataMoverNFSTest(TrilioDataMoverBaseTest, TrilioGhostNFSShareTest): class TrilioDataMoverS3Test(TrilioDataMoverBaseTest): """Tests for Trilio Data Mover charm backed by S3.""" + + +class TrilioWLMNFSTest(TrilioWLMBaseTest, TrilioGhostNFSShareTest): + """Tests for Trilio WLM charm backed by NFS.""" + + +class TrilioWLMS3Test(TrilioWLMBaseTest): + """Tests for Trilio WLM charm backed by S3.""" From 855847339404f7cb37b63e65a7f38e1a812d4980 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 26 May 2021 13:49:46 +0200 Subject: [PATCH 167/180] Re-enable test_003_test_override_is_observed This test was disabled for releases older than groovy-victoria because of https://bugs.launchpad.net/ubuntu/+source/python-oslo.policy/+bug/1880959 This bug has now been fixed and released for xenial-queens and newer. Also made it more robust in GlanceTests // fixes #578 --- zaza/openstack/charm_tests/policyd/tests.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index 4722755..a87f2ef 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -401,9 +401,10 @@ class BasePolicydSpecialization(PolicydTest, def test_003_test_override_is_observed(self): """Test that the override is observed by the underlying service.""" if (openstack_utils.get_os_release() < - openstack_utils.get_os_release('groovy_victoria')): + openstack_utils.get_os_release('xenial_queens')): raise unittest.SkipTest( - "Test skipped until Bug #1880959 is fix released") + "Test skipped because bug #1880959 won't be fixed for " + "releases older than Queens") if self._test_name is None: logging.info("Doing policyd override for {}" .format(self._service_name)) @@ -571,6 +572,13 @@ class GlanceTests(BasePolicydSpecialization): super(GlanceTests, cls).setUpClass(application_name="glance") cls.application_name = "glance" + # NOTE(lourot): Same as NeutronApiTests. There is a race between the glance + # charm signalling its readiness and the service actually being ready to + # serve requests. The test will fail intermittently unless we gracefully + # accept this. + # Issue: openstack-charmers/zaza-openstack-tests#578 + @tenacity.retry(wait=tenacity.wait_fixed(1), + reraise=True, stop=tenacity.stop_after_delay(8)) def get_client_and_attempt_operation(self, ip): """Attempt to list the images as a policyd override. From 7ba3141cdb8c530d3bf92a4743c062141169a8db Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 27 May 2021 13:26:58 +0200 Subject: [PATCH 168/180] glance_simplestreams_sync/setup: Add tenacity The very nature of the action ran by the configure step will cause it to fail frequently unless it is retried. Fixes #580 --- .../glance_simplestreams_sync/setup.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py index 06e172d..ec6107b 100644 --- a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py @@ -17,6 +17,7 @@ """Code for configuring glance-simplestreams-sync.""" import logging +import tenacity import zaza.model as zaza_model import zaza.openstack.utilities.generic as generic_utils @@ -30,11 +31,17 @@ def sync_images(): deployment. """ logging.info("Synchronising images using glance-simplestreams-sync") - generic_utils.assertActionRanOK( - zaza_model.run_action_on_leader( - "glance-simplestreams-sync", - "sync-images", - raise_on_failure=True, - action_params={}, - ) - ) + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10), + reraise=True): + with attempt: + generic_utils.assertActionRanOK( + zaza_model.run_action_on_leader( + "glance-simplestreams-sync", + "sync-images", + raise_on_failure=True, + action_params={}, + ) + ) From 2e7d52b9cda5ca4a2ca07f45ddf6f9ad4396c0b5 Mon Sep 17 00:00:00 2001 From: Nobuto Murata Date: Fri, 28 May 2021 22:12:51 +0900 Subject: [PATCH 169/180] fix typo --- zaza/openstack/charm_tests/glance/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/glance/setup.py b/zaza/openstack/charm_tests/glance/setup.py index 32f7d4e..367c980 100644 --- a/zaza/openstack/charm_tests/glance/setup.py +++ b/zaza/openstack/charm_tests/glance/setup.py @@ -188,7 +188,7 @@ def configure_external_s3_backend(): region="RegionOne", ) - logging.info("Creating a test S3 bucketfor Glance") + logging.info("Creating a test S3 bucket for Glance") bucket_name = "zaza-glance-s3-test" s3_client = boto3.client( "s3", @@ -198,7 +198,7 @@ def configure_external_s3_backend(): ) s3_client.create_bucket(Bucket=bucket_name) - logging.info("Updating Glance configs with S3 endpoint infomation") + logging.info("Updating Glance configs with S3 endpoint information") model.set_application_config( "glance", { From e67b0ea1e11e5f7bb0a7058199a765077caabb8c Mon Sep 17 00:00:00 2001 From: Hernan Garcia Date: Wed, 26 May 2021 07:10:45 -0500 Subject: [PATCH 170/180] added glance functional test for image-conversion config --- .../test_zaza_utilities_openstack.py | 6 +++-- zaza/openstack/charm_tests/glance/tests.py | 26 +++++++++++++++++++ zaza/openstack/utilities/openstack.py | 23 +++++++++++++--- 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/unit_tests/utilities/test_zaza_utilities_openstack.py b/unit_tests/utilities/test_zaza_utilities_openstack.py index b7f2eeb..0434a2b 100644 --- a/unit_tests/utilities/test_zaza_utilities_openstack.py +++ b/unit_tests/utilities/test_zaza_utilities_openstack.py @@ -550,7 +550,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): backend=None, disk_format='qcow2', visibility='public', - container_format='bare') + container_format='bare', + force_import=False) def test_create_image_pass_directory(self): glance_mock = mock.MagicMock() @@ -574,7 +575,8 @@ class TestOpenStackUtils(ut_utils.BaseTestCase): backend=None, disk_format='qcow2', visibility='public', - container_format='bare') + container_format='bare', + force_import=False) self.gettempdir.assert_not_called() def test_create_ssh_key(self): diff --git a/zaza/openstack/charm_tests/glance/tests.py b/zaza/openstack/charm_tests/glance/tests.py index 5d45338..d81e440 100644 --- a/zaza/openstack/charm_tests/glance/tests.py +++ b/zaza/openstack/charm_tests/glance/tests.py @@ -67,6 +67,32 @@ class GlanceTest(test_utils.OpenStackBaseTest): {'image_format': {'disk_formats': ['qcow2']}}, ['glance-api']) + def test_412_image_conversion(self): + """Check image-conversion config. + + When image-conversion config is enabled glance will convert images + to raw format, this is only performed for interoperable image import + docs.openstack.org/glance/train/admin/interoperable-image-import.html + image conversion is done at server-side for better image handling + """ + current_release = openstack_utils.get_os_release() + bionic_stein = openstack_utils.get_os_release('bionic_stein') + if current_release < bionic_stein: + self.skipTest('image-conversion config is supported since ' + 'bionic_stein or newer versions') + + with self.config_change({'image-conversion': 'false'}, + {'image-conversion': 'true'}): + image_url = openstack_utils.find_cirros_image(arch='x86_64') + image = openstack_utils.create_image( + self.glance_client, + image_url, + 'cirros-test-import', + force_import=True) + + disk_format = self.glance_client.images.get(image.id).disk_format + self.assertEqual('raw', disk_format) + def test_900_restart_on_config_change(self): """Checking restart happens on config change.""" # Config file affected by juju set config change diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 237e02d..3082068 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -2407,7 +2407,7 @@ def delete_volume_backup(cinder, vol_backup_id): def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', visibility='public', container_format='bare', - backend=None): + backend=None, force_import=False): """Upload the given image to glance and apply the given label. :param glance: Authenticated glanceclient @@ -2424,6 +2424,9 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', format that also contains metadata about the actual virtual machine. :type container_format: str + :param force_import: Force the use of glance image import + instead of direct upload + :type force_import: boolean :returns: glance image pointer :rtype: glanceclient.common.utils.RequestIdProxy """ @@ -2433,7 +2436,15 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', disk_format=disk_format, visibility=visibility, container_format=container_format) - glance.images.upload(image.id, open(local_path, 'rb'), backend=backend) + + if force_import: + logging.info('Forcing image import') + glance.images.stage(image.id, open(local_path, 'rb')) + glance.images.image_import( + image.id, method='glance-direct', backend=backend) + else: + glance.images.upload( + image.id, open(local_path, 'rb'), backend=backend) resource_reaches_status( glance.images, @@ -2446,7 +2457,8 @@ def upload_image_to_glance(glance, local_path, image_name, disk_format='qcow2', def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], properties=None, backend=None, disk_format='qcow2', - visibility='public', container_format='bare'): + visibility='public', container_format='bare', + force_import=False): """Download the image and upload it to glance. Download an image from image_url and upload it to glance labelling @@ -2465,6 +2477,9 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], :type tags: list of str :param properties: Properties and values to add to image :type properties: dict + :param force_import: Force the use of glance image import + instead of direct upload + :type force_import: boolean :returns: glance image pointer :rtype: glanceclient.common.utils.RequestIdProxy """ @@ -2487,7 +2502,7 @@ def create_image(glance, image_url, image_name, image_cache_dir=None, tags=[], image = upload_image_to_glance( glance, local_path, image_name, backend=backend, disk_format=disk_format, visibility=visibility, - container_format=container_format) + container_format=container_format, force_import=force_import) for tag in tags: result = glance.image_tags.update(image.id, tag) logging.debug( From 7e46cde193cfa1f17b68cda261f40f9515825076 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Wed, 2 Jun 2021 15:12:13 +0200 Subject: [PATCH 171/180] policyd: Add keystoneauth1 403 to accepted exceptions Fixes #584 --- zaza/openstack/charm_tests/policyd/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/policyd/tests.py b/zaza/openstack/charm_tests/policyd/tests.py index a87f2ef..fde3866 100644 --- a/zaza/openstack/charm_tests/policyd/tests.py +++ b/zaza/openstack/charm_tests/policyd/tests.py @@ -688,5 +688,6 @@ class OctaviaTests(BasePolicydSpecialization): self.get_keystone_session_admin_user(ip)) try: octavia_client.provider_list() - except octaviaclient.OctaviaClientException: + except (octaviaclient.OctaviaClientException, + keystoneauth1.exceptions.http.Forbidden): raise PolicydOperationFailedException() From 38581b70e533c16b0f191874eefac44a735b91b1 Mon Sep 17 00:00:00 2001 From: Frode Nordahl Date: Thu, 3 Jun 2021 08:11:54 +0200 Subject: [PATCH 172/180] gss: Log Keystone service catalog on failure The simplestreams service relies on the Keystone Service catalog. To help debug any issues occurring during test execution, log the contents of the Keystone service catalog in the event of failure. Related-Bug: #1930654 --- .../glance_simplestreams_sync/setup.py | 58 ++++++++++++++----- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py index ec6107b..96648af 100644 --- a/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py +++ b/zaza/openstack/charm_tests/glance_simplestreams_sync/setup.py @@ -18,9 +18,31 @@ import logging import tenacity +import pprint import zaza.model as zaza_model import zaza.openstack.utilities.generic as generic_utils +import zaza.openstack.utilities.openstack as openstack_utils + + +def _get_catalog(): + """Retrieve the Keystone service catalog. + + :returns: The raw Keystone service catalog. + :rtype: List[Dict] + """ + keystone_session = openstack_utils.get_overcloud_keystone_session() + keystone_client = openstack_utils.get_keystone_session_client( + keystone_session) + + token = keystone_session.get_token() + token_data = keystone_client.tokens.get_token_data(token) + + if 'catalog' not in token_data['token']: + raise ValueError('catalog not in token data: "{}"' + .format(pprint.pformat(token_data))) + + return token_data['token']['catalog'] def sync_images(): @@ -31,17 +53,27 @@ def sync_images(): deployment. """ logging.info("Synchronising images using glance-simplestreams-sync") - for attempt in tenacity.Retrying( - stop=tenacity.stop_after_attempt(3), - wait=tenacity.wait_exponential( - multiplier=1, min=2, max=10), - reraise=True): - with attempt: - generic_utils.assertActionRanOK( - zaza_model.run_action_on_leader( - "glance-simplestreams-sync", - "sync-images", - raise_on_failure=True, - action_params={}, + + catalog = None + try: + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=1, min=2, max=10), + reraise=True): + with attempt: + # Proactively retrieve the Keystone service catalog so that we + # can log it in the event of a failure. + catalog = _get_catalog() + generic_utils.assertActionRanOK( + zaza_model.run_action_on_leader( + "glance-simplestreams-sync", + "sync-images", + raise_on_failure=True, + action_params={}, + ) ) - ) + except Exception: + logging.info('Contents of Keystone service catalog: "{}"' + .format(pprint.pformat(catalog))) + raise From 6645c60e93bd70ae5ef0302cacf63d83299f418d Mon Sep 17 00:00:00 2001 From: David Ames Date: Wed, 9 Jun 2021 03:20:53 -0700 Subject: [PATCH 173/180] Handle Vault running tests twice (#588) Vault's runs its tests twice. In some circumstances the removal of the vault key from the test_config dictionary can throw a KeyError. Do not fail when this happens. --- zaza/openstack/charm_tests/vault/tests.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 29719d5..9014928 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -111,7 +111,11 @@ class UnsealVault(BaseVaultTest): vault_utils.run_charm_authorize(self.vault_creds['root_token']) if not test_config: test_config = lifecycle_utils.get_charm_config() - del test_config['target_deploy_status']['vault'] + try: + del test_config['target_deploy_status']['vault'] + except KeyError: + # Already removed + pass zaza.model.wait_for_application_states( states=test_config.get('target_deploy_status', {})) @@ -154,7 +158,11 @@ class VaultTest(BaseVaultTest): allowed_domains='openstack.local') test_config = lifecycle_utils.get_charm_config() - del test_config['target_deploy_status']['vault'] + try: + del test_config['target_deploy_status']['vault'] + except KeyError: + # Already removed + pass zaza.openstack.utilities.openstack.block_until_ca_exists( 'keystone', cacert.decode().strip()) From dee6032fc3bafe57f1098e89b1ab7a65bd7a081d Mon Sep 17 00:00:00 2001 From: Nobuto Murata Date: Sun, 13 Jun 2021 16:51:51 +0900 Subject: [PATCH 174/180] Skip test_vault_reload with enabling/disabling mlock in LXD When Vault is running in a LXD container, test_vault_reload is expected to fail because mlock cannot be enabled in LXD containers. Skip it when running in containers is detected. It's useful for local development environments with Juju's localhost/LXD provider. The test won't be skipped in the OpenStack Charm CI since the Vault units will be run in KVM, not in LXD containers. --- zaza/openstack/charm_tests/vault/tests.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 9014928..6058b21 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -18,6 +18,7 @@ import contextlib import hvac +import json import logging import time import unittest @@ -277,6 +278,15 @@ class VaultTest(BaseVaultTest): raise unittest.SkipTest("The version of charm-vault tested does " "not have reload action") + container_results = zaza.model.run_on_leader( + "vault", "systemd-detect-virt --container" + ) + container_rc = json.loads(container_results["Code"]) + if container_rc == 0: + raise unittest.SkipTest( + "Vault unit is running in a container. Cannot use mlock." + ) + lead_client = vault_utils.get_cluster_leader(self.clients) running_config = vault_utils.get_running_config(lead_client) value_to_set = not running_config['data']['disable_mlock'] @@ -286,7 +296,7 @@ class VaultTest(BaseVaultTest): 'vault', {'disable-mlock': str(value_to_set)}) - logging.info("Waiting for modle to be idle ...") + logging.info("Waiting for model to be idle ...") zaza.model.block_until_all_units_idle(model_name=self.model_name) logging.info("Testing action reload on {}".format(lead_client)) From e9cb5aea5af93e99e65cb98da39b27250e45a953 Mon Sep 17 00:00:00 2001 From: David Ames Date: Mon, 14 Jun 2021 20:25:19 +0000 Subject: [PATCH 175/180] Retry request on test csr for Vault --- zaza/openstack/charm_tests/vault/tests.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/vault/tests.py b/zaza/openstack/charm_tests/vault/tests.py index 6058b21..9b898bf 100644 --- a/zaza/openstack/charm_tests/vault/tests.py +++ b/zaza/openstack/charm_tests/vault/tests.py @@ -24,6 +24,7 @@ import time import unittest import uuid import tempfile +import tenacity import requests import zaza.charm_lifecycle.utils as lifecycle_utils @@ -171,10 +172,19 @@ class VaultTest(BaseVaultTest): states=test_config.get('target_deploy_status', {})) ip = zaza.model.get_app_ips( 'keystone')[0] + with tempfile.NamedTemporaryFile(mode='w') as fp: fp.write(cacert.decode()) fp.flush() - requests.get('https://{}:5000'.format(ip), verify=fp.name) + # Avoid race condition and retry + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3), + wait=tenacity.wait_exponential( + multiplier=2, min=2, max=10)): + with attempt: + logging.info( + "Attempting to connect to https://{}:5000".format(ip)) + requests.get('https://{}:5000'.format(ip), verify=fp.name) def test_all_clients_authenticated(self): """Check all vault clients are authenticated.""" From b7e542636f742f022fe449633789770cae67c84d Mon Sep 17 00:00:00 2001 From: Liam Young Date: Wed, 16 Jun 2021 09:14:29 +0100 Subject: [PATCH 176/180] Extend trilio snapshot timeout (#594) Extend trilio snapshot timeout, closes issue #595. --- zaza/openstack/charm_tests/trilio/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/trilio/tests.py b/zaza/openstack/charm_tests/trilio/tests.py index 64239ae..ad477d8 100644 --- a/zaza/openstack/charm_tests/trilio/tests.py +++ b/zaza/openstack/charm_tests/trilio/tests.py @@ -262,7 +262,7 @@ class WorkloadmgrCLIHelper(object): retryer = tenacity.Retrying( wait=tenacity.wait_exponential(multiplier=1, max=30), - stop=tenacity.stop_after_delay(720), + stop=tenacity.stop_after_delay(900), reraise=True, ) From 8ba11347870dd7c8d2be00f3b973c3326a52d9a3 Mon Sep 17 00:00:00 2001 From: Rodrigo Barbieri Date: Thu, 17 Jun 2021 15:18:29 -0300 Subject: [PATCH 177/180] Add pci-alias test for nova-compute (#587) Moved test that was previously existing for nova-cloud-controller to a common class to be run for nova-compute as well. --- zaza/openstack/charm_tests/nova/tests.py | 161 ++++++++++++----------- 1 file changed, 87 insertions(+), 74 deletions(-) diff --git a/zaza/openstack/charm_tests/nova/tests.py b/zaza/openstack/charm_tests/nova/tests.py index 5f6709a..66ca957 100644 --- a/zaza/openstack/charm_tests/nova/tests.py +++ b/zaza/openstack/charm_tests/nova/tests.py @@ -83,6 +83,79 @@ class LTSGuestCreateVolumeBackedTest(test_utils.OpenStackBaseTest): self.resource_cleanup() +class NovaCommonTests(test_utils.OpenStackBaseTest): + """nova-compute and nova-cloud-controller common tests.""" + + XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') + XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') + XENIAL_QUEENS = openstack_utils.get_os_release('xenial_queens') + BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens') + BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky') + BIONIC_TRAIN = openstack_utils.get_os_release('bionic_train') + + @classmethod + def setUpClass(cls): + """Run class setup for running nova-cloud-controller tests.""" + super(NovaCommonTests, cls).setUpClass() + cls.current_release = openstack_utils.get_os_release() + + def _test_pci_alias_config(self, app_name, service_list): + logging.info('Checking pci aliases in nova config...') + + # Expected default and alternate values + current_value = zaza.model.get_application_config( + app_name)['pci-alias'] + try: + current_value = current_value['value'] + except KeyError: + current_value = None + new_value = '[{}, {}]'.format( + json.dumps({ + 'name': 'IntelNIC', + 'capability_type': 'pci', + 'product_id': '1111', + 'vendor_id': '8086', + 'device_type': 'type-PF' + }, sort_keys=True), + json.dumps({ + 'name': ' Cirrus Logic ', + 'capability_type': 'pci', + 'product_id': '0ff2', + 'vendor_id': '10de', + 'device_type': 'type-PCI' + }, sort_keys=True)) + + set_default = {'pci-alias': current_value} + set_alternate = {'pci-alias': new_value} + + expected_conf_section = 'pci' + expected_conf_key = 'alias' + + default_entry = {expected_conf_section: {}} + alternate_entry = {expected_conf_section: { + expected_conf_key: [ + ('{"capability_type": "pci", "device_type": "type-PF", ' + '"name": "IntelNIC", "product_id": "1111", ' + '"vendor_id": "8086"}'), + ('{"capability_type": "pci", "device_type": "type-PCI", ' + '"name": " Cirrus Logic ", "product_id": "0ff2", ' + '"vendor_id": "10de"}')]}} + + # Config file affected by juju set config change + conf_file = '/etc/nova/nova.conf' + + # Make config change, check for service restarts + logging.info( + 'Setting config on {} to {}'.format(app_name, set_alternate)) + self.restart_on_changed( + conf_file, + set_default, + set_alternate, + default_entry, + alternate_entry, + service_list) + + class CloudActions(test_utils.OpenStackBaseTest): """Test actions from actions/cloud.py.""" @@ -242,9 +315,19 @@ class CloudActions(test_utils.OpenStackBaseTest): "nova-cloud-controller as expected.") -class NovaCompute(test_utils.OpenStackBaseTest): +class NovaCompute(NovaCommonTests): """Run nova-compute specific tests.""" + def test_311_pci_alias_config_compute(self): + """Verify that the pci alias data is rendered properly. + + Change pci-alias and assert that change propagates to the correct + file and that services are restarted as a result + """ + # We are not touching the behavior of anything older than QUEENS + if self.current_release >= self.XENIAL_QUEENS: + self._test_pci_alias_config("nova-compute", ['nova-compute']) + def test_500_hugepagereport_action(self): """Test hugepagereport action.""" for unit in zaza.model.get_units('nova-compute', @@ -407,22 +490,9 @@ class NovaCloudControllerActionTest(test_utils.OpenStackBaseTest): self.assertIn(hypervisor.hypervisor_hostname, aggregate.hosts) -class NovaCloudController(test_utils.OpenStackBaseTest): +class NovaCloudController(NovaCommonTests): """Run nova-cloud-controller specific tests.""" - XENIAL_MITAKA = openstack_utils.get_os_release('xenial_mitaka') - XENIAL_OCATA = openstack_utils.get_os_release('xenial_ocata') - XENIAL_QUEENS = openstack_utils.get_os_release('xenial_queens') - BIONIC_QUEENS = openstack_utils.get_os_release('bionic_queens') - BIONIC_ROCKY = openstack_utils.get_os_release('bionic_rocky') - BIONIC_TRAIN = openstack_utils.get_os_release('bionic_train') - - @classmethod - def setUpClass(cls): - """Run class setup for running nova-cloud-controller tests.""" - super(NovaCloudController, cls).setUpClass() - cls.current_release = openstack_utils.get_os_release() - @property def services(self): """Return a list of services for the selected OpenStack release.""" @@ -518,70 +588,13 @@ class NovaCloudController(test_utils.OpenStackBaseTest): 'filter:legacy_ratelimit': { 'limits': ["( POST, '*', .*, 9999, MINUTE );"]}}) - def test_310_pci_alias_config(self): + def test_310_pci_alias_config_ncc(self): """Verify that the pci alias data is rendered properly. Change pci-alias and assert that change propagates to the correct file and that services are restarted as a result """ - logging.info('Checking pci aliases in nova config...') - - # Expected default and alternate values - current_value = zaza.model.get_application_config( - 'nova-cloud-controller')['pci-alias'] - try: - current_value = current_value['value'] - except KeyError: - current_value = None - new_value = '[{}, {}]'.format( - json.dumps({ - 'name': 'IntelNIC', - 'capability_type': 'pci', - 'product_id': '1111', - 'vendor_id': '8086', - 'device_type': 'type-PF' - }, sort_keys=True), - json.dumps({ - 'name': ' Cirrus Logic ', - 'capability_type': 'pci', - 'product_id': '0ff2', - 'vendor_id': '10de', - 'device_type': 'type-PCI' - }, sort_keys=True)) - - set_default = {'pci-alias': current_value} - set_alternate = {'pci-alias': new_value} - - expected_conf_section = 'DEFAULT' - expected_conf_key = 'pci_alias' - if self.current_release >= self.XENIAL_OCATA: - expected_conf_section = 'pci' - expected_conf_key = 'alias' - - default_entry = {expected_conf_section: {}} - alternate_entry = {expected_conf_section: { - expected_conf_key: [ - ('{"capability_type": "pci", "device_type": "type-PF", ' - '"name": "IntelNIC", "product_id": "1111", ' - '"vendor_id": "8086"}'), - ('{"capability_type": "pci", "device_type": "type-PCI", ' - '"name": " Cirrus Logic ", "product_id": "0ff2", ' - '"vendor_id": "10de"}')]}} - - # Config file affected by juju set config change - conf_file = '/etc/nova/nova.conf' - - # Make config change, check for service restarts - logging.info( - 'Setting config on nova-cloud-controller to {}'.format( - set_alternate)) - self.restart_on_changed( - conf_file, - set_default, - set_alternate, - default_entry, - alternate_entry, - self.services) + self._test_pci_alias_config("nova-cloud-controller", self.services) def test_900_restart_on_config_change(self): """Checking restart happens on config change. From 110df7f2c7b801e16da063b78c5909c6456a55c7 Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 24 Jun 2021 09:19:34 +0000 Subject: [PATCH 178/180] Add very basic ceph dashboard tests. --- .../charm_tests/ceph/dashboard/tests.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 zaza/openstack/charm_tests/ceph/dashboard/tests.py diff --git a/zaza/openstack/charm_tests/ceph/dashboard/tests.py b/zaza/openstack/charm_tests/ceph/dashboard/tests.py new file mode 100644 index 0000000..7b21ab1 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/dashboard/tests.py @@ -0,0 +1,97 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Encapsulating `ceph-dashboard` testing.""" + +import collections +import os +import requests + +import zaza +import zaza.openstack.charm_tests.test_utils as test_utils +import zaza.utilities.deployment_env as deployment_env + + +class CephDashboardTest(test_utils.BaseCharmTest): + """Class for `ceph-dashboard` tests.""" + + REMOTE_CERT_FILE = ('/usr/local/share/ca-certificates/' + 'vault_ca_cert_dashboard.crt') + + @classmethod + def setUpClass(cls): + """Run class setup for running ceph dashboard tests.""" + super().setUpClass() + cls.application_name = 'ceph-dashboard' + cls.local_ca_cert = cls.collect_ca() + + @classmethod + def collect_ca(cls): + """Collect CA from ceph-dashboard unit.""" + local_ca_cert = os.path.join( + deployment_env.get_tmpdir(), + os.path.basename(cls.REMOTE_CERT_FILE)) + if not os.path.isfile(local_ca_cert): + units = zaza.model.get_units(cls.application_name) + zaza.model.scp_from_unit( + units[0].entity_id, + cls.REMOTE_CERT_FILE, + local_ca_cert) + return local_ca_cert + + def test_dashboard_units(self): + """Check dashboard units are configured correctly.""" + # XXX: Switch to using CA for verification when + # https://bugs.launchpad.net/cloud-archive/+bug/1933410 + # is fix released. + # verify = self.local_ca_cert + verify = False + units = zaza.model.get_units(self.application_name) + rcs = collections.defaultdict(list) + for unit in units: + r = requests.get( + 'https://{}:8443'.format(unit.public_address), + verify=verify, + allow_redirects=False) + rcs[r.status_code].append(unit.public_address) + self.assertEqual(len(rcs[requests.codes.ok]), 1) + self.assertEqual(len(rcs[requests.codes.see_other]), len(units) - 1) + + def create_user(self, username, role='administrator'): + """Store the model aliases in a global. + + :param username: Username to create. + :type username: str + :param role: Role to grant to user. + :type role: str + :returns: Results from action. + :rtype: juju.action.Action + """ + action = zaza.model.run_action_on_leader( + 'ceph-dashboard', + 'add-user', + action_params={ + 'username': username, + 'role': role}) + return action + + def test_create_user(self): + """Test create user action.""" + test_user = 'marvin' + action = self.create_user(test_user) + self.assertEqual(action.status, "completed") + self.assertTrue(action.data['results']['password']) + action = self.create_user(test_user) + # Action should fail as the user already exists + self.assertEqual(action.status, "failed") From dc78fb6616d86eb8f9637f36522a3be941dd5acc Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 24 Jun 2021 09:51:06 +0000 Subject: [PATCH 179/180] Add missing __init__ --- .../charm_tests/ceph/dashboard/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 zaza/openstack/charm_tests/ceph/dashboard/__init__.py diff --git a/zaza/openstack/charm_tests/ceph/dashboard/__init__.py b/zaza/openstack/charm_tests/ceph/dashboard/__init__.py new file mode 100644 index 0000000..f34c394 --- /dev/null +++ b/zaza/openstack/charm_tests/ceph/dashboard/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Collection of code for setting up and testing ``ceph-dashboard``.""" From 7cec6a27b56041b07ba616346b64a8327d0469cd Mon Sep 17 00:00:00 2001 From: Liam Young Date: Thu, 24 Jun 2021 11:57:21 +0000 Subject: [PATCH 180/180] Fix doc string --- zaza/openstack/charm_tests/ceph/dashboard/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/ceph/dashboard/tests.py b/zaza/openstack/charm_tests/ceph/dashboard/tests.py index 7b21ab1..e7c8863 100644 --- a/zaza/openstack/charm_tests/ceph/dashboard/tests.py +++ b/zaza/openstack/charm_tests/ceph/dashboard/tests.py @@ -69,7 +69,7 @@ class CephDashboardTest(test_utils.BaseCharmTest): self.assertEqual(len(rcs[requests.codes.see_other]), len(units) - 1) def create_user(self, username, role='administrator'): - """Store the model aliases in a global. + """Create a dashboard user. :param username: Username to create. :type username: str