From 49268243ba11b8cb8c1510ffb2eea28943f5cb3b Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 22 Sep 2020 16:18:09 +0200 Subject: [PATCH 1/6] Improve logging --- zaza/openstack/charm_tests/cinder_backup/tests.py | 8 ++++---- zaza/openstack/utilities/openstack.py | 13 ++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 7d3442e..1cdd887 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -63,7 +63,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): self.cinder_client.volumes, vol_new.id, expected_status="available", - msg="Volume status wait") + msg="Extended volume") def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self): """Create, backup, delete, restore a ceph-backed cinder volume. @@ -99,7 +99,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): wait_iteration_max_time=180, stop_after_attempt=30, expected_status='available', - msg='Volume status wait') + msg='ceph-backed cinder volume') # Backup the volume vol_backup = self.cinder_client.backups.create( @@ -111,7 +111,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): wait_iteration_max_time=180, stop_after_attempt=30, expected_status='available', - msg='Volume status wait') + msg='Backup volume') # Delete the volume openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) # Restore the volume @@ -122,7 +122,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): wait_iteration_max_time=180, stop_after_attempt=15, expected_status='available', - msg='Backup status wait') + msg='Restored backup volume') # Delete the backup openstack_utils.delete_volume_backup( self.cinder_client, diff --git a/zaza/openstack/utilities/openstack.py b/zaza/openstack/utilities/openstack.py index 35972c6..35e188d 100644 --- a/zaza/openstack/utilities/openstack.py +++ b/zaza/openstack/utilities/openstack.py @@ -1896,14 +1896,13 @@ def _resource_reaches_status(resource, resource_id, :param expected_status: status to expect resource to reach :type expected_status: str :param msg: text to identify purpose in logging - :type msy: str + :type msg: str :raises: AssertionError """ resource_status = resource.get(resource_id).status - logging.info(resource_status) - assert resource_status == expected_status, ( - "Resource in {} state, waiting for {}" .format(resource_status, - expected_status,)) + logging.info("{}: resource {} in {} state, waiting for {}".format( + msg, resource_id, resource_status, expected_status)) + assert resource_status == expected_status def resource_reaches_status(resource, @@ -1965,8 +1964,8 @@ def _resource_removed(resource, resource_id, msg="resource"): :raises: AssertionError """ matching = [r for r in resource.list() if r.id == resource_id] - logging.debug("Resource {} still present".format(resource_id)) - assert len(matching) == 0, "Resource {} still present".format(resource_id) + logging.debug("{}: resource {} still present".format(msg, resource_id)) + assert len(matching) == 0 def resource_removed(resource, From a71a0f38ff4a8491c1c8fcd94f52eec2cedfa230 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Fri, 25 Sep 2020 14:57:18 +0200 Subject: [PATCH 2/6] Add some tenacity around cinder backup creation --- .../charm_tests/cinder_backup/tests.py | 63 +++++++++++-------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 1cdd887..1acbf9e 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -89,29 +89,43 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): self.assertEqual(pool_name, expected_pool) - # Create ceph-backed cinder volume - cinder_vol = self.cinder_client.volumes.create( - name='{}-410-vol'.format(self.RESOURCE_PREFIX), - size=1) - openstack_utils.resource_reaches_status( - self.cinder_client.volumes, - cinder_vol.id, - wait_iteration_max_time=180, - stop_after_attempt=30, - expected_status='available', - msg='ceph-backed cinder volume') + for attempt in tenacity.Retrying( + stop=tenacity.stop_after_attempt(3)): + with attempt: + # Create ceph-backed cinder volume + cinder_vol_name = '{}-410-vol-{}'.format( + self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) + cinder_vol = self.cinder_client.volumes.create( + name=cinder_vol_name, size=1) + openstack_utils.resource_reaches_status( + self.cinder_client.volumes, + cinder_vol.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='ceph-backed cinder volume') + + # Back up the volume + # NOTE(lourot): sometimes, especially on Mitaka, the backup + # remains stuck forever in 'creating' state and the volume in + # 'backing-up' state. See lp:1877076 + # Attempting to create another volume and another backup + # usually then succeeds. Release notes and bug trackers show + # that many things have been fixed and are still left to be + # fixed in this area. + # When the backup creation succeeds, it usually does within + # 12 minutes. + vol_backup_name = cinder_vol_name + '-backup' + vol_backup = self.cinder_client.backups.create( + cinder_vol.id, vol_backup_name) + openstack_utils.resource_reaches_status( + self.cinder_client.backups, + vol_backup.id, + wait_iteration_max_time=180, + stop_after_attempt=15, + expected_status='available', + msg='Backup volume') - # Backup the volume - vol_backup = self.cinder_client.backups.create( - cinder_vol.id, - name='{}-410-backup-vol'.format(self.RESOURCE_PREFIX)) - openstack_utils.resource_reaches_status( - self.cinder_client.backups, - vol_backup.id, - wait_iteration_max_time=180, - stop_after_attempt=30, - expected_status='available', - msg='Backup volume') # Delete the volume openstack_utils.delete_volume(self.cinder_client, cinder_vol.id) # Restore the volume @@ -143,12 +157,11 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): obj_count_samples.append(obj_count) pool_size_samples.append(kb_used) - name = '{}-410-vol'.format(self.RESOURCE_PREFIX) vols = self.cinder_client.volumes.list() try: - cinder_vols = [v for v in vols if v.name == name] + cinder_vols = [v for v in vols if v.name == cinder_vol_name] except AttributeError: - cinder_vols = [v for v in vols if v.display_name == name] + cinder_vols = [v for v in vols if v.display_name == cinder_vol_name] if not cinder_vols: # NOTE(hopem): it appears that at some point cinder-backup stopped # restoring volume metadata properly so revert to default name if From f9d28f16b421699a478554f74b6b3a5895864610 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Mon, 28 Sep 2020 11:03:33 +0200 Subject: [PATCH 3/6] Make linter happy --- zaza/openstack/charm_tests/cinder_backup/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 1acbf9e..2281cc1 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -161,7 +161,8 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): try: cinder_vols = [v for v in vols if v.name == cinder_vol_name] except AttributeError: - cinder_vols = [v for v in vols if v.display_name == cinder_vol_name] + cinder_vols = [v for v in vols if + v.display_name == cinder_vol_name] if not cinder_vols: # NOTE(hopem): it appears that at some point cinder-backup stopped # restoring volume metadata properly so revert to default name if From 82c1a5321b0cf6498b161059f4d1f8d5b8e719fe Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Tue, 10 Nov 2020 15:34:32 +0100 Subject: [PATCH 4/6] Fix volume and backup name --- zaza/openstack/charm_tests/cinder_backup/tests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 2281cc1..41da8ab 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -93,7 +93,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): stop=tenacity.stop_after_attempt(3)): with attempt: # Create ceph-backed cinder volume - cinder_vol_name = '{}-410-vol-{}'.format( + cinder_vol_name = '{}-410{}-vol'.format( self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) cinder_vol = self.cinder_client.volumes.create( name=cinder_vol_name, size=1) @@ -115,7 +115,12 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): # fixed in this area. # When the backup creation succeeds, it usually does within # 12 minutes. - vol_backup_name = cinder_vol_name + '-backup' + # NOTE(lourot): it seems like we have to stick to the + # `-backup-vol` naming convention otherwise + # cinder-backup fails with a ceph/rados error about not being + # able to find the pool. See lp:1897587 + vol_backup_name = '{}-410{}-backup-vol'.format( + self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) vol_backup = self.cinder_client.backups.create( cinder_vol.id, vol_backup_name) openstack_utils.resource_reaches_status( From ca40adc07878543896ef369f28b5eafe8a3a6f83 Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 11 Nov 2020 08:44:51 +0100 Subject: [PATCH 5/6] Fix comment --- zaza/openstack/charm_tests/cinder_backup/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 41da8ab..c16e207 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -116,7 +116,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): # When the backup creation succeeds, it usually does within # 12 minutes. # NOTE(lourot): it seems like we have to stick to the - # `-backup-vol` naming convention otherwise + # `--backup-vol` naming convention otherwise # cinder-backup fails with a ceph/rados error about not being # able to find the pool. See lp:1897587 vol_backup_name = '{}-410{}-backup-vol'.format( From 64fd8026975837282f335790ea1f8af5ac7882df Mon Sep 17 00:00:00 2001 From: Aurelien Lourot Date: Wed, 11 Nov 2020 14:55:09 +0100 Subject: [PATCH 6/6] Fix call to cinder_client.backups.create() --- zaza/openstack/charm_tests/cinder_backup/tests.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index c16e207..97b3658 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -93,7 +93,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): stop=tenacity.stop_after_attempt(3)): with attempt: # Create ceph-backed cinder volume - cinder_vol_name = '{}-410{}-vol'.format( + cinder_vol_name = '{}-410-{}-vol'.format( self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) cinder_vol = self.cinder_client.volumes.create( name=cinder_vol_name, size=1) @@ -115,14 +115,10 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): # fixed in this area. # When the backup creation succeeds, it usually does within # 12 minutes. - # NOTE(lourot): it seems like we have to stick to the - # `--backup-vol` naming convention otherwise - # cinder-backup fails with a ceph/rados error about not being - # able to find the pool. See lp:1897587 - vol_backup_name = '{}-410{}-backup-vol'.format( + vol_backup_name = '{}-410-{}-backup-vol'.format( self.RESOURCE_PREFIX, attempt.retry_state.attempt_number) vol_backup = self.cinder_client.backups.create( - cinder_vol.id, vol_backup_name) + cinder_vol.id, name=vol_backup_name) openstack_utils.resource_reaches_status( self.cinder_client.backups, vol_backup.id,