From 45146b6c45d306cb0312fc97509b5eca4c075c95 Mon Sep 17 00:00:00 2001 From: Alex Kavanagh Date: Wed, 3 Mar 2021 14:54:47 +0000 Subject: [PATCH] Add ObjectRetrier to perform retries on openstack client calls This adds a wrapper class that detects if a callable object in any of the descendent objects raises an Exception. If so, then it retries that exception. This is to attempt to make the zaza tests a little more robust in the face of small network failures or strange restarts. This is a test, and robust logging a reporting should be used to determine whether it is covering up actual bugs rather than CI system issues. Related Bug: (zot repo)#348 --- unit_tests/utilities/test_utilities.py | 166 +++++++++++++++++++++++++ zaza/openstack/utilities/__init__.py | 128 +++++++++++++++++++ 2 files changed, 294 insertions(+) create mode 100644 unit_tests/utilities/test_utilities.py diff --git a/unit_tests/utilities/test_utilities.py b/unit_tests/utilities/test_utilities.py new file mode 100644 index 0000000..e961de1 --- /dev/null +++ b/unit_tests/utilities/test_utilities.py @@ -0,0 +1,166 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import unit_tests.utils as ut_utils + +import zaza.openstack.utilities as utilities + + +class SomeException(Exception): + pass + + +class SomeException2(Exception): + pass + + +class SomeException3(Exception): + pass + + +class TestObjectRetrier(ut_utils.BaseTestCase): + + def test_object_wrap(self): + + class A: + + def func(self, a, b=1): + return a + b + + a = A() + wrapped_a = utilities.ObjectRetrier(a) + self.assertEqual(wrapped_a.func(3), 4) + + def test_object_multilevel_wrap(self): + + class A: + + def f1(self, a, b): + return a * b + + class B: + + @property + def f2(self): + + return A() + + b = B() + wrapped_b = utilities.ObjectRetrier(b) + self.assertEqual(wrapped_b.f2.f1(5, 6), 30) + + def test_object_wrap_number(self): + + class A: + + class_a = 5 + + def __init__(self): + self.instance_a = 10 + + def f1(self, a, b): + return a * b + + a = A() + wrapped_a = utilities.ObjectRetrier(a) + self.assertEqual(wrapped_a.class_a, 5) + self.assertEqual(wrapped_a.instance_a, 10) + + @mock.patch("time.sleep") + def test_object_wrap_exception(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + # retry on a specific exception + wrapped_a = utilities.ObjectRetrier(a, num_retries=1, + retry_exceptions=[SomeException]) + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_called_once_with(5) + + # also retry on any exception if none specified + wrapped_a = utilities.ObjectRetrier(a, num_retries=1) + mock_sleep.reset_mock() + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_called_once_with(5) + + # no retry if exception isn't listed. + wrapped_a = utilities.ObjectRetrier(a, num_retries=1, + retry_exceptions=[SomeException2]) + mock_sleep.reset_mock() + with self.assertRaises(SomeException): + wrapped_a.func() + + mock_sleep.assert_not_called() + + @mock.patch("time.sleep") + def test_log_called(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + mock_log = mock.Mock() + wrapped_a = utilities.ObjectRetrier(a, num_retries=1, log=mock_log) + with self.assertRaises(SomeException): + wrapped_a.func() + + # there should be two calls; one for the single retry and one for the + # failure. + self.assertEqual(mock_log.call_count, 2) + + @mock.patch("time.sleep") + def test_back_off_maximum(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + wrapped_a = utilities.ObjectRetrier(a, num_retries=3, backoff=2) + with self.assertRaises(SomeException): + wrapped_a.func() + # Note third call hits maximum wait time of 15. + mock_sleep.assert_has_calls([mock.call(5), + mock.call(10), + mock.call(15)]) + + @mock.patch("time.sleep") + def test_total_wait(self, mock_sleep): + + class A: + + def func(self): + raise SomeException() + + a = A() + wrapped_a = utilities.ObjectRetrier(a, num_retries=3, total_wait=9) + with self.assertRaises(SomeException): + wrapped_a.func() + # Note only two calls, as total wait is 9, so a 3rd retry would exceed + # that. + mock_sleep.assert_has_calls([mock.call(5), + mock.call(5)]) diff --git a/zaza/openstack/utilities/__init__.py b/zaza/openstack/utilities/__init__.py index 35b5a14..5011689 100644 --- a/zaza/openstack/utilities/__init__.py +++ b/zaza/openstack/utilities/__init__.py @@ -13,3 +13,131 @@ # limitations under the License. """Collection of utilities to support zaza tests etc.""" + + +import time + + +class ObjectRetrier(object): + """An automatic retrier for an object. + + This is designed to be used with an instance of an object. Basically, it + wraps the object and any attributes that are fetched. Essentially, it is + used to provide retries on method calls on openstack client objects in + tests to increase robustness of tests. + + Although, technically this is bad, retries can be logged with the optional + log method. + + Usage: + + # get a client that does 3 retries, waits 5 seconds between retries and + # retries on any error. + some_client = ObjectRetrier(get_some_client) + # this gets retried up to 3 times. + things = some_client.list_things() + + Note, it is quite simple. It wraps the object and on a getattr(obj, name) + it finds the name and then returns a wrapped version of that name. On a + call, it returns the value of that call. It only wraps objects in the + chain that are either callable or have a __getattr__() method. i.e. one + that can then be retried or further fetched. This means that if a.b.c() is + a chain of objects, and we just wrap 'a', then 'b' and 'c' will both be + wrapped that the 'c' object __call__() method will be the one that is + actually retried. + + Note: this means that properties that do method calls won't be retried. + This is a limitation that may be addressed in the future, if it is needed. + """ + + def __init__(self, obj, num_retries=3, initial_interval=5.0, backoff=1.0, + max_interval=15.0, total_wait=30.0, retry_exceptions=None, + log=None): + """Initialise the retrier object. + + :param obj: The object to wrap. Ought to be an instance of something + that you want to get methods on to call or be called itself. + :type obj: Any + :param num_retries: The (maximum) number of retries. May not be hit if + the total_wait time is exceeded. + :type num_retries: int + :param initial_interval: The initial or starting interval between + retries. + :type initial_interval: float + :param backoff: The exponential backoff multiple. 1 is linear. + :type backoff: float + :param max_interval: The maximum interval between retries. + If backoff is >1 then the initial_interval will never grow larger + than max_interval. + :type max_interval: float + :param retry_exceptions: The list of exceptions to retry on, or None. + If a list, then it will only retry if the exception is one of the + ones in the list. + :type retry_exceptions: List[Exception] + """ + # Note we use semi-private variable names that shouldn't clash with any + # on the actual object. + self.__obj = obj + self.__kwargs = { + 'num_retries': num_retries, + 'initial_interval': initial_interval, + 'backoff': backoff, + 'max_interval': max_interval, + 'total_wait': total_wait, + 'retry_exceptions': retry_exceptions, + 'log': log or (lambda x: None), + } + + def __getattr__(self, name): + """Get attribute; delegates to wrapped object.""" + # Note the above may generate an attribute error; we expect this and + # will fail with an attribute error. + attr = getattr(self.__obj, name) + if callable(attr) or hasattr(attr, "__getattr__"): + return ObjectRetrier(attr, **self.__kwargs) + else: + return attr + # TODO(ajkavanagh): Note detecting a property is a bit trickier. we + # can do isinstance(attr, property), but then the act of accessing it + # is what calls it. i.e. it would fail at the getattr(self.__obj, + # name) stage. The solution is to check first, and if it's a property, + # then treat it like the retrier. However, I think this is too + # complex for the first go, and to use manual retries in that instance. + + def __call__(self, *args, **kwargs): + """Call the object; delegates to the wrapped object.""" + obj = self.__obj + retry = 0 + wait = self.__kwargs['initial_interval'] + max_interval = self.__kwargs['max_interval'] + log = self.__kwargs['log'] + backoff = self.__kwargs['backoff'] + total_wait = self.__kwargs['total_wait'] + num_retries = self.__kwargs['num_retries'] + retry_exceptions = self.__kwargs['retry_exceptions'] + wait_so_far = 0 + while True: + try: + return obj(*args, **kwargs) + except Exception as e: + # if retry_exceptions is None, or the type of the exception is + # not in the list of retries, then raise an exception + # immediately. This means that if retry_exceptions is None, + # then the method is always retried. + if (retry_exceptions is not None and + type(e) not in retry_exceptions): + raise + retry += 1 + if retry > num_retries: + log("{}: exceeded number of retries, so erroring out" + .format(str(obj))) + raise e + log("{}: call failed: retrying in {} seconds" + .format(str(obj), wait)) + time.sleep(wait) + wait_so_far += wait + if wait_so_far >= total_wait: + raise e + wait = wait * backoff + if wait > max_interval: + wait = max_interval