Alex Kavanagh
45146b6c45
This adds a wrapper class that detects if a callable object in any of the descendent objects raises an Exception. If so, then it retries that exception. This is to attempt to make the zaza tests a little more robust in the face of small network failures or strange restarts. This is a test, and robust logging a reporting should be used to determine whether it is covering up actual bugs rather than CI system issues. Related Bug: (zot repo)#348
144 lines
6.1 KiB
Python
144 lines
6.1 KiB
Python
# Copyright 2018 Canonical Ltd.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Collection of utilities to support zaza tests etc."""
|
|
|
|
|
|
import time
|
|
|
|
|
|
class ObjectRetrier(object):
|
|
"""An automatic retrier for an object.
|
|
|
|
This is designed to be used with an instance of an object. Basically, it
|
|
wraps the object and any attributes that are fetched. Essentially, it is
|
|
used to provide retries on method calls on openstack client objects in
|
|
tests to increase robustness of tests.
|
|
|
|
Although, technically this is bad, retries can be logged with the optional
|
|
log method.
|
|
|
|
Usage:
|
|
|
|
# get a client that does 3 retries, waits 5 seconds between retries and
|
|
# retries on any error.
|
|
some_client = ObjectRetrier(get_some_client)
|
|
# this gets retried up to 3 times.
|
|
things = some_client.list_things()
|
|
|
|
Note, it is quite simple. It wraps the object and on a getattr(obj, name)
|
|
it finds the name and then returns a wrapped version of that name. On a
|
|
call, it returns the value of that call. It only wraps objects in the
|
|
chain that are either callable or have a __getattr__() method. i.e. one
|
|
that can then be retried or further fetched. This means that if a.b.c() is
|
|
a chain of objects, and we just wrap 'a', then 'b' and 'c' will both be
|
|
wrapped that the 'c' object __call__() method will be the one that is
|
|
actually retried.
|
|
|
|
Note: this means that properties that do method calls won't be retried.
|
|
This is a limitation that may be addressed in the future, if it is needed.
|
|
"""
|
|
|
|
def __init__(self, obj, num_retries=3, initial_interval=5.0, backoff=1.0,
|
|
max_interval=15.0, total_wait=30.0, retry_exceptions=None,
|
|
log=None):
|
|
"""Initialise the retrier object.
|
|
|
|
:param obj: The object to wrap. Ought to be an instance of something
|
|
that you want to get methods on to call or be called itself.
|
|
:type obj: Any
|
|
:param num_retries: The (maximum) number of retries. May not be hit if
|
|
the total_wait time is exceeded.
|
|
:type num_retries: int
|
|
:param initial_interval: The initial or starting interval between
|
|
retries.
|
|
:type initial_interval: float
|
|
:param backoff: The exponential backoff multiple. 1 is linear.
|
|
:type backoff: float
|
|
:param max_interval: The maximum interval between retries.
|
|
If backoff is >1 then the initial_interval will never grow larger
|
|
than max_interval.
|
|
:type max_interval: float
|
|
:param retry_exceptions: The list of exceptions to retry on, or None.
|
|
If a list, then it will only retry if the exception is one of the
|
|
ones in the list.
|
|
:type retry_exceptions: List[Exception]
|
|
"""
|
|
# Note we use semi-private variable names that shouldn't clash with any
|
|
# on the actual object.
|
|
self.__obj = obj
|
|
self.__kwargs = {
|
|
'num_retries': num_retries,
|
|
'initial_interval': initial_interval,
|
|
'backoff': backoff,
|
|
'max_interval': max_interval,
|
|
'total_wait': total_wait,
|
|
'retry_exceptions': retry_exceptions,
|
|
'log': log or (lambda x: None),
|
|
}
|
|
|
|
def __getattr__(self, name):
|
|
"""Get attribute; delegates to wrapped object."""
|
|
# Note the above may generate an attribute error; we expect this and
|
|
# will fail with an attribute error.
|
|
attr = getattr(self.__obj, name)
|
|
if callable(attr) or hasattr(attr, "__getattr__"):
|
|
return ObjectRetrier(attr, **self.__kwargs)
|
|
else:
|
|
return attr
|
|
# TODO(ajkavanagh): Note detecting a property is a bit trickier. we
|
|
# can do isinstance(attr, property), but then the act of accessing it
|
|
# is what calls it. i.e. it would fail at the getattr(self.__obj,
|
|
# name) stage. The solution is to check first, and if it's a property,
|
|
# then treat it like the retrier. However, I think this is too
|
|
# complex for the first go, and to use manual retries in that instance.
|
|
|
|
def __call__(self, *args, **kwargs):
|
|
"""Call the object; delegates to the wrapped object."""
|
|
obj = self.__obj
|
|
retry = 0
|
|
wait = self.__kwargs['initial_interval']
|
|
max_interval = self.__kwargs['max_interval']
|
|
log = self.__kwargs['log']
|
|
backoff = self.__kwargs['backoff']
|
|
total_wait = self.__kwargs['total_wait']
|
|
num_retries = self.__kwargs['num_retries']
|
|
retry_exceptions = self.__kwargs['retry_exceptions']
|
|
wait_so_far = 0
|
|
while True:
|
|
try:
|
|
return obj(*args, **kwargs)
|
|
except Exception as e:
|
|
# if retry_exceptions is None, or the type of the exception is
|
|
# not in the list of retries, then raise an exception
|
|
# immediately. This means that if retry_exceptions is None,
|
|
# then the method is always retried.
|
|
if (retry_exceptions is not None and
|
|
type(e) not in retry_exceptions):
|
|
raise
|
|
retry += 1
|
|
if retry > num_retries:
|
|
log("{}: exceeded number of retries, so erroring out"
|
|
.format(str(obj)))
|
|
raise e
|
|
log("{}: call failed: retrying in {} seconds"
|
|
.format(str(obj), wait))
|
|
time.sleep(wait)
|
|
wait_so_far += wait
|
|
if wait_so_far >= total_wait:
|
|
raise e
|
|
wait = wait * backoff
|
|
if wait > max_interval:
|
|
wait = max_interval
|