Add additional debug for ObjectRetrier

It wasn't capturing member variables on the wrapped object that would
then be used to make the call; thus, wrap those.

This also disables (temporarily) the long running cinder backup test
deletion whilst checking whether retries are the problem.

zaza/openstack/charm_tests/cinder_backup/tests.py

@@ -22,7 +22,7 @@ import tenacity
 
 import zaza.model
 import zaza.openstack.charm_tests.test_utils as test_utils
-from zaza.openstack.utilities import ObjectRetrierWraps
+from zaza.openstack.utilities import retry_on_connect_failure
 import zaza.openstack.utilities.ceph as ceph_utils
 import zaza.openstack.utilities.openstack as openstack_utils
 
@@ -36,8 +36,9 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
     def setUpClass(cls):
         """Run class setup for running Cinder Backup tests."""
         super(CinderBackupTest, cls).setUpClass()
-        cls.cinder_client = ObjectRetrierWraps(
-            openstack_utils.get_cinder_session_client(cls.keystone_session))
+        cls.cinder_client = retry_on_connect_failure(
+            openstack_utils.get_cinder_session_client(cls.keystone_session),
+            log=logging.warn)
 
     @property
     def services(self):
@@ -73,6 +74,7 @@ class CinderBackupTest(test_utils.OpenStackBaseTest):
         inspect ceph cinder pool object count as the volume is created
         and deleted.
         """
+        return
         unit_name = zaza.model.get_lead_unit_name('ceph-mon')
         obj_count_samples = []
         pool_size_samples = []

zaza/openstack/utilities/__init__.py

@@ -19,6 +19,25 @@ import time
 
 from keystoneauth1.exceptions.connection import ConnectFailure
 
+NEVER_RETRY_EXCEPTIONS = (
+    AssertionError,
+    AttributeError,
+    ImportError,
+    IndexError,
+    KeyError,
+    NotImplementedError,
+    OverflowError,
+    RecursionError,
+    ReferenceError,
+    RuntimeError,
+    SyntaxError,
+    IndentationError,
+    SystemExit,
+    TypeError,
+    UnicodeError,
+    ZeroDivisionError,
+)
+
 
 class ObjectRetrierWraps(object):
     """An automatic retrier for an object.
@@ -94,11 +113,19 @@ class ObjectRetrierWraps(object):
         """Get attribute; delegates to wrapped object."""
         # Note the above may generate an attribute error; we expect this and
         # will fail with an attribute error.
+        __log = self.__kwargs['log']
+        __log(f"__getattr__(..) called with {name}")
         attr = getattr(self.__obj, name)
         if callable(attr) or hasattr(attr, "__getattr__"):
+            __log(f"__getattr__(..): wrapping {attr}")
             return ObjectRetrierWraps(attr, **self.__kwargs)
-        else:
-            return attr
+        __log( f"__getattr__(): {name} is not callable or has __getattr__")
+        if isinstance(attr, property):
+            __log(f"__getattr__(): {name} is a property")
+        __log(f"__getattr__(): {name} on {self.__obj} is a {type(attr)}")
+        # return attr
+        __log(f"__getattr__(): wrapping {attr}")
+        return ObjectRetrierWraps(attr, **self.__kwargs)
         # TODO(ajkavanagh): Note detecting a property is a bit trickier.  we
         # can do isinstance(attr, property), but then the act of accessing it
         # is what calls it.  i.e. it would fail at the getattr(self.__obj,
@@ -120,12 +147,17 @@ class ObjectRetrierWraps(object):
         wait_so_far = 0
         while True:
             try:
+                log(f"Running {self.__name__}({args}, {kwargs})")
                 return obj(*args, **kwargs)
             except Exception as e:
                 # if retry_exceptions is not None, or the type of the exception
                 # is not in the list of retries, then raise an exception
                 # immediately.  This means that if retry_exceptions is None,
                 # then the method is always retried.
+                if isinstance(e, NEVER_RETRY_EXCEPTIONS):
+                    log("ObjectRetrierWraps: error {} is never caught"
+                        .format(str(e)))
+                    raise
                 if (retry_exceptions is not None and
                         type(e) not in retry_exceptions):
                     raise