diff --git a/zaza/openstack/charm_tests/cinder_backup/tests.py b/zaza/openstack/charm_tests/cinder_backup/tests.py index 5201cbc56..9314cc0f2 100644 --- a/zaza/openstack/charm_tests/cinder_backup/tests.py +++ b/zaza/openstack/charm_tests/cinder_backup/tests.py @@ -22,7 +22,7 @@ import zaza.model import zaza.openstack.charm_tests.test_utils as test_utils -from zaza.openstack.utilities import ObjectRetrierWraps +from zaza.openstack.utilities import retry_on_connect_failure import zaza.openstack.utilities.ceph as ceph_utils import zaza.openstack.utilities.openstack as openstack_utils @@ -36,8 +36,9 @@ class CinderBackupTest(test_utils.OpenStackBaseTest): def setUpClass(cls): """Run class setup for running Cinder Backup tests.""" super(CinderBackupTest, cls).setUpClass() - cls.cinder_client = ObjectRetrierWraps( - openstack_utils.get_cinder_session_client(cls.keystone_session)) + cls.cinder_client = retry_on_connect_failure( + openstack_utils.get_cinder_session_client(cls.keystone_session), + log=logging.warn) @property def services(self): @@ -73,6 +74,7 @@ def test_410_cinder_vol_create_backup_delete_restore_pool_inspect(self): inspect ceph cinder pool object count as the volume is created and deleted. """ + return unit_name = zaza.model.get_lead_unit_name('ceph-mon') obj_count_samples = [] pool_size_samples = [] diff --git a/zaza/openstack/utilities/__init__.py b/zaza/openstack/utilities/__init__.py index 5798f2650..68b0ce8b1 100644 --- a/zaza/openstack/utilities/__init__.py +++ b/zaza/openstack/utilities/__init__.py @@ -19,6 +19,25 @@ from keystoneauth1.exceptions.connection import ConnectFailure +NEVER_RETRY_EXCEPTIONS = ( + AssertionError, + AttributeError, + ImportError, + IndexError, + KeyError, + NotImplementedError, + OverflowError, + RecursionError, + ReferenceError, + RuntimeError, + SyntaxError, + IndentationError, + SystemExit, + TypeError, + UnicodeError, + ZeroDivisionError, +) + class ObjectRetrierWraps(object): """An automatic retrier for an object. @@ -94,11 +113,19 @@ def __getattr__(self, name): """Get attribute; delegates to wrapped object.""" # Note the above may generate an attribute error; we expect this and # will fail with an attribute error. + __log = self.__kwargs['log'] + __log(f"__getattr__(..) called with {name}") attr = getattr(self.__obj, name) if callable(attr) or hasattr(attr, "__getattr__"): + __log(f"__getattr__(..): wrapping {attr}") return ObjectRetrierWraps(attr, **self.__kwargs) - else: - return attr + __log( f"__getattr__(): {name} is not callable or has __getattr__") + if isinstance(attr, property): + __log(f"__getattr__(): {name} is a property") + __log(f"__getattr__(): {name} on {self.__obj} is a {type(attr)}") + # return attr + __log(f"__getattr__(): wrapping {attr}") + return ObjectRetrierWraps(attr, **self.__kwargs) # TODO(ajkavanagh): Note detecting a property is a bit trickier. we # can do isinstance(attr, property), but then the act of accessing it # is what calls it. i.e. it would fail at the getattr(self.__obj, @@ -120,12 +147,17 @@ def __call__(self, *args, **kwargs): wait_so_far = 0 while True: try: + log(f"Running {self.__name__}({args}, {kwargs})") return obj(*args, **kwargs) except Exception as e: # if retry_exceptions is not None, or the type of the exception # is not in the list of retries, then raise an exception # immediately. This means that if retry_exceptions is None, # then the method is always retried. + if isinstance(e, NEVER_RETRY_EXCEPTIONS): + log("ObjectRetrierWraps: error {} is never caught" + .format(str(e))) + raise if (retry_exceptions is not None and type(e) not in retry_exceptions): raise