Merge pull request #42 from AbsaOSS/bugfix/module-register-double-loading-modules

Bugfix: Module Loader Clear before re-enabling disabled decorators

Author: MDobransky

rialto/jobs/module_register.py

@@ -64,6 +64,7 @@ def remove_module(cls, module):
         :param module: The module to be removed.
         """
         cls._storage.pop(module.__name__, None)
+        cls._dependency_tree.pop(module.__name__, None)
 
     @classmethod
     def register_dependency(cls, module, parent_name):

tests/jobs/dependency_checks_job/datasources_config.py

@@ -0,0 +1,6 @@
+from rialto.jobs import config_parser
+
+
+@config_parser
+def my_config():
+    return 42

tests/jobs/dependency_checks_job/dependency_checks_job.py

@@ -30,3 +30,8 @@ def default_dependency_job(run_date, spark, config, table_reader):
     assert spark is not None
     assert config is not None
     assert table_reader is not None
+
+
+@job
+def dependency_with_config_job(dependency_with_config):
+    assert dependency_with_config == 54  # 1 + 11 + 42

tests/jobs/dependency_checks_job/main_datasources.py

@@ -1,4 +1,7 @@
-from rialto.jobs import datasource
+import tests.jobs.dependency_checks_job.datasources_config as cfg
+from rialto.jobs import datasource, register_dependency_module
+
+register_dependency_module(cfg)
 
 
 @datasource
@@ -35,3 +38,8 @@ def circle_third(circle_first):
 @datasource
 def self_dependency(a, b, c, self_dependency):
     return a
+
+
+@datasource
+def dependency_with_config(a, b, my_config):
+    return a + b + my_config

tests/jobs/test_test_utils.py

@@ -106,3 +106,12 @@ def test_duplicate_dependency_fails_on_duplicate(spark):
 
     assert exc_info is not None
     assert str(exc_info.value) == f"Multiple functions with the same name i found !"
+
+
+def test_resolver_with_disable_decorators(spark):
+    with disable_job_decorators(dependency_checks_job):
+        # Doesn't really matter what we call here, as long as it is a job
+        dependency_checks_job.dependency_with_config_job(54)
+
+    # Now, resolver should work as normal, as decorators are re-enabled and the module is reloaded
+    assert resolver_resolves(spark, dependency_checks_job.dependency_with_config_job)