Fix full test errors (#255)

* debugging

* working fast tests

* passing tests

* pin lifelines<0.28 as 0.28 does not support python 3.8

* debugging lifelines files error

* lifelines==0.27.7

* fix version pin

* revert to strict pin

* lifelines version constraints as generic as possible

* split core tests into fast and slow and increase timeout

* split slow tests into two

* update version

Author: robsdavis

.github/workflows/test_full.yml

@@ -30,10 +30,21 @@ jobs:
         run: |
           python -m pip install -U pip
           pip install -r prereq.txt
-      - name: Test Core
+      - name: Test Core - slow part one
+        timeout-minutes: 1000
         run: |
           pip install .[testing]
-          pytest -vvvs --durations=50
+          pytest -vvvs --durations=50 -m "slow_1"
+      - name: Test Core - slow part two
+        timeout-minutes: 1000
+        run: |
+          pip install .[testing]
+          pytest -vvvs --durations=50 -m "slow_2"
+      - name: Test Core - fast
+        timeout-minutes: 1000
+        run: |
+          pip install .[testing]
+          pytest -vvvs --durations=50 -m "not slow"
       - name: Test GOGGLE
         run: |
           pip install .[testing,goggle]

setup.cfg

@@ -38,7 +38,7 @@ install_requires =
     scikit-learn>=1.2
     nflows>=0.14
     numpy>=1.20, <1.24
-    lifelines>=0.27,!= 0.27.5
+    lifelines>=0.27,!= 0.27.5, <0.27.8
     opacus>=1.3
     decaf-synthetic-data>=0.1.6
     optuna>=3.1
@@ -117,6 +117,8 @@ testpaths = tests
 # Use pytest markers to select/deselect specific tests
 markers =
     slow: mark tests as slow (deselect with '-m "not slow"')
+    slow_1: mark tests as slow (deselect with '-m "not slow_1"')
+    slow_2: mark tests as slow (deselect with '-m "not slow_1"')
 
 [devpi:upload]
 # Options for the devpi: PyPI server and packaging tool

src/synthcity/plugins/core/dataloader.py

@@ -928,12 +928,13 @@ def unpack(self, as_numpy: bool = False, pad: bool = False) -> Any:
                 self.data["observation_times"],
                 self.data["outcome"],
             )
+
         if as_numpy:
             longest_observation_seq = max([len(seq) for seq in temporal_data])
             return (
                 np.asarray(static_data),
                 np.asarray(
-                    pd.concat(temporal_data)
+                    temporal_data
                 ),  # TODO: check this works with time series benchmarks
                 # masked array to handle variable length sequences
                 ma.vstack(

src/synthcity/plugins/core/plugin.py

@@ -560,7 +560,6 @@ class PluginLoader:
 
     @validate_arguments
     def __init__(self, plugins: list, expected_type: Type, categories: list) -> None:
-        # self.reload()
         global PLUGIN_CATEGORY_REGISTRY
         PLUGIN_CATEGORY_REGISTRY = {cat: [] for cat in categories}
         self._refresh()
@@ -639,7 +638,6 @@ def list(self) -> List[str]:
         for plugin in all_plugins:
             if self.get_type(plugin).type() in self._categories:
                 plugins.append(plugin)
-
         return list(set(plugins))
 
     def types(self) -> List[Type]:

src/synthcity/plugins/privacy/plugin_dpgan.py

@@ -101,6 +101,8 @@ class DPGANPlugin(Plugin):
         >>>
         >>> plugin.generate(50)
 
+    Note: There is a known issue with the training step for training GANs with conditionals with dp_enabled set to True, as is the case for DPGAN.
+
     """
 
     @validate_arguments(config=dict(arbitrary_types_allowed=True))

src/synthcity/version.py

@@ -1,4 +1,4 @@
-__version__ = "0.2.9"
+__version__ = "0.2.10"
 
 MAJOR_VERSION = ".".join(__version__.split(".")[:-1])
 PATCH_VERSION = __version__.split(".")[-1]

tests/metrics/test_detection.py

@@ -154,6 +154,7 @@ def test_detect_synth_timeseries(test_plugin: Plugin, evaluator_t: Type) -> None
     assert evaluator.direction() == "minimize"
 
 
+@pytest.mark.slow_1
 @pytest.mark.slow
 def test_image_support_detection() -> None:
     dataset = datasets.MNIST(".", download=True)

tests/metrics/test_performance.py

@@ -94,6 +94,7 @@ def test_evaluate_performance_classifier(
 @pytest.mark.xfail
 @pytest.mark.skipif(sys.platform != "linux", reason="Linux only for faster results")
 @pytest.mark.skipif(sys.version_info < (3, 9), reason="requires python3.9 or higher")
+@pytest.mark.slow_1
 @pytest.mark.slow
 def test_evaluate_feature_importance_rank_dist_clf(
     distance: str, test_plugin: Plugin
@@ -183,6 +184,7 @@ def test_evaluate_performance_regression(
 @pytest.mark.xfail
 @pytest.mark.skipif(sys.platform != "linux", reason="Linux only for faster results")
 @pytest.mark.skipif(sys.version_info < (3, 9), reason="requires python3.9 or higher")
+@pytest.mark.slow_1
 @pytest.mark.slow
 def test_evaluate_feature_importance_rank_dist_reg(
     distance: str, test_plugin: Plugin
@@ -211,6 +213,7 @@ def test_evaluate_feature_importance_rank_dist_reg(
     assert score["pvalue"] > 0
 
 
+@pytest.mark.slow_1
 @pytest.mark.slow
 @pytest.mark.parametrize("test_plugin", [Plugins().get("marginal_distributions")])
 @pytest.mark.parametrize(
@@ -296,6 +299,7 @@ def test_evaluate_performance_survival_analysis(
 @pytest.mark.xfail
 @pytest.mark.skipif(sys.platform != "linux", reason="Linux only for faster results")
 @pytest.mark.skipif(sys.version_info < (3, 9), reason="requires python3.9 or higher")
+@pytest.mark.slow_1
 @pytest.mark.slow
 def test_evaluate_feature_importance_rank_dist_surv(
     distance: str, test_plugin: Plugin
@@ -362,6 +366,7 @@ def test_evaluate_performance_custom_labels(
     assert "syn_ood" in good_score
 
 
+@pytest.mark.slow_1
 @pytest.mark.slow
 @pytest.mark.parametrize("test_plugin", [Plugins().get("timegan")])
 @pytest.mark.parametrize(
@@ -472,6 +477,7 @@ def test_evaluate_performance_time_series_survival(
     assert def_score == good_score["syn_id.c_index"] - good_score["syn_id.brier_score"]
 
 
+@pytest.mark.slow_1
 @pytest.mark.slow
 def test_image_support_perf() -> None:
     dataset = datasets.MNIST(".", download=True)

tests/plugins/core/models/test_tabular_gan.py

@@ -174,6 +174,7 @@ def test_gan_generation_with_early_stopping(patience_metric: Tuple[str, str]) ->
     assert generated.shape == (10, X.shape[1])
 
 
+@pytest.mark.slow_1
 @pytest.mark.slow
 def test_gan_sampling_adjustment() -> None:
     X = get_airfoil_dataset()

tests/plugins/core/models/test_ts_gan.py

@@ -129,6 +129,7 @@ def test_ts_gan_generation(source: Any) -> None:
     assert observation_times_gen.shape == (10, temporal.shape[1])
 
 
+@pytest.mark.slow_1
 @pytest.mark.slow
 @pytest.mark.parametrize("source", [GoogleStocksDataloader])
 def test_ts_gan_generation_schema(source: Any) -> None: