Specialize benchmark creation helpers

avalanche/benchmarks/classic/clear.py

@@ -24,8 +24,7 @@
 We support both evaluation protocols for benchmark construction."""
 
 from pathlib import Path
-from typing import List, Sequence, Union, Any, Optional
-from typing_extensions import Literal
+from typing import Sequence, Union, Any, Optional
 
 from avalanche.benchmarks.datasets.clear import (
     _CLEARImage,
@@ -34,9 +33,12 @@
     CLEAR_FEATURE_TYPES,
     _CLEAR_DATA_SPLITS,
 )
-from avalanche.benchmarks.scenarios.generic_benchmark_creation import (
-    create_generic_benchmark_from_paths,
-    create_generic_benchmark_from_tensor_lists,
+from avalanche.benchmarks.scenarios.classification_benchmark_creation import (
+    create_classification_benchmark_from_paths,
+    create_classification_benchmark_from_tensor_lists,
+)
+from avalanche.benchmarks.scenarios.classification_scenario import (
+    CommonClassificationScenarioType,
 )
 
 EVALUATION_PROTOCOLS = ["iid", "streaming"]
@@ -108,7 +110,7 @@ def CLEAR(
         Defaults to None, which means that the default location for
         str(data_name) will be used.
 
-    :returns: a properly initialized :class:`GenericCLScenario` instance.
+    :returns: a properly initialized :class:`ClassificationScenario` instance.
     """
     assert data_name in _CLEAR_DATA_SPLITS
 
@@ -130,6 +132,7 @@ def CLEAR(
     else:
         raise NotImplementedError()
 
+    benchmark_obj: CommonClassificationScenarioType
     if feature_type is None:
         clear_dataset_train = _CLEARImage(
             root=dataset_root,
@@ -153,7 +156,7 @@ def CLEAR(
         test_samples_paths = clear_dataset_test.get_paths_and_targets(
             root_appended=True
         )
-        benchmark_obj = create_generic_benchmark_from_paths(
+        benchmark_obj = create_classification_benchmark_from_paths(
             train_samples_paths,
             test_samples_paths,
             task_labels=list(range(len(train_samples_paths))),
@@ -181,7 +184,7 @@ def CLEAR(
         train_samples = clear_dataset_train.tensors_and_targets
         test_samples = clear_dataset_test.tensors_and_targets
 
-        benchmark_obj = create_generic_benchmark_from_tensor_lists(
+        benchmark_obj = create_classification_benchmark_from_tensor_lists(
             train_samples,
             test_samples,
             task_labels=list(range(len(train_samples))),

avalanche/benchmarks/classic/core50.py

@@ -10,8 +10,8 @@
 ################################################################################
 
 """ This module contains the high-level CORe50 benchmark generator. It
-basically returns a iterable benchmark object ``GenericCLScenario`` given a
-number of configuration parameters."""
+basically returns a iterable benchmark object :class:`ClassificationScenario`
+given a number of configuration parameters."""
 from pathlib import Path
 from typing import Union, Optional, Any
 
@@ -26,10 +26,13 @@
     check_vision_benchmark,
 )
 from avalanche.benchmarks.datasets import default_dataset_location
-from avalanche.benchmarks.scenarios.generic_benchmark_creation import (
-    create_generic_benchmark_from_filelists,
+from avalanche.benchmarks.scenarios.classification_benchmark_creation import (
+    create_classification_benchmark_from_filelists,
 )
 from avalanche.benchmarks.datasets.core50.core50 import CORe50Dataset
+from avalanche.benchmarks.scenarios.classification_scenario import (
+    CommonClassificationScenarioType,
+)
 
 nbatch = {
     "ni": 8,
@@ -109,7 +112,7 @@ def CORe50(
         location for
         'core50' will be used.
 
-    :returns: a properly initialized :class:`GenericCLScenario` instance.
+    :returns: a properly initialized :class:`ClassificationScenario` instance.
     """
 
     assert 0 <= run <= 9, (
@@ -149,15 +152,29 @@ def CORe50(
             / ("train_batch_" + str(batch_id).zfill(2) + "_filelist.txt")
         )
 
-    benchmark_obj = create_generic_benchmark_from_filelists(
-        root_img,
-        train_failists_paths,
-        [root / filelists_bp / "test_filelist.txt"],
-        task_labels=[0 for _ in range(nbatch[scenario])],
-        complete_test_set_only=True,
-        train_transform=train_transform,
-        eval_transform=eval_transform,
-    )
+    benchmark_obj: CommonClassificationScenarioType = \
+        create_classification_benchmark_from_filelists(
+            root_img,
+            train_failists_paths,
+            [root / filelists_bp / "test_filelist.txt"],
+            task_labels=[0 for _ in range(nbatch[scenario])],
+            complete_test_set_only=True,
+            train_transform=train_transform,
+            eval_transform=eval_transform,
+        )
+
+    if scenario == 'nc':
+        n_classes_per_exp = []
+        classes_order = []
+        for exp in benchmark_obj.train_stream:
+            exp_dataset = exp.dataset
+            unique_targets = list(sorted(
+                set(int(x) for x in exp_dataset.targets)  # type: ignore
+            ))
+            n_classes_per_exp.append(len(unique_targets))
+            classes_order.extend(unique_targets)
+        setattr(benchmark_obj, 'n_classes_per_exp', n_classes_per_exp)
+        setattr(benchmark_obj, 'classes_order', classes_order)
 
     return benchmark_obj
 

avalanche/benchmarks/classic/ctrl.py

@@ -18,9 +18,12 @@
 import torchvision.transforms.functional as F
 from torchvision import transforms
 from tqdm import tqdm
+from avalanche.benchmarks.generators.benchmark_generators import (
+    dataset_classification_benchmark,
+)
 
 from avalanche.benchmarks.utils.classification_dataset import (
-    SupervisedClassificationDataset,
+    ClassificationDataset,
 )
 
 try:
@@ -83,7 +86,7 @@ def CTrL(
         folder = path / "ctrl" / stream_name / f"seed_{seed}"
 
     # Train, val and test experiences
-    exps: List[List[SupervisedClassificationDataset]] = [[], [], []]
+    exps: List[List[ClassificationDataset]] = [[], [], []]
     for t_id, t in enumerate(
         tqdm(stream, desc=f"Loading {stream_name}"),
     ):
@@ -104,7 +107,7 @@ def CTrL(
                 common_root, exp_paths_list = common_paths_root(files)
                 paths_dataset: PathsDataset[Image, int] = \
                     PathsDataset(common_root, exp_paths_list)
-                dataset: SupervisedClassificationDataset = \
+                dataset: ClassificationDataset = \
                     make_classification_dataset(
                         paths_dataset,
                         task_labels=task_labels,
@@ -126,7 +129,7 @@ def CTrL(
             if t_id == n_tasks - 1:
                 break
 
-    return dataset_benchmark(
+    return dataset_classification_benchmark(
         train_datasets=exps[0],
         test_datasets=exps[2],
         other_streams_datasets=dict(val=exps[1]),

avalanche/benchmarks/classic/endless_cl_sim.py

@@ -12,9 +12,15 @@
 """
 This module contains the high-level EndlessCLSim scenario 
 generator. It returns an iterable scenario object 
-``GenericCLScenario`` given a number of configuration parameters.
+:class:`ClassificationScenario` given a number of configuration parameters.
 """
 
+from avalanche.benchmarks.generators.benchmark_generators import (
+    dataset_classification_benchmark,
+)
+from avalanche.benchmarks.scenarios.classification_scenario import (
+    CommonClassificationScenarioType,
+)
 from avalanche.benchmarks.utils.classification_dataset import (
     make_classification_dataset,
 )
@@ -27,11 +33,7 @@
 from torchvision.transforms import ToTensor
 from torchvision.transforms.transforms import Compose
 
-from avalanche.benchmarks.classic.classic_benchmarks_utils import (
-    check_vision_benchmark,
-)
 from avalanche.benchmarks.datasets import default_dataset_location
-from avalanche.benchmarks.generators import dataset_benchmark
 from avalanche.benchmarks.utils import make_classification_dataset
 
 _default_transform = Compose([ToTensor()])
@@ -146,7 +148,11 @@ def EndlessCLSim(
             )
         )
 
-    scenario_obj = dataset_benchmark(train_datasets, eval_datasets)
+    scenario_obj: CommonClassificationScenarioType = \
+        dataset_classification_benchmark(
+            train_datasets,
+            eval_datasets
+        )
 
     return scenario_obj
 

avalanche/benchmarks/classic/openloris.py

@@ -10,8 +10,8 @@
 ################################################################################
 
 """ This module contains the high-level OpenLORIS benchmark/factor generator.
-It basically returns a iterable benchmark object ``GenericCLScenario`` given
-a number of configuration parameters."""
+It basically returns a iterable benchmark object :class:`ClassificationScenario`
+given a number of configuration parameters."""
 
 from pathlib import Path
 from typing import Union, Any, Optional
@@ -23,8 +23,11 @@
 from avalanche.benchmarks.datasets.openloris import (
     OpenLORIS as OpenLORISDataset,
 )
-from avalanche.benchmarks.scenarios.generic_benchmark_creation import (
-    create_generic_benchmark_from_filelists,
+from avalanche.benchmarks.scenarios.classification_benchmark_creation import (
+    create_classification_benchmark_from_filelists,
+)
+from avalanche.benchmarks.scenarios.classification_scenario import (
+    CommonClassificationScenarioType,
 )
 
 
@@ -92,7 +95,7 @@ def OpenLORIS(
         Defaults to None, which means that the default location for
         'openloris' will be used.
 
-    :returns: a properly initialized :class:`GenericCLScenario` instance.
+    :returns: a properly initialized :class:`ClassificationScenario` instance.
     """
 
     assert factor in nbatch.keys(), (
@@ -117,15 +120,16 @@ def OpenLORIS(
             / ("train_batch_" + str(i).zfill(2) + ".txt")
         )
 
-    factor_obj = create_generic_benchmark_from_filelists(
-        dataset_root,
-        train_failists_paths,
-        [dataset_root / filelists_bp / "test.txt"],
-        task_labels=[0 for _ in range(nbatch[factor])],
-        complete_test_set_only=True,
-        train_transform=train_transform,
-        eval_transform=eval_transform,
-    )
+    factor_obj: CommonClassificationScenarioType = \
+        create_classification_benchmark_from_filelists(
+            dataset_root,
+            train_failists_paths,
+            [dataset_root / filelists_bp / "test.txt"],
+            task_labels=[0 for _ in range(nbatch[factor])],
+            complete_test_set_only=True,
+            train_transform=train_transform,
+            eval_transform=eval_transform,
+        )
 
     return factor_obj
 

avalanche/benchmarks/classic/stream51.py

@@ -9,18 +9,26 @@
 # Website: www.continualai.org                                                 #
 ################################################################################
 from pathlib import Path
-from typing import Any, List, Optional, Sequence, Tuple, Union
+from typing import List, Optional, Union
 
 from typing_extensions import Literal
 
+
 from avalanche.benchmarks.datasets import Stream51
+from avalanche.benchmarks.scenarios.classification_benchmark_creation import (
+    create_classification_benchmark_from_paths,
+)
+from avalanche.benchmarks.scenarios.classification_scenario import (
+    CommonClassificationScenarioType,
+)
 from avalanche.benchmarks.scenarios.generic_benchmark_creation import (
-    create_generic_benchmark_from_paths, FileAndLabel
+    FileAndLabel,
 )
 from torchvision import transforms
 import math
 import os
 
+
 _mu = [0.485, 0.456, 0.406]
 _std = [0.229, 0.224, 0.225]
 _default_stream51_transform = transforms.Compose(
@@ -71,7 +79,7 @@ def CLStream51(
     train_transform=_default_stream51_transform,
     eval_transform=_default_stream51_transform,
     dataset_root: Optional[Union[str, Path]] = None
-):
+) -> CommonClassificationScenarioType:
     """
     Creates a CL benchmark for Stream-51.
 
@@ -125,7 +133,7 @@ def CLStream51(
         Defaults to None, which means that the default location for
         'stream51' will be used.
 
-    :returns: A properly initialized :class:`GenericCLScenario` instance.
+    :returns: A properly initialized :class:`ClassificationScenario` instance.
     """
 
     # get train and test sets and order them by benchmark
@@ -283,14 +291,15 @@ def CLStream51(
                 [(j[0], j[1]) for j in i] for i in test_ood_filelists_paths
             ]
 
-    benchmark_obj = create_generic_benchmark_from_paths(
-        train_lists_of_files=train_filelists_paths,
-        test_lists_of_files=test_filelists_paths,
-        task_labels=[0 for _ in range(num_tasks)],
-        complete_test_set_only=scenario == "instance",
-        train_transform=train_transform,
-        eval_transform=eval_transform,
-    )
+    benchmark_obj: CommonClassificationScenarioType = \
+        create_classification_benchmark_from_paths(
+            train_lists_of_files=train_filelists_paths,
+            test_lists_of_files=test_filelists_paths,
+            task_labels=[0 for _ in range(num_tasks)],
+            complete_test_set_only=scenario == "instance",
+            train_transform=train_transform,
+            eval_transform=eval_transform,
+        )
 
     return benchmark_obj
 

avalanche/benchmarks/generators/__init__.py

@@ -1,2 +1 @@
-from .scenario_generators import *
 from .benchmark_generators import *