Merge pull request #53 from Giskard-AI/refactoring-detectors

refactoring detectors

Author: rabah-khalek

giskard_vision/core/dataloaders/base.py

@@ -12,23 +12,10 @@
     get_image_channel_number,
     get_image_size,
 )
-from giskard_vision.core.detectors.base import IssueGroup
+from giskard_vision.core.issues import AttributesIssueMeta
 
 from ..types import TypesBase
 
-EthicalIssueMeta = IssueGroup(
-    "Ethical",
-    description="The data are filtered by metadata like age, facial hair, or gender to detect ethical biases.",
-)
-PerformanceIssueMeta = IssueGroup(
-    "Performance",
-    description="The data are filtered by metadata like emotion, head pose, or exposure value to detect performance issues.",
-)
-AttributesIssueMeta = IssueGroup(
-    "Attributes",
-    description="The data are filtered by the image attributes like width, height, or brightness value to detect issues.",
-)
-
 
 class DataIteratorBase(ABC):
     """Abstract class serving as a base template for DataLoaderBase and DataLoaderWrapper classes.

giskard_vision/core/dataloaders/hf.py

@@ -7,8 +7,9 @@
 
 from PIL.Image import Image as PILImage
 
-from giskard_vision.core.dataloaders.base import AttributesIssueMeta, DataIteratorBase
+from giskard_vision.core.dataloaders.base import DataIteratorBase
 from giskard_vision.core.dataloaders.meta import MetaData, get_pil_image_depth
+from giskard_vision.core.issues import AttributesIssueMeta
 from giskard_vision.utils.errors import GiskardError, GiskardImportError
 
 

giskard_vision/core/dataloaders/meta.py

@@ -4,7 +4,7 @@
 import numpy as np
 from PIL.Image import Image as PILImage
 
-from giskard_vision.core.detectors.base import IssueGroup
+from giskard_vision.core.issues import IssueGroup
 
 
 class MetaData:

giskard_vision/core/detectors/base.py

@@ -2,13 +2,10 @@
 from dataclasses import dataclass
 from typing import Any, List, Optional, Sequence, Tuple
 
+from giskard_vision.core.issues import IssueGroup
 from giskard_vision.utils.errors import GiskardImportError
 
-
-@dataclass(frozen=True)
-class IssueGroup:
-    name: str
-    description: str
+from .specs import DetectorSpecsBase
 
 
 @dataclass
@@ -51,7 +48,7 @@ def get_meta_required(self) -> dict:
         }
 
 
-class DetectorVisionBase:
+class DetectorVisionBase(DetectorSpecsBase):
     """
     Abstract class for Vision Detectors
 
@@ -67,12 +64,6 @@ class DetectorVisionBase:
             evaluation results for the scan.
     """
 
-    issue_group: IssueGroup
-    warning_messages: dict
-    issue_level_threshold: float = 0.2
-    deviation_threshold: float = 0.05
-    num_images: int = 0
-
     def run(
         self,
         model: Any,
@@ -139,6 +130,42 @@ def get_issues(
 
         return issues
 
+    def get_scan_result(
+        self, metric_value, metric_reference_value, metric_name, filename_examples, name, size_data, issue_group
+    ) -> ScanResult:
+        try:
+            from giskard.scanner.issues import IssueLevel
+        except (ImportError, ModuleNotFoundError) as e:
+            raise GiskardImportError(["giskard"]) from e
+
+        relative_delta = metric_value - metric_reference_value
+        if self.metric_type == "relative":
+            relative_delta /= metric_reference_value
+
+        issue_level = IssueLevel.MINOR
+        if self.metric_direction == "better_lower":
+            if relative_delta > self.issue_level_threshold + self.deviation_threshold:
+                issue_level = IssueLevel.MAJOR
+            elif relative_delta > self.issue_level_threshold:
+                issue_level = IssueLevel.MEDIUM
+        elif self.metric_direction == "better_higher":
+            if relative_delta < -(self.issue_level_threshold + self.deviation_threshold):
+                issue_level = IssueLevel.MAJOR
+            elif relative_delta < -self.issue_level_threshold:
+                issue_level = IssueLevel.MEDIUM
+
+        return ScanResult(
+            name=name,
+            metric_name=metric_name,
+            metric_value=metric_value,
+            metric_reference_value=metric_reference_value,
+            issue_level=issue_level,
+            slice_size=size_data,
+            filename_examples=filename_examples,
+            relative_delta=relative_delta,
+            issue_group=issue_group,
+        )
+
     @abstractmethod
     def get_results(self, model: Any, dataset: Any) -> List[ScanResult]:
         """Returns a list of ScanResult

giskard_vision/core/detectors/metadata_scan_detector.py

@@ -4,8 +4,8 @@
 import numpy as np
 import pandas as pd
 
-from giskard_vision.core.dataloaders.base import PerformanceIssueMeta
 from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult
+from giskard_vision.core.issues import PerformanceIssueMeta
 from giskard_vision.core.tests.base import MetricBase
 from giskard_vision.utils.errors import GiskardImportError
 
@@ -258,39 +258,3 @@ def get_df_for_scan(self, model: Any, dataset: Any, list_metadata: Sequence[str]
                 pass
 
         return pd.DataFrame(df)
-
-    def get_scan_result(
-        self, metric_value, metric_reference_value, metric_name, filename_examples, name, size_data, issue_group
-    ) -> ScanResult:
-        try:
-            from giskard.scanner.issues import IssueLevel
-        except (ImportError, ModuleNotFoundError) as e:
-            raise GiskardImportError(["giskard"]) from e
-
-        relative_delta = metric_value - metric_reference_value
-        if self.metric_type == "relative":
-            relative_delta /= metric_reference_value
-
-        issue_level = IssueLevel.MINOR
-        if self.metric_direction == "better_lower":
-            if relative_delta > self.issue_level_threshold + self.deviation_threshold:
-                issue_level = IssueLevel.MAJOR
-            elif relative_delta > self.issue_level_threshold:
-                issue_level = IssueLevel.MEDIUM
-        elif self.metric_direction == "better_higher":
-            if relative_delta < -(self.issue_level_threshold + self.deviation_threshold):
-                issue_level = IssueLevel.MAJOR
-            elif relative_delta < -self.issue_level_threshold:
-                issue_level = IssueLevel.MEDIUM
-
-        return ScanResult(
-            name=name,
-            metric_name=metric_name,
-            metric_value=metric_value,
-            metric_reference_value=metric_reference_value,
-            issue_level=issue_level,
-            slice_size=size_data,
-            filename_examples=filename_examples,
-            relative_delta=relative_delta,
-            issue_group=issue_group,
-        )

giskard_vision/core/detectors/metrics.py

@@ -1,25 +1,15 @@
 import os
 from abc import abstractmethod
+from importlib import import_module
 from pathlib import Path
 from typing import Any, Sequence
 
 import cv2
 
 from giskard_vision.core.dataloaders.wrappers import FilteredDataLoader
-from giskard_vision.core.detectors.base import (
-    DetectorVisionBase,
-    IssueGroup,
-    ScanResult,
-)
+from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult
+from giskard_vision.core.issues import Robustness
 from giskard_vision.core.tests.base import TestDiffBase
-from giskard_vision.utils.errors import GiskardImportError
-
-from .metrics import detector_metrics
-
-Robustness = IssueGroup(
-    "Robustness",
-    description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.",
-)
 
 
 class PerturbationBaseDetector(DetectorVisionBase):
@@ -40,15 +30,28 @@ class PerturbationBaseDetector(DetectorVisionBase):
 
     issue_group = Robustness
 
+    def set_specs_from_model_type(self, model_type):
+        module = import_module(f"giskard_vision.{model_type}.detectors.specs")
+        DetectorSpecs = getattr(module, "DetectorSpecs")
+
+        if DetectorSpecs:
+            # Only set attributes that are not part of Python's special attributes (those starting with __)
+            for attr_name, attr_value in vars(DetectorSpecs).items():
+                if not attr_name.startswith("__") and hasattr(self, attr_name):
+                    setattr(self, attr_name, attr_value)
+        else:
+            raise ValueError(f"No detector specifications found for model type: {model_type}")
+
     @abstractmethod
     def get_dataloaders(self, dataset: Any) -> Sequence[Any]: ...
 
     def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]:
+        self.set_specs_from_model_type(model.model_type)
         dataloaders = self.get_dataloaders(dataset)
 
         results = []
         for dl in dataloaders:
-            test_result = TestDiffBase(metric=detector_metrics[model.model_type], threshold=1).run(
+            test_result = TestDiffBase(metric=self.metric, threshold=1).run(
                 model=model,
                 dataloader=dl,
                 dataloader_ref=dataset,
@@ -63,40 +66,22 @@ def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]:
 
             if isinstance(dl, FilteredDataLoader):
                 filename_example_dataloader_ref = str(Path() / "examples_images" / f"{dataset.name}_{index_worst}.png")
-                cv2.imwrite(
-                    filename_example_dataloader_ref, cv2.resize(dataset[index_worst][0][0], (0, 0), fx=0.3, fy=0.3)
-                )
+                cv2.imwrite(filename_example_dataloader_ref, dataset[index_worst][0][0])
                 filename_examples.append(filename_example_dataloader_ref)
 
             filename_example_dataloader = str(Path() / "examples_images" / f"{dl.name}_{index_worst}.png")
-            cv2.imwrite(filename_example_dataloader, cv2.resize(dl[index_worst][0][0], (0, 0), fx=0.3, fy=0.3))
+            cv2.imwrite(filename_example_dataloader, dl[index_worst][0][0])
             filename_examples.append(filename_example_dataloader)
-            results.append(self.get_scan_result(test_result, filename_examples, dl.name, len(dl)))
+            results.append(
+                self.get_scan_result(
+                    test_result.metric_value_test,
+                    test_result.metric_value_ref,
+                    test_result.metric_name,
+                    filename_examples,
+                    dl.name,
+                    len(dl),
+                    issue_group=self.issue_group,
+                )
+            )
 
         return results
-
-    def get_scan_result(self, test_result, filename_examples, name, size_data) -> ScanResult:
-        try:
-            from giskard.scanner.issues import IssueLevel
-        except (ImportError, ModuleNotFoundError) as e:
-            raise GiskardImportError(["giskard"]) from e
-
-        relative_delta = (test_result.metric_value_test - test_result.metric_value_ref) / test_result.metric_value_ref
-
-        if relative_delta > self.issue_level_threshold + self.deviation_threshold:
-            issue_level = IssueLevel.MAJOR
-        elif relative_delta > self.issue_level_threshold:
-            issue_level = IssueLevel.MEDIUM
-        else:
-            issue_level = IssueLevel.MINOR
-
-        return ScanResult(
-            name=name,
-            metric_name=test_result.metric_name,
-            metric_value=test_result.metric_value_test,
-            metric_reference_value=test_result.metric_value_ref,
-            issue_level=issue_level,
-            slice_size=size_data,
-            filename_examples=filename_examples,
-            relative_delta=relative_delta,
-        )

giskard_vision/core/detectors/perturbation.py

@@ -0,0 +1,13 @@
+from giskard_vision.core.issues import IssueGroup
+from giskard_vision.image_classification.tests.performance import MetricBase
+
+
+class DetectorSpecsBase:
+    issue_group: IssueGroup
+    warning_messages: dict
+    metric: MetricBase = None
+    metric_type: str = None
+    metric_direction: str = None
+    deviation_threshold: float = 0.10
+    issue_level_threshold: float = 0.05
+    num_images: int = 0

giskard_vision/core/detectors/specs.py

@@ -5,7 +5,7 @@
 
 
 @maybe_detector("blurring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"])
-class TransformationBlurringDetectorLandmark(PerturbationBaseDetector):
+class TransformationBlurringDetector(PerturbationBaseDetector):
     """
     Detector that evaluates models performance on blurred images
     """

giskard_vision/core/detectors/transformation_blurring_detector.py

@@ -1,17 +1,15 @@
 from giskard_vision.core.dataloaders.wrappers import ColoredDataLoader
 
 from ...core.detectors.decorator import maybe_detector
-from .perturbation import PerturbationBaseDetector, Robustness
+from .perturbation import PerturbationBaseDetector
 
 
 @maybe_detector("coloring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"])
-class TransformationColorDetectorLandmark(PerturbationBaseDetector):
+class TransformationColorDetector(PerturbationBaseDetector):
     """
     Detector that evaluates models performance depending on images in grayscale
     """
 
-    issue_group = Robustness
-
     def get_dataloaders(self, dataset):
         dl = ColoredDataLoader(dataset)
 

giskard_vision/core/detectors/transformation_color_detector.py

@@ -4,8 +4,8 @@
 from .perturbation import PerturbationBaseDetector
 
 
-@maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"])
-class TransformationNoiseDetectorLandmark(PerturbationBaseDetector):
+@maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection", "noise"])
+class TransformationNoiseDetector(PerturbationBaseDetector):
     """
     Detector that evaluates models performance on noisy images
     """

giskard_vision/core/detectors/transformation_noise_detector.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class IssueGroup:
+    name: str
+    description: str
+
+
+EthicalIssueMeta = IssueGroup(
+    "Ethical",
+    description="The data are filtered by metadata like age, facial hair, or gender to detect ethical biases.",
+)
+PerformanceIssueMeta = IssueGroup(
+    "Performance",
+    description="The data are filtered by metadata like emotion, head pose, or exposure value to detect performance issues.",
+)
+AttributesIssueMeta = IssueGroup(
+    "Attributes",
+    description="The data are filtered by the image attributes like width, height, or brightness value to detect issues.",
+)
+Robustness = IssueGroup(
+    "Robustness",
+    description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.",
+)

giskard_vision/core/issues.py

@@ -3,11 +3,11 @@
 import numpy as np
 from PIL.Image import Image as PILImage
 
-from giskard_vision.core.dataloaders.base import EthicalIssueMeta, PerformanceIssueMeta
 from giskard_vision.core.dataloaders.hf import HFDataLoader
 from giskard_vision.core.dataloaders.meta import MetaData
 from giskard_vision.core.dataloaders.tfds import DataLoaderTensorFlowDatasets
 from giskard_vision.core.dataloaders.utils import flatten_dict
+from giskard_vision.core.issues import EthicalIssueMeta, PerformanceIssueMeta
 from giskard_vision.image_classification.types import Types