diff --git a/giskard_vision/core/dataloaders/base.py b/giskard_vision/core/dataloaders/base.py index 5c91091b..73f9dfaf 100644 --- a/giskard_vision/core/dataloaders/base.py +++ b/giskard_vision/core/dataloaders/base.py @@ -12,23 +12,10 @@ get_image_channel_number, get_image_size, ) -from giskard_vision.core.detectors.base import IssueGroup +from giskard_vision.core.issues import AttributesIssueMeta from ..types import TypesBase -EthicalIssueMeta = IssueGroup( - "Ethical", - description="The data are filtered by metadata like age, facial hair, or gender to detect ethical biases.", -) -PerformanceIssueMeta = IssueGroup( - "Performance", - description="The data are filtered by metadata like emotion, head pose, or exposure value to detect performance issues.", -) -AttributesIssueMeta = IssueGroup( - "Attributes", - description="The data are filtered by the image attributes like width, height, or brightness value to detect issues.", -) - class DataIteratorBase(ABC): """Abstract class serving as a base template for DataLoaderBase and DataLoaderWrapper classes. diff --git a/giskard_vision/core/dataloaders/hf.py b/giskard_vision/core/dataloaders/hf.py index 10100319..60dfc274 100644 --- a/giskard_vision/core/dataloaders/hf.py +++ b/giskard_vision/core/dataloaders/hf.py @@ -7,8 +7,9 @@ from PIL.Image import Image as PILImage -from giskard_vision.core.dataloaders.base import AttributesIssueMeta, DataIteratorBase +from giskard_vision.core.dataloaders.base import DataIteratorBase from giskard_vision.core.dataloaders.meta import MetaData, get_pil_image_depth +from giskard_vision.core.issues import AttributesIssueMeta from giskard_vision.utils.errors import GiskardError, GiskardImportError diff --git a/giskard_vision/core/dataloaders/meta.py b/giskard_vision/core/dataloaders/meta.py index 93c06c14..f91d7b67 100644 --- a/giskard_vision/core/dataloaders/meta.py +++ b/giskard_vision/core/dataloaders/meta.py @@ -4,7 +4,7 @@ import numpy as np from PIL.Image import Image as PILImage -from giskard_vision.core.detectors.base import IssueGroup +from giskard_vision.core.issues import IssueGroup class MetaData: diff --git a/giskard_vision/core/detectors/base.py b/giskard_vision/core/detectors/base.py index d6197753..1ca41ca3 100644 --- a/giskard_vision/core/detectors/base.py +++ b/giskard_vision/core/detectors/base.py @@ -2,13 +2,10 @@ from dataclasses import dataclass from typing import Any, List, Optional, Sequence, Tuple +from giskard_vision.core.issues import IssueGroup from giskard_vision.utils.errors import GiskardImportError - -@dataclass(frozen=True) -class IssueGroup: - name: str - description: str +from .specs import DetectorSpecsBase @dataclass @@ -51,7 +48,7 @@ def get_meta_required(self) -> dict: } -class DetectorVisionBase: +class DetectorVisionBase(DetectorSpecsBase): """ Abstract class for Vision Detectors @@ -67,12 +64,6 @@ class DetectorVisionBase: evaluation results for the scan. """ - issue_group: IssueGroup - warning_messages: dict - issue_level_threshold: float = 0.2 - deviation_threshold: float = 0.05 - num_images: int = 0 - def run( self, model: Any, @@ -139,6 +130,42 @@ def get_issues( return issues + def get_scan_result( + self, metric_value, metric_reference_value, metric_name, filename_examples, name, size_data, issue_group + ) -> ScanResult: + try: + from giskard.scanner.issues import IssueLevel + except (ImportError, ModuleNotFoundError) as e: + raise GiskardImportError(["giskard"]) from e + + relative_delta = metric_value - metric_reference_value + if self.metric_type == "relative": + relative_delta /= metric_reference_value + + issue_level = IssueLevel.MINOR + if self.metric_direction == "better_lower": + if relative_delta > self.issue_level_threshold + self.deviation_threshold: + issue_level = IssueLevel.MAJOR + elif relative_delta > self.issue_level_threshold: + issue_level = IssueLevel.MEDIUM + elif self.metric_direction == "better_higher": + if relative_delta < -(self.issue_level_threshold + self.deviation_threshold): + issue_level = IssueLevel.MAJOR + elif relative_delta < -self.issue_level_threshold: + issue_level = IssueLevel.MEDIUM + + return ScanResult( + name=name, + metric_name=metric_name, + metric_value=metric_value, + metric_reference_value=metric_reference_value, + issue_level=issue_level, + slice_size=size_data, + filename_examples=filename_examples, + relative_delta=relative_delta, + issue_group=issue_group, + ) + @abstractmethod def get_results(self, model: Any, dataset: Any) -> List[ScanResult]: """Returns a list of ScanResult diff --git a/giskard_vision/core/detectors/metadata_scan_detector.py b/giskard_vision/core/detectors/metadata_scan_detector.py index 50ae8d3b..83fcab7e 100644 --- a/giskard_vision/core/detectors/metadata_scan_detector.py +++ b/giskard_vision/core/detectors/metadata_scan_detector.py @@ -4,8 +4,8 @@ import numpy as np import pandas as pd -from giskard_vision.core.dataloaders.base import PerformanceIssueMeta from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult +from giskard_vision.core.issues import PerformanceIssueMeta from giskard_vision.core.tests.base import MetricBase from giskard_vision.utils.errors import GiskardImportError @@ -258,39 +258,3 @@ def get_df_for_scan(self, model: Any, dataset: Any, list_metadata: Sequence[str] pass return pd.DataFrame(df) - - def get_scan_result( - self, metric_value, metric_reference_value, metric_name, filename_examples, name, size_data, issue_group - ) -> ScanResult: - try: - from giskard.scanner.issues import IssueLevel - except (ImportError, ModuleNotFoundError) as e: - raise GiskardImportError(["giskard"]) from e - - relative_delta = metric_value - metric_reference_value - if self.metric_type == "relative": - relative_delta /= metric_reference_value - - issue_level = IssueLevel.MINOR - if self.metric_direction == "better_lower": - if relative_delta > self.issue_level_threshold + self.deviation_threshold: - issue_level = IssueLevel.MAJOR - elif relative_delta > self.issue_level_threshold: - issue_level = IssueLevel.MEDIUM - elif self.metric_direction == "better_higher": - if relative_delta < -(self.issue_level_threshold + self.deviation_threshold): - issue_level = IssueLevel.MAJOR - elif relative_delta < -self.issue_level_threshold: - issue_level = IssueLevel.MEDIUM - - return ScanResult( - name=name, - metric_name=metric_name, - metric_value=metric_value, - metric_reference_value=metric_reference_value, - issue_level=issue_level, - slice_size=size_data, - filename_examples=filename_examples, - relative_delta=relative_delta, - issue_group=issue_group, - ) diff --git a/giskard_vision/core/detectors/metrics.py b/giskard_vision/core/detectors/metrics.py deleted file mode 100644 index 744ee1f7..00000000 --- a/giskard_vision/core/detectors/metrics.py +++ /dev/null @@ -1,9 +0,0 @@ -from giskard_vision.image_classification.tests.performance import Accuracy -from giskard_vision.landmark_detection.tests.performance import NMEMean -from giskard_vision.object_detection.tests.performance import IoUMean - -detector_metrics = { - "image_classification": Accuracy, - "landmark": NMEMean, - "object_detection": IoUMean, -} diff --git a/giskard_vision/core/detectors/perturbation.py b/giskard_vision/core/detectors/perturbation.py index 61a767f8..91edbbe7 100644 --- a/giskard_vision/core/detectors/perturbation.py +++ b/giskard_vision/core/detectors/perturbation.py @@ -1,25 +1,15 @@ import os from abc import abstractmethod +from importlib import import_module from pathlib import Path from typing import Any, Sequence import cv2 from giskard_vision.core.dataloaders.wrappers import FilteredDataLoader -from giskard_vision.core.detectors.base import ( - DetectorVisionBase, - IssueGroup, - ScanResult, -) +from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult +from giskard_vision.core.issues import Robustness from giskard_vision.core.tests.base import TestDiffBase -from giskard_vision.utils.errors import GiskardImportError - -from .metrics import detector_metrics - -Robustness = IssueGroup( - "Robustness", - description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.", -) class PerturbationBaseDetector(DetectorVisionBase): @@ -40,15 +30,28 @@ class PerturbationBaseDetector(DetectorVisionBase): issue_group = Robustness + def set_specs_from_model_type(self, model_type): + module = import_module(f"giskard_vision.{model_type}.detectors.specs") + DetectorSpecs = getattr(module, "DetectorSpecs") + + if DetectorSpecs: + # Only set attributes that are not part of Python's special attributes (those starting with __) + for attr_name, attr_value in vars(DetectorSpecs).items(): + if not attr_name.startswith("__") and hasattr(self, attr_name): + setattr(self, attr_name, attr_value) + else: + raise ValueError(f"No detector specifications found for model type: {model_type}") + @abstractmethod def get_dataloaders(self, dataset: Any) -> Sequence[Any]: ... def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]: + self.set_specs_from_model_type(model.model_type) dataloaders = self.get_dataloaders(dataset) results = [] for dl in dataloaders: - test_result = TestDiffBase(metric=detector_metrics[model.model_type], threshold=1).run( + test_result = TestDiffBase(metric=self.metric, threshold=1).run( model=model, dataloader=dl, dataloader_ref=dataset, @@ -63,40 +66,22 @@ def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]: if isinstance(dl, FilteredDataLoader): filename_example_dataloader_ref = str(Path() / "examples_images" / f"{dataset.name}_{index_worst}.png") - cv2.imwrite( - filename_example_dataloader_ref, cv2.resize(dataset[index_worst][0][0], (0, 0), fx=0.3, fy=0.3) - ) + cv2.imwrite(filename_example_dataloader_ref, dataset[index_worst][0][0]) filename_examples.append(filename_example_dataloader_ref) filename_example_dataloader = str(Path() / "examples_images" / f"{dl.name}_{index_worst}.png") - cv2.imwrite(filename_example_dataloader, cv2.resize(dl[index_worst][0][0], (0, 0), fx=0.3, fy=0.3)) + cv2.imwrite(filename_example_dataloader, dl[index_worst][0][0]) filename_examples.append(filename_example_dataloader) - results.append(self.get_scan_result(test_result, filename_examples, dl.name, len(dl))) + results.append( + self.get_scan_result( + test_result.metric_value_test, + test_result.metric_value_ref, + test_result.metric_name, + filename_examples, + dl.name, + len(dl), + issue_group=self.issue_group, + ) + ) return results - - def get_scan_result(self, test_result, filename_examples, name, size_data) -> ScanResult: - try: - from giskard.scanner.issues import IssueLevel - except (ImportError, ModuleNotFoundError) as e: - raise GiskardImportError(["giskard"]) from e - - relative_delta = (test_result.metric_value_test - test_result.metric_value_ref) / test_result.metric_value_ref - - if relative_delta > self.issue_level_threshold + self.deviation_threshold: - issue_level = IssueLevel.MAJOR - elif relative_delta > self.issue_level_threshold: - issue_level = IssueLevel.MEDIUM - else: - issue_level = IssueLevel.MINOR - - return ScanResult( - name=name, - metric_name=test_result.metric_name, - metric_value=test_result.metric_value_test, - metric_reference_value=test_result.metric_value_ref, - issue_level=issue_level, - slice_size=size_data, - filename_examples=filename_examples, - relative_delta=relative_delta, - ) diff --git a/giskard_vision/core/detectors/specs.py b/giskard_vision/core/detectors/specs.py new file mode 100644 index 00000000..3bf01c69 --- /dev/null +++ b/giskard_vision/core/detectors/specs.py @@ -0,0 +1,13 @@ +from giskard_vision.core.issues import IssueGroup +from giskard_vision.image_classification.tests.performance import MetricBase + + +class DetectorSpecsBase: + issue_group: IssueGroup + warning_messages: dict + metric: MetricBase = None + metric_type: str = None + metric_direction: str = None + deviation_threshold: float = 0.10 + issue_level_threshold: float = 0.05 + num_images: int = 0 diff --git a/giskard_vision/core/detectors/transformation_blurring_detector.py b/giskard_vision/core/detectors/transformation_blurring_detector.py index be1b912c..05f21e1c 100644 --- a/giskard_vision/core/detectors/transformation_blurring_detector.py +++ b/giskard_vision/core/detectors/transformation_blurring_detector.py @@ -5,7 +5,7 @@ @maybe_detector("blurring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) -class TransformationBlurringDetectorLandmark(PerturbationBaseDetector): +class TransformationBlurringDetector(PerturbationBaseDetector): """ Detector that evaluates models performance on blurred images """ diff --git a/giskard_vision/core/detectors/transformation_color_detector.py b/giskard_vision/core/detectors/transformation_color_detector.py index 70488ef0..6b23e9c3 100644 --- a/giskard_vision/core/detectors/transformation_color_detector.py +++ b/giskard_vision/core/detectors/transformation_color_detector.py @@ -1,17 +1,15 @@ from giskard_vision.core.dataloaders.wrappers import ColoredDataLoader from ...core.detectors.decorator import maybe_detector -from .perturbation import PerturbationBaseDetector, Robustness +from .perturbation import PerturbationBaseDetector @maybe_detector("coloring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) -class TransformationColorDetectorLandmark(PerturbationBaseDetector): +class TransformationColorDetector(PerturbationBaseDetector): """ Detector that evaluates models performance depending on images in grayscale """ - issue_group = Robustness - def get_dataloaders(self, dataset): dl = ColoredDataLoader(dataset) diff --git a/giskard_vision/core/detectors/transformation_noise_detector.py b/giskard_vision/core/detectors/transformation_noise_detector.py index 0ce63b13..11f8f2ba 100644 --- a/giskard_vision/core/detectors/transformation_noise_detector.py +++ b/giskard_vision/core/detectors/transformation_noise_detector.py @@ -4,8 +4,8 @@ from .perturbation import PerturbationBaseDetector -@maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) -class TransformationNoiseDetectorLandmark(PerturbationBaseDetector): +@maybe_detector("noise", tags=["vision", "robustness", "image_classification", "landmark", "object_detection", "noise"]) +class TransformationNoiseDetector(PerturbationBaseDetector): """ Detector that evaluates models performance on noisy images """ diff --git a/giskard_vision/core/issues.py b/giskard_vision/core/issues.py new file mode 100644 index 00000000..9e4cebc7 --- /dev/null +++ b/giskard_vision/core/issues.py @@ -0,0 +1,25 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class IssueGroup: + name: str + description: str + + +EthicalIssueMeta = IssueGroup( + "Ethical", + description="The data are filtered by metadata like age, facial hair, or gender to detect ethical biases.", +) +PerformanceIssueMeta = IssueGroup( + "Performance", + description="The data are filtered by metadata like emotion, head pose, or exposure value to detect performance issues.", +) +AttributesIssueMeta = IssueGroup( + "Attributes", + description="The data are filtered by the image attributes like width, height, or brightness value to detect issues.", +) +Robustness = IssueGroup( + "Robustness", + description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.", +) diff --git a/giskard_vision/image_classification/dataloaders/loaders.py b/giskard_vision/image_classification/dataloaders/loaders.py index c2550fa7..753fc1cc 100644 --- a/giskard_vision/image_classification/dataloaders/loaders.py +++ b/giskard_vision/image_classification/dataloaders/loaders.py @@ -3,11 +3,11 @@ import numpy as np from PIL.Image import Image as PILImage -from giskard_vision.core.dataloaders.base import EthicalIssueMeta, PerformanceIssueMeta from giskard_vision.core.dataloaders.hf import HFDataLoader from giskard_vision.core.dataloaders.meta import MetaData from giskard_vision.core.dataloaders.tfds import DataLoaderTensorFlowDatasets from giskard_vision.core.dataloaders.utils import flatten_dict +from giskard_vision.core.issues import EthicalIssueMeta, PerformanceIssueMeta from giskard_vision.image_classification.types import Types diff --git a/giskard_vision/image_classification/detectors/metadata_detector.py b/giskard_vision/image_classification/detectors/metadata_detector.py index 42a876da..4e567b24 100644 --- a/giskard_vision/image_classification/detectors/metadata_detector.py +++ b/giskard_vision/image_classification/detectors/metadata_detector.py @@ -1,14 +1,9 @@ from giskard_vision.core.detectors.metadata_scan_detector import MetaDataScanDetector -from giskard_vision.image_classification.tests.performance import Accuracy from ...core.detectors.decorator import maybe_detector +from .specs import DetectorSpecs @maybe_detector("metadata_classification", tags=["vision", "image_classification", "metadata"]) -class MetaDataScanDetectorClassification(MetaDataScanDetector): - metric = Accuracy - type_task = "classification" - metric_type = "absolute" - metric_direction = "better_higher" - deviation_threshold = 0.10 - issue_level_threshold = 0.05 +class MetaDataScanDetectorClassification(DetectorSpecs, MetaDataScanDetector): + pass diff --git a/giskard_vision/image_classification/detectors/specs.py b/giskard_vision/image_classification/detectors/specs.py new file mode 100644 index 00000000..109ba5a1 --- /dev/null +++ b/giskard_vision/image_classification/detectors/specs.py @@ -0,0 +1,11 @@ +from giskard_vision.core.detectors.specs import DetectorSpecsBase +from giskard_vision.image_classification.tests.performance import Accuracy, MetricBase + + +class DetectorSpecs(DetectorSpecsBase): + metric: MetricBase = Accuracy + type_task: str = "classification" + metric_type: str = "absolute" + metric_direction: str = "better_higher" + deviation_threshold: float = 0.10 + issue_level_threshold: float = 0.05 diff --git a/giskard_vision/landmark_detection/dataloaders/loaders.py b/giskard_vision/landmark_detection/dataloaders/loaders.py index d44f8123..24060a44 100644 --- a/giskard_vision/landmark_detection/dataloaders/loaders.py +++ b/giskard_vision/landmark_detection/dataloaders/loaders.py @@ -5,10 +5,10 @@ import cv2 import numpy as np -from giskard_vision.core.dataloaders.base import EthicalIssueMeta, PerformanceIssueMeta from giskard_vision.core.dataloaders.meta import MetaData from giskard_vision.core.dataloaders.tfds import DataLoaderTensorFlowDatasets from giskard_vision.core.dataloaders.utils import flatten_dict +from giskard_vision.core.issues import EthicalIssueMeta, PerformanceIssueMeta from giskard_vision.landmark_detection.types import Types from .base import DataLoaderBase diff --git a/giskard_vision/landmark_detection/detectors/__init__.py b/giskard_vision/landmark_detection/detectors/__init__.py index e9726a55..d90f809e 100644 --- a/giskard_vision/landmark_detection/detectors/__init__.py +++ b/giskard_vision/landmark_detection/detectors/__init__.py @@ -1,13 +1,9 @@ from .cropping_detector import CroppingDetectorLandmark from .metadata_detector import MetaDataScanDetectorLandmark -from .transformation_blurring_detector import TransformationBlurringDetectorLandmark -from .transformation_color_detector import TransformationColorDetectorLandmark from .transformation_resize_detector import TransformationResizeDetectorLandmark __all__ = [ "CroppingDetectorLandmark", - "TransformationBlurringDetectorLandmark", - "TransformationColorDetectorLandmark", "TransformationResizeDetectorLandmark", "MetaDataScanDetectorLandmark", ] diff --git a/giskard_vision/landmark_detection/detectors/base.py b/giskard_vision/landmark_detection/detectors/base.py index a1b1b8d7..d94d2560 100644 --- a/giskard_vision/landmark_detection/detectors/base.py +++ b/giskard_vision/landmark_detection/detectors/base.py @@ -6,34 +6,11 @@ import cv2 from giskard_vision.core.dataloaders.wrappers import FilteredDataLoader -from giskard_vision.core.detectors.base import ( - DetectorVisionBase, - IssueGroup, - ScanResult, -) +from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult from giskard_vision.landmark_detection.tests.base import TestDiff from giskard_vision.landmark_detection.tests.performance import NMEMean from giskard_vision.utils.errors import GiskardImportError -Cropping = IssueGroup( - "Cropping", description="Cropping involves evaluating the landmark detection model on specific face areas." -) - -Ethical = IssueGroup( - "Ethical", - description="The data are filtered by ethnicity to detect ethical biases in the landmark detection model.", -) - -Pose = IssueGroup( - "Head Pose", - description="The data are filtered by head pose to detect biases in the landmark detection model.", -) - -Robustness = IssueGroup( - "Robustness", - description="Images from the dataset are blurred, recolored and resized to test the robustness of the model to transformations.", -) - class LandmarkDetectionBaseDetector(DetectorVisionBase): """ diff --git a/giskard_vision/landmark_detection/detectors/cropping_detector.py b/giskard_vision/landmark_detection/detectors/cropping_detector.py index 96a710e7..8689374f 100644 --- a/giskard_vision/landmark_detection/detectors/cropping_detector.py +++ b/giskard_vision/landmark_detection/detectors/cropping_detector.py @@ -1,7 +1,9 @@ +from giskard_vision.core.issues import Robustness + from ...core.detectors.decorator import maybe_detector from ..dataloaders.wrappers import CroppedDataLoader from ..marks.facial_parts import FacialParts -from .base import Cropping, LandmarkDetectionBaseDetector +from .base import LandmarkDetectionBaseDetector @maybe_detector("cropping_landmark", tags=["vision", "face", "landmark", "transformed", "cropped"]) @@ -10,7 +12,7 @@ class CroppingDetectorLandmark(LandmarkDetectionBaseDetector): Detector that evaluates models performance relative to a facial part """ - issue_group = Cropping + issue_group = Robustness def get_dataloaders(self, dataset): facial_parts = [elt.value for elt in FacialParts] diff --git a/giskard_vision/landmark_detection/detectors/metadata_detector.py b/giskard_vision/landmark_detection/detectors/metadata_detector.py index 327ca570..6e182239 100644 --- a/giskard_vision/landmark_detection/detectors/metadata_detector.py +++ b/giskard_vision/landmark_detection/detectors/metadata_detector.py @@ -3,14 +3,11 @@ SurrogateNME, SurrogateVolumeConvexHull, ) -from giskard_vision.landmark_detection.tests.performance import NMEMean from ...core.detectors.decorator import maybe_detector +from .specs import DetectorSpecs @maybe_detector("metadata_landmark", tags=["vision", "face", "landmark", "metadata"]) -class MetaDataScanDetectorLandmark(MetaDataScanDetector): +class MetaDataScanDetectorLandmark(DetectorSpecs, MetaDataScanDetector): surrogates = [SurrogateVolumeConvexHull, SurrogateNME] - metric = NMEMean - type_task = "regression" - metric_type = "relative" diff --git a/giskard_vision/landmark_detection/detectors/specs.py b/giskard_vision/landmark_detection/detectors/specs.py new file mode 100644 index 00000000..9e8afa80 --- /dev/null +++ b/giskard_vision/landmark_detection/detectors/specs.py @@ -0,0 +1,8 @@ +from giskard_vision.core.detectors.specs import DetectorSpecsBase +from giskard_vision.landmark_detection.tests.performance import NMEMean + + +class DetectorSpecs(DetectorSpecsBase): + metric = NMEMean + type_task = "regression" + metric_type = "relative" diff --git a/giskard_vision/landmark_detection/detectors/transformation_blurring_detector.py b/giskard_vision/landmark_detection/detectors/transformation_blurring_detector.py deleted file mode 100644 index 7e811089..00000000 --- a/giskard_vision/landmark_detection/detectors/transformation_blurring_detector.py +++ /dev/null @@ -1,24 +0,0 @@ -from giskard_vision.core.dataloaders.wrappers import BlurredDataLoader - -from ...core.detectors.decorator import maybe_detector -from .base import LandmarkDetectionBaseDetector, Robustness - - -@maybe_detector("blurring_landmark", tags=["vision", "face", "landmark", "transformed", "blurred"]) -class TransformationBlurringDetectorLandmark(LandmarkDetectionBaseDetector): - """ - Detector that evaluates models performance on blurred images - """ - - issue_group = Robustness - - def __init__(self, kernel_size=(11, 11), sigma=(3, 3)): - self.kernel_size = kernel_size - self.sigma = sigma - - def get_dataloaders(self, dataset): - dl = BlurredDataLoader(dataset, self.kernel_size, self.sigma) - - dls = [dl] - - return dls diff --git a/giskard_vision/landmark_detection/detectors/transformation_color_detector.py b/giskard_vision/landmark_detection/detectors/transformation_color_detector.py deleted file mode 100644 index f4559df5..00000000 --- a/giskard_vision/landmark_detection/detectors/transformation_color_detector.py +++ /dev/null @@ -1,20 +0,0 @@ -from giskard_vision.core.dataloaders.wrappers import ColoredDataLoader - -from ...core.detectors.decorator import maybe_detector -from .base import LandmarkDetectionBaseDetector, Robustness - - -@maybe_detector("color_landmark", tags=["vision", "face", "landmark", "filtered", "colored"]) -class TransformationColorDetectorLandmark(LandmarkDetectionBaseDetector): - """ - Detector that evaluates models performance depending on images in grayscale - """ - - issue_group = Robustness - - def get_dataloaders(self, dataset): - dl = ColoredDataLoader(dataset) - - dls = [dl] - - return dls diff --git a/giskard_vision/landmark_detection/detectors/transformation_resize_detector.py b/giskard_vision/landmark_detection/detectors/transformation_resize_detector.py index 68950fe1..f017df22 100644 --- a/giskard_vision/landmark_detection/detectors/transformation_resize_detector.py +++ b/giskard_vision/landmark_detection/detectors/transformation_resize_detector.py @@ -1,7 +1,8 @@ +from giskard_vision.core.issues import Robustness from giskard_vision.landmark_detection.dataloaders.wrappers import ResizedDataLoader from ...core.detectors.decorator import maybe_detector -from .base import LandmarkDetectionBaseDetector, Robustness +from .base import LandmarkDetectionBaseDetector @maybe_detector("resize_landmark", tags=["vision", "face", "landmark", "transformed", "resized"]) diff --git a/giskard_vision/landmark_detection/models/base.py b/giskard_vision/landmark_detection/models/base.py index 95047c20..738427e3 100644 --- a/giskard_vision/landmark_detection/models/base.py +++ b/giskard_vision/landmark_detection/models/base.py @@ -13,7 +13,7 @@ class FaceLandmarksModelBase(ModelBase): """Abstract class that serves as a template for all landmark model predictions""" - model_type = "landmark" + model_type = "landmark_detection" prediction_result_cls = Types.prediction_result def __init__(self, n_landmarks: int, n_dimensions: int, name: Optional[str] = None) -> None: diff --git a/giskard_vision/object_detection/dataloaders/loaders.py b/giskard_vision/object_detection/dataloaders/loaders.py index 88366040..a38946cd 100644 --- a/giskard_vision/object_detection/dataloaders/loaders.py +++ b/giskard_vision/object_detection/dataloaders/loaders.py @@ -9,9 +9,10 @@ from numpy import ndarray from PIL.Image import Image as PILImage -from giskard_vision.core.dataloaders.base import DataIteratorBase, PerformanceIssueMeta +from giskard_vision.core.dataloaders.base import DataIteratorBase from giskard_vision.core.dataloaders.hf import HFDataLoader from giskard_vision.core.dataloaders.meta import MetaData +from giskard_vision.core.issues import PerformanceIssueMeta from giskard_vision.landmark_detection.dataloaders.loaders import ( DataLoader300W, DataLoaderFFHQ, diff --git a/giskard_vision/object_detection/detectors/metadata_detector.py b/giskard_vision/object_detection/detectors/metadata_detector.py index 74844b98..b1b740f2 100644 --- a/giskard_vision/object_detection/detectors/metadata_detector.py +++ b/giskard_vision/object_detection/detectors/metadata_detector.py @@ -15,13 +15,13 @@ SurrogateRelativeTopLeftY, SurrogateStdIntensity, ) -from giskard_vision.object_detection.tests.performance import IoUMean from ...core.detectors.decorator import maybe_detector +from .specs import DetectorSpecs @maybe_detector("metadata_object_detection", tags=["vision", "object_detection", "metadata"]) -class MetaDataScanDetectorObjectDetection(MetaDataScanDetector): +class MetaDataScanDetectorObjectDetection(DetectorSpecs, MetaDataScanDetector): surrogates = [ SurrogateCenterMassX, SurrogateCenterMassY, @@ -38,9 +38,3 @@ class MetaDataScanDetectorObjectDetection(MetaDataScanDetector): SurrogateRelativeTopLeftY, SurrogateNormalizedPerimeter, ] - metric = IoUMean - type_task = "regression" - metric_type = "absolute" - metric_direction = "better_higher" - deviation_threshold = 0.10 - issue_level_threshold = 0.05 diff --git a/giskard_vision/object_detection/detectors/specs.py b/giskard_vision/object_detection/detectors/specs.py new file mode 100644 index 00000000..2f13483a --- /dev/null +++ b/giskard_vision/object_detection/detectors/specs.py @@ -0,0 +1,11 @@ +from giskard_vision.core.detectors.specs import DetectorSpecsBase +from giskard_vision.object_detection.tests.performance import IoUMean + + +class DetectorSpecs(DetectorSpecsBase): + metric = IoUMean + type_task = "regression" + metric_type = "absolute" + metric_direction = "better_higher" + deviation_threshold = 0.10 + issue_level_threshold = 0.05 diff --git a/tests/landmark_detection/detectors/test_detectors.py b/tests/landmark_detection/detectors/test_detectors.py index f812e740..1e893809 100644 --- a/tests/landmark_detection/detectors/test_detectors.py +++ b/tests/landmark_detection/detectors/test_detectors.py @@ -1,11 +1,18 @@ from giskard.scanner.issues import Issue, IssueLevel from pytest import mark +from giskard_vision.core.detectors.transformation_blurring_detector import ( + TransformationBlurringDetector, +) +from giskard_vision.core.detectors.transformation_color_detector import ( + TransformationColorDetector, +) +from giskard_vision.core.detectors.transformation_noise_detector import ( + TransformationNoiseDetector, +) from giskard_vision.landmark_detection.detectors import ( CroppingDetectorLandmark, MetaDataScanDetectorLandmark, - TransformationBlurringDetectorLandmark, - TransformationColorDetectorLandmark, TransformationResizeDetectorLandmark, ) from giskard_vision.landmark_detection.detectors.base import ScanResult @@ -15,8 +22,9 @@ "detector", [ CroppingDetectorLandmark, - TransformationBlurringDetectorLandmark, - TransformationColorDetectorLandmark, + TransformationBlurringDetector, + TransformationColorDetector, + TransformationNoiseDetector, TransformationResizeDetectorLandmark, ], )