diff --git a/giskard_vision/core/detectors/base.py b/giskard_vision/core/detectors/base.py index 1ca41ca3..09743097 100644 --- a/giskard_vision/core/detectors/base.py +++ b/giskard_vision/core/detectors/base.py @@ -36,15 +36,16 @@ class ScanResult: relative_delta: float issue_group: Optional[IssueGroup] = None - def get_meta_required(self) -> dict: + def get_meta_required(self, add_slice_size=True) -> dict: # Get the meta required by the original scan API deviation = f"{self.relative_delta * 100:+.2f}% than global" + extra_meta = dict(slice_size=self.slice_size) if add_slice_size else dict() return { "metric": self.metric_name, "metric_value": self.metric_value, "metric_reference_value": self.metric_reference_value, "deviation": deviation, - "slice_size": self.slice_size, + **extra_meta, } @@ -64,6 +65,8 @@ class DetectorVisionBase(DetectorSpecsBase): evaluation results for the scan. """ + slicing: bool = True + def run( self, model: Any, @@ -115,16 +118,20 @@ def get_issues( for result in results: if result.issue_level in issue_levels: + issue_attribute = ( + dict(slicing_fn=result.name, meta=result.get_meta_required()) + if self.slicing + else dict(transformation_fn=result.name, meta=result.get_meta_required(False)) + ) issues.append( Issue( model, dataset, level=result.issue_level, - slicing_fn=result.name, group=result.issue_group if result.issue_group else self.issue_group, - meta=result.get_meta_required(), scan_examples=ImagesScanExamples(result.filename_examples, embed=embed), display_footer_info=False, + **issue_attribute, ) ) diff --git a/giskard_vision/core/detectors/perturbation.py b/giskard_vision/core/detectors/perturbation.py index 91edbbe7..cf3c17b1 100644 --- a/giskard_vision/core/detectors/perturbation.py +++ b/giskard_vision/core/detectors/perturbation.py @@ -6,7 +6,6 @@ import cv2 -from giskard_vision.core.dataloaders.wrappers import FilteredDataLoader from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult from giskard_vision.core.issues import Robustness from giskard_vision.core.tests.base import TestDiffBase @@ -29,6 +28,7 @@ class PerturbationBaseDetector(DetectorVisionBase): """ issue_group = Robustness + slicing = False def set_specs_from_model_type(self, model_type): module = import_module(f"giskard_vision.{model_type}.detectors.specs") @@ -64,10 +64,9 @@ def get_results(self, model: Any, dataset: Any) -> Sequence[ScanResult]: index_worst = 0 if test_result.indexes_examples is None else test_result.indexes_examples[0] - if isinstance(dl, FilteredDataLoader): - filename_example_dataloader_ref = str(Path() / "examples_images" / f"{dataset.name}_{index_worst}.png") - cv2.imwrite(filename_example_dataloader_ref, dataset[index_worst][0][0]) - filename_examples.append(filename_example_dataloader_ref) + filename_example_dataloader_ref = str(Path() / "examples_images" / f"{dataset.name}_{index_worst}.png") + cv2.imwrite(filename_example_dataloader_ref, dataset[index_worst][0][0]) + filename_examples.append(filename_example_dataloader_ref) filename_example_dataloader = str(Path() / "examples_images" / f"{dl.name}_{index_worst}.png") cv2.imwrite(filename_example_dataloader, dl[index_worst][0][0]) diff --git a/giskard_vision/core/detectors/transformation_color_detector.py b/giskard_vision/core/detectors/transformation_color_detector.py index 6b23e9c3..10d7ad2e 100644 --- a/giskard_vision/core/detectors/transformation_color_detector.py +++ b/giskard_vision/core/detectors/transformation_color_detector.py @@ -4,7 +4,9 @@ from .perturbation import PerturbationBaseDetector -@maybe_detector("coloring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection"]) +@maybe_detector( + "coloring", tags=["vision", "robustness", "image_classification", "landmark", "object_detection", "coloring"] +) class TransformationColorDetector(PerturbationBaseDetector): """ Detector that evaluates models performance depending on images in grayscale