diff --git a/giskard_vision/core/detectors/metadata_scan_detector.py b/giskard_vision/core/detectors/metadata_scan_detector.py index 854db7c5..6850551a 100644 --- a/giskard_vision/core/detectors/metadata_scan_detector.py +++ b/giskard_vision/core/detectors/metadata_scan_detector.py @@ -85,26 +85,27 @@ def get_results(self, model: Any, dataset: Any) -> List[ScanResult]: # For each slice found, get appropriate scan results with the metric for issue in results.issues: - current_data_slice = giskard_dataset.slice(issue.slicing_fn) - indices = list(current_data_slice.df.sort_values(by="metric", ascending=False)["index"].values) - if not self.check_slice_already_selected(issue.slicing_fn.meta.display_name, current_issues): - current_issues.append(issue.slicing_fn.meta.display_name) - filenames = ( - [dataset.get_image_path(int(idx)) for idx in indices[: self.num_images]] - if hasattr(dataset, "get_image_path") - else [] - ) - list_scan_results.append( - self.get_scan_result( - metric_value=current_data_slice.df["metric"].mean(), - metric_reference_value=giskard_dataset.df["metric"].mean(), - metric_name=self.metric.name, - filename_examples=filenames, - name=issue.slicing_fn.meta.display_name, - size_data=len(current_data_slice.df), - issue_group=meta.issue_group(issue.features[0]), + if issue.slicing_fn is not None: + current_data_slice = giskard_dataset.slice(issue.slicing_fn) + indices = list(current_data_slice.df.sort_values(by="metric", ascending=False)["index"].values) + if not self.check_slice_already_selected(issue.slicing_fn.meta.display_name, current_issues): + current_issues.append(issue.slicing_fn.meta.display_name) + filenames = ( + [dataset.get_image_path(int(idx)) for idx in indices[: self.num_images]] + if hasattr(dataset, "get_image_path") + else [] + ) + list_scan_results.append( + self.get_scan_result( + metric_value=current_data_slice.df["metric"].mean(), + metric_reference_value=giskard_dataset.df["metric"].mean(), + metric_name=self.metric.name, + filename_examples=filenames, + name=issue.slicing_fn.meta.display_name, + size_data=len(current_data_slice.df), + issue_group=meta.issue_group(issue.features[0]), + ) ) - ) return list_scan_results @@ -131,6 +132,8 @@ def get_giskard_results_from_surrogate(self, surrogate, model, df_for_scan, list prediction_function = self.get_prediction_function(surrogate, model, df_for_scan) # Create Giskard dataset and model, and get scan results + if list_categories is None: + list_categories = [] giskard_dataset = Dataset( df=df_for_scan, target=f"target_{surrogate.name}", cat_columns=list_categories + ["index"] ) @@ -140,7 +143,9 @@ def get_giskard_results_from_surrogate(self, surrogate, model, df_for_scan, list feature_names=list_metadata + ["index"], classification_labels=model.classification_labels if self.type_task == "classification" else None, ) - results = scan(giskard_model, giskard_dataset, max_issues_per_detector=None, verbose=False) + results = scan( + giskard_model, giskard_dataset, max_issues_per_detector=None, verbose=False, raise_exceptions=True + ) return giskard_dataset, results @@ -217,8 +222,9 @@ def get_df_for_scan(self, model: Any, dataset: Any, list_metadata: Sequence[str] # we need the metadata, labels and image path on an individual basis, # and sometimes the model may fail on an image. # TODO: make this cleaner and more efficient with batch computations + from tqdm import tqdm - for i in range(len(dataset)): + for i in tqdm(range(len(dataset))): try: metadata = dataset.get_meta(i) diff --git a/giskard_vision/object_detection/dataloaders/loaders.py b/giskard_vision/object_detection/dataloaders/loaders.py index 04b13390..b282fb0a 100644 --- a/giskard_vision/object_detection/dataloaders/loaders.py +++ b/giskard_vision/object_detection/dataloaders/loaders.py @@ -115,7 +115,7 @@ def get_meta(self, idx: int) -> MetaData | None: MetaData | None: Metadata associated with the image. """ meta_list = ["domain", "country", "location", "development_stage"] - data = {self.ds[idx][elt] for elt in meta_list} + data = {elt: self.ds[idx][elt] for elt in meta_list} return MetaData(data, categories=meta_list) diff --git a/giskard_vision/object_detection/detectors/__init__.py b/giskard_vision/object_detection/detectors/__init__.py new file mode 100644 index 00000000..839efc64 --- /dev/null +++ b/giskard_vision/object_detection/detectors/__init__.py @@ -0,0 +1,5 @@ +from .metadata_detector import MetaDataScanDetectorObjectDetection + +__all__ = [ + "MetaDataScanDetectorObjectDetection", +] diff --git a/giskard_vision/object_detection/detectors/metadata_detector.py b/giskard_vision/object_detection/detectors/metadata_detector.py new file mode 100644 index 00000000..1d4f02eb --- /dev/null +++ b/giskard_vision/object_detection/detectors/metadata_detector.py @@ -0,0 +1,46 @@ +from giskard_vision.core.detectors.metadata_scan_detector import MetaDataScanDetector +from giskard_vision.object_detection.detectors.surrogate_functions import ( + SurrogateArea, + SurrogateAspectRatio, + SurrogateCenterMassX, + SurrogateCenterMassY, + SurrogateDistanceFromCenter, + SurrogateMeanIntensity, + SurrogateNormalizedHeight, + SurrogateNormalizedPerimeter, + SurrogateNormalizedWidth, + SurrogateRelativeBottomRightX, + SurrogateRelativeBottomRightY, + SurrogateRelativeTopLeftX, + SurrogateRelativeTopLeftY, + SurrogateStdIntensity, +) +from giskard_vision.object_detection.tests.performance import IoU + +from ...core.detectors.decorator import maybe_detector + + +@maybe_detector("metadata_object_detection", tags=["vision", "object_detection", "metadata"]) +class MetaDataScanDetectorObjectDetection(MetaDataScanDetector): + surrogates = [ + SurrogateCenterMassX, + SurrogateCenterMassY, + SurrogateArea, + SurrogateAspectRatio, + SurrogateMeanIntensity, + SurrogateStdIntensity, + SurrogateNormalizedHeight, + SurrogateNormalizedWidth, + SurrogateDistanceFromCenter, + SurrogateRelativeBottomRightX, + SurrogateRelativeBottomRightY, + SurrogateRelativeTopLeftX, + SurrogateRelativeTopLeftY, + SurrogateNormalizedPerimeter, + ] + metric = IoU + type_task = "regression" + metric_type = "absolute" + metric_direction = "better_higher" + deviation_threshold = 0.10 + issue_level_threshold = 0.05 diff --git a/giskard_vision/object_detection/detectors/surrogate_functions.py b/giskard_vision/object_detection/detectors/surrogate_functions.py new file mode 100644 index 00000000..69732dab --- /dev/null +++ b/giskard_vision/object_detection/detectors/surrogate_functions.py @@ -0,0 +1,159 @@ +import numpy as np + +from giskard_vision.core.detectors.metadata_scan_detector import Surrogate + + +@staticmethod +def center_mass_x(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + center_x = (x_min + x_max) / 2 + return center_x / image.shape[0] + + +SurrogateCenterMassX = Surrogate("center_mass_x", center_mass_x) + + +@staticmethod +def center_mass_y(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + center_y = (y_min + y_max) / 2 + return center_y / image.shape[1] + + +SurrogateCenterMassY = Surrogate("center_mass_y", center_mass_y) + + +@staticmethod +def area(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + area = (x_max - x_min) * (y_max - y_min) + return area / (image.shape[0] * image.shape[1]) + + +SurrogateArea = Surrogate("area", area) + + +@staticmethod +def aspect_ratio(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + width = x_max - x_min + height = y_max - y_min + return width / height + + +SurrogateAspectRatio = Surrogate("aspect_ratio", aspect_ratio) + + +@staticmethod +def normalized_width(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + width = x_max - x_min + normalized_width = width / image.shape[1] + return normalized_width + + +SurrogateNormalizedWidth = Surrogate("normalized_width", normalized_width) + + +@staticmethod +def normalized_height(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + height = y_max - y_min + normalized_height = height / image.shape[0] + return normalized_height + + +SurrogateNormalizedHeight = Surrogate("normalized_height", normalized_height) + + +@staticmethod +def normalized_perimeter(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + width = x_max - x_min + height = y_max - y_min + perimeter = 2 * (width + height) + normalized_perimeter = perimeter / (2 * (image.shape[0] + image.shape[1])) + return normalized_perimeter + + +SurrogateNormalizedPerimeter = Surrogate("normalized_perimeter", normalized_perimeter) + + +@staticmethod +def relative_top_left_x(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + relative_x = x_min / float(image.shape[0]) + return relative_x + + +SurrogateRelativeTopLeftX = Surrogate("relative_top_left_x", relative_top_left_x) + + +@staticmethod +def relative_top_left_y(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + relative_y = y_min / float(image.shape[1]) + return relative_y + + +SurrogateRelativeTopLeftY = Surrogate("relative_top_left_y", relative_top_left_y) + + +@staticmethod +def relative_bottom_right_x(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + relative_x = x_max / float(image.shape[0]) + return relative_x + + +SurrogateRelativeBottomRightX = Surrogate("relative_bottom_right_x", relative_bottom_right_x) + + +@staticmethod +def relative_bottom_right_y(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + relative_y = y_max / float(image.shape[1]) + return relative_y + + +SurrogateRelativeBottomRightY = Surrogate("relative_bottom_right_y", relative_bottom_right_y) + + +@staticmethod +def distance_from_center(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + center_x = (x_min + x_max) / 2 + center_y = (y_min + y_max) / 2 + image_center_x = image.shape[1] / 2 + image_center_y = image.shape[0] / 2 + distance = np.sqrt((center_x - image_center_x) ** 2 + (center_y - image_center_y) ** 2) + return distance + + +SurrogateDistanceFromCenter = Surrogate("distance_from_center", distance_from_center) + + +@staticmethod +def mean_intensity(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + y_min = max(0, y_min) + x_min = max(0, x_min) + roi = image[int(y_min) : int(y_max), int(x_min) : int(x_max)] + mean_intensity = roi.mean() + return mean_intensity + + +SurrogateMeanIntensity = Surrogate("mean_intensity", mean_intensity) + + +@staticmethod +def std_intensity(result, image): + x_min, y_min, x_max, y_max = result[0]["boxes"] + y_min = max(0, y_min) + x_min = max(0, x_min) + roi = image[int(y_min) : int(y_max), int(x_min) : int(x_max)] + std_intensity = roi.std() + return std_intensity + + +SurrogateStdIntensity = Surrogate("std_intensity", std_intensity) diff --git a/giskard_vision/object_detection/models/wrappers.py b/giskard_vision/object_detection/models/wrappers.py index 4bc0c92b..431be41d 100644 --- a/giskard_vision/object_detection/models/wrappers.py +++ b/giskard_vision/object_detection/models/wrappers.py @@ -213,6 +213,7 @@ def preprocessing(self, image): class RacoonDetection(ModelBase): model_weights: str = "racoon_detection.h5" + model_type: str = "object_detection" image_size: int = 128 alpha: float = 1.0 diff --git a/giskard_vision/object_detection/tests/base.py b/giskard_vision/object_detection/tests/base.py new file mode 100644 index 00000000..e8007f8d --- /dev/null +++ b/giskard_vision/object_detection/tests/base.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass + +from giskard_vision.core.tests.base import MetricBase + +from ..types import Types + + +@dataclass +class Metric(MetricBase): + @classmethod + def validation(cls, prediction_result: Types.prediction_result, ground_truth: Types.label, **kwargs) -> None: + """Validate the input types for the metric calculation. + + Args: + prediction_result (Types.prediction_result): The prediction result to evaluate. + labels (Dict[str, Iterable[float]]): Ground truth for object detection. + + Raises: + ValueError: If the input types are incorrect. + + """ + pass diff --git a/giskard_vision/object_detection/tests/performance.py b/giskard_vision/object_detection/tests/performance.py new file mode 100644 index 00000000..933f6c67 --- /dev/null +++ b/giskard_vision/object_detection/tests/performance.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass + +from ..types import Types +from .base import Metric + + +@dataclass +class IoU(Metric): + """Intersection over Union distance between a prediction and a ground truth""" + + name = "IoU" + description = "Intersection over Union" + + @staticmethod + def definition(prediction_result: Types.prediction_result, ground_truth: Types.label): + + # if prediction_result.prediction.item().get("labels") != ground_truth.item().get("labels"): + # return 0 + + gt_box = prediction_result.prediction.item().get("boxes") + pred_box = ground_truth.item().get("boxes") + + x1_min, y1_min, x1_max, y1_max = gt_box + x2_min, y2_min, x2_max, y2_max = pred_box + + # Calculate the coordinates of the intersection rectangle + x_inter_min = max(x1_min, x2_min) + y_inter_min = max(y1_min, y2_min) + x_inter_max = min(x1_max, x2_max) + y_inter_max = min(y1_max, y2_max) + + # Compute the area of the intersection rectangle + if x_inter_max < x_inter_min or y_inter_max < y_inter_min: + inter_area = 0 + else: + inter_area = (x_inter_max - x_inter_min) * (y_inter_max - y_inter_min) + + # Compute the area of both the prediction and ground-truth rectangles + box1_area = (x1_max - x1_min) * (y1_max - y1_min) + box2_area = (x2_max - x2_min) * (y2_max - y2_min) + + # Compute the union area + union_area = box1_area + box2_area - inter_area + + # Compute the IoU + iou = inter_area / union_area + + return iou