Scan for object detection

giskard_vision/core/detectors/metadata_scan_detector.py

@@ -217,8 +217,10 @@ def get_df_for_scan(self, model: Any, dataset: Any, list_metadata: Sequence[str]
  # we need the metadata, labels and image path on an individual basis,
  # and sometimes the model may fail on an image.
  # TODO: make this cleaner and more efficient with batch computations
+ from tqdm import tqdm
 
- for i in range(len(dataset)):
+ total = min(len(dataset), 200)
+ for i in tqdm(range(total)):
  try:
  metadata = dataset.get_meta(i)
 

giskard_vision/object_detection/dataloaders/loaders.py

@@ -109,6 +109,6 @@ def get_meta(self, idx: int) -> MetaData | None:
  MetaData | None: Metadata associated with the image.
  """
  meta_list = ["domain", "country", "location", "development_stage"]
- data = {self.ds[idx][elt] for elt in meta_list}
+ data = {elt: self.ds[idx][elt] for elt in meta_list}
 
  return MetaData(data, categories=meta_list)

giskard_vision/object_detection/detectors/__init__.py

@@ -0,0 +1,5 @@
+from .metadata_detector import MetaDataScanDetectorObjectDetection
+
+__all__ = [
+ "MetaDataScanDetectorObjectDetection",
+]

giskard_vision/object_detection/detectors/metadata_detector.py

@@ -0,0 +1,46 @@
+from giskard_vision.core.detectors.metadata_scan_detector import MetaDataScanDetector
+from giskard_vision.object_detection.detectors.surrogate_functions import (
+ SurrogateArea,
+ SurrogateAspectRatio,
+ SurrogateCenterMassX,
+ SurrogateCenterMassY,
+ SurrogateDistanceFromCenter,
+ SurrogateMeanIntensity,
+ SurrogateNormalizedHeight,
+ SurrogateNormalizedPerimeter,
+ SurrogateNormalizedWidth,
+ SurrogateRelativeBottomRightX,
+ SurrogateRelativeBottomRightY,
+ SurrogateRelativeTopLeftX,
+ SurrogateRelativeTopLeftY,
+ SurrogateStdIntensity,
+)
+from giskard_vision.object_detection.tests.performance import IoU
+
+from ...core.detectors.decorator import maybe_detector
+
+
+@maybe_detector("metadata_object_detection", tags=["vision", "object_detection", "metadata"])
+class MetaDataScanDetectorObjectDetection(MetaDataScanDetector):
+ surrogates = [
+ SurrogateCenterMassX,
+ SurrogateCenterMassY,
+ SurrogateArea,
+ SurrogateAspectRatio,
+ SurrogateMeanIntensity,
+ SurrogateStdIntensity,
+ SurrogateNormalizedHeight,
+ SurrogateNormalizedWidth,
+ SurrogateDistanceFromCenter,
+ SurrogateRelativeBottomRightX,
+ SurrogateRelativeBottomRightY,
+ SurrogateRelativeTopLeftX,
+ SurrogateRelativeTopLeftY,
+ SurrogateNormalizedPerimeter,
+ ]
+ metric = IoU
+ type_task = "regression"
+ metric_type = "absolute"
+ metric_direction = "better_higher"
+ deviation_threshold = 0.10
+ issue_level_threshold = 0.05

giskard_vision/object_detection/detectors/surrogate_functions.py

@@ -0,0 +1,155 @@
+import numpy as np
+
+from giskard_vision.core.detectors.metadata_scan_detector import Surrogate
+
+
+@staticmethod
+def center_mass_x(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ center_x = (x_min + x_max) / 2
+ return center_x / image.shape[0]
+
+
+SurrogateCenterMassX = Surrogate("center_mass_x", center_mass_x)
+
+
+@staticmethod
+def center_mass_y(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ center_y = (y_min + y_max) / 2
+ return center_y / image.shape[1]
+
+
+SurrogateCenterMassY = Surrogate("center_mass_y", center_mass_y)
+
+
+@staticmethod
+def area(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ area = (x_max - x_min) * (y_max - y_min)
+ return area / (image.shape[0] * image.shape[1])
+
+
+SurrogateArea = Surrogate("area", area)
+
+
+@staticmethod
+def aspect_ratio(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ width = x_max - x_min
+ height = y_max - y_min
+ return width / height
+
+
+SurrogateAspectRatio = Surrogate("aspect_ratio", aspect_ratio)
+
+
+@staticmethod
+def normalized_width(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ width = x_max - x_min
+ normalized_width = width / image.shape[1]
+ return normalized_width
+
+
+SurrogateNormalizedWidth = Surrogate("normalized_width", normalized_width)
+
+
+@staticmethod
+def normalized_height(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ height = y_max - y_min
+ normalized_height = height / image.shape[0]
+ return normalized_height
+
+
+SurrogateNormalizedHeight = Surrogate("normalized_height", normalized_height)
+
+
+@staticmethod
+def normalized_perimeter(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ width = x_max - x_min
+ height = y_max - y_min
+ perimeter = 2 * (width + height)
+ normalized_perimeter = perimeter / (2 * (image.shape[0] + image.shape[1]))
+ return normalized_perimeter
+
+
+SurrogateNormalizedPerimeter = Surrogate("normalized_perimeter", normalized_perimeter)
+
+
+@staticmethod
+def relative_top_left_x(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ relative_x = x_min / float(image.shape[0])
+ return relative_x
+
+
+SurrogateRelativeTopLeftX = Surrogate("relative_top_left_x", relative_top_left_x)
+
+
+@staticmethod
+def relative_top_left_y(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ relative_y = y_min / float(image.shape[1])
+ return relative_y
+
+
+SurrogateRelativeTopLeftY = Surrogate("relative_top_left_y", relative_top_left_y)
+
+
+@staticmethod
+def relative_bottom_right_x(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ relative_x = x_max / float(image.shape[0])
+ return relative_x
+
+
+SurrogateRelativeBottomRightX = Surrogate("relative_bottom_right_x", relative_bottom_right_x)
+
+
+@staticmethod
+def relative_bottom_right_y(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ relative_y = y_max / float(image.shape[1])
+ return relative_y
+
+
+SurrogateRelativeBottomRightY = Surrogate("relative_bottom_right_y", relative_bottom_right_y)
+
+
+@staticmethod
+def distance_from_center(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ center_x = (x_min + x_max) / 2
+ center_y = (y_min + y_max) / 2
+ image_center_x = image.shape[1] / 2
+ image_center_y = image.shape[0] / 2
+ distance = np.sqrt((center_x - image_center_x) ** 2 + (center_y - image_center_y) ** 2)
+ return distance
+
+
+SurrogateDistanceFromCenter = Surrogate("distance_from_center", distance_from_center)
+
+
+@staticmethod
+def mean_intensity(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ roi = image[int(y_min) : int(y_max), int(x_min) : int(x_max)]
+ mean_intensity = roi.mean()
+ return mean_intensity
+
+
+SurrogateMeanIntensity = Surrogate("mean_intensity", mean_intensity)
+
+
+@staticmethod
+def std_intensity(result, image):
+ x_min, y_min, x_max, y_max = result[0]["boxes"]
+ roi = image[int(y_min) : int(y_max), int(x_min) : int(x_max)]
+ std_intensity = roi.std()
+ return std_intensity
+
+
+SurrogateStdIntensity = Surrogate("std_intensity", std_intensity)

giskard_vision/object_detection/tests/base.py

@@ -0,0 +1,22 @@
+from dataclasses import dataclass
+
+from giskard_vision.core.tests.base import MetricBase
+
+from ..types import Types
+
+
+@dataclass
+class Metric(MetricBase):
+ @classmethod
+ def validation(cls, prediction_result: Types.prediction_result, ground_truth: Types.label, **kwargs) -> None:
+ """Validate the input types for the metric calculation.
+
+ Args:
+ prediction_result (Types.prediction_result): The prediction result to evaluate.
+ labels (Dict[str, Iterable[float]]): Ground truth for object detection.
+
+ Raises:
+ ValueError: If the input types are incorrect.
+
+ """
+ pass

giskard_vision/object_detection/tests/performance.py

@@ -0,0 +1,48 @@
+from dataclasses import dataclass
+
+from ..types import Types
+from .base import Metric
+
+
+@dataclass
+class IoU(Metric):
+ """Intersection over Union distance between a prediction and a ground truth"""
+
+ name = "IoU"
+ description = "Intersection over Union"
+
+ @staticmethod
+ def definition(prediction_result: Types.prediction_result, ground_truth: Types.label):
+
+ if prediction_result.prediction.item().get("labels") != ground_truth.item().get("labels"):
+ return 0
+
+ gt_box = prediction_result.prediction.item().get("boxes")
+ pred_box = ground_truth.item().get("boxes")
+
+ x1_min, y1_min, x1_max, y1_max = gt_box
+ x2_min, y2_min, x2_max, y2_max = pred_box
+
+ # Calculate the coordinates of the intersection rectangle
+ x_inter_min = max(x1_min, x2_min)
+ y_inter_min = max(y1_min, y2_min)
+ x_inter_max = min(x1_max, x2_max)
+ y_inter_max = min(y1_max, y2_max)
+
+ # Compute the area of the intersection rectangle
+ if x_inter_max < x_inter_min or y_inter_max < y_inter_min:
+ inter_area = 0
+ else:
+ inter_area = (x_inter_max - x_inter_min) * (y_inter_max - y_inter_min)
+
+ # Compute the area of both the prediction and ground-truth rectangles
+ box1_area = (x1_max - x1_min) * (y1_max - y1_min)
+ box2_area = (x2_max - x2_min) * (y2_max - y2_min)
+
+ # Compute the union area
+ union_area = box1_area + box2_area - inter_area
+
+ # Compute the IoU
+ iou = inter_area / union_area
+
+ return iou