Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Scan for object detection #46

Merged
merged 7 commits into from
Aug 5, 2024
48 changes: 27 additions & 21 deletions giskard_vision/core/detectors/metadata_scan_detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,26 +85,27 @@ def get_results(self, model: Any, dataset: Any) -> List[ScanResult]:

# For each slice found, get appropriate scan results with the metric
for issue in results.issues:
current_data_slice = giskard_dataset.slice(issue.slicing_fn)
indices = list(current_data_slice.df.sort_values(by="metric", ascending=False)["index"].values)
if not self.check_slice_already_selected(issue.slicing_fn.meta.display_name, current_issues):
current_issues.append(issue.slicing_fn.meta.display_name)
filenames = (
[dataset.get_image_path(int(idx)) for idx in indices[: self.num_images]]
if hasattr(dataset, "get_image_path")
else []
)
list_scan_results.append(
self.get_scan_result(
metric_value=current_data_slice.df["metric"].mean(),
metric_reference_value=giskard_dataset.df["metric"].mean(),
metric_name=self.metric.name,
filename_examples=filenames,
name=issue.slicing_fn.meta.display_name,
size_data=len(current_data_slice.df),
issue_group=meta.issue_group(issue.features[0]),
if issue.slicing_fn is not None:
current_data_slice = giskard_dataset.slice(issue.slicing_fn)
indices = list(current_data_slice.df.sort_values(by="metric", ascending=False)["index"].values)
if not self.check_slice_already_selected(issue.slicing_fn.meta.display_name, current_issues):
current_issues.append(issue.slicing_fn.meta.display_name)
filenames = (
[dataset.get_image_path(int(idx)) for idx in indices[: self.num_images]]
if hasattr(dataset, "get_image_path")
else []
)
list_scan_results.append(
self.get_scan_result(
metric_value=current_data_slice.df["metric"].mean(),
metric_reference_value=giskard_dataset.df["metric"].mean(),
metric_name=self.metric.name,
filename_examples=filenames,
name=issue.slicing_fn.meta.display_name,
size_data=len(current_data_slice.df),
issue_group=meta.issue_group(issue.features[0]),
)
)
)

return list_scan_results

Expand All @@ -131,6 +132,8 @@ def get_giskard_results_from_surrogate(self, surrogate, model, df_for_scan, list
prediction_function = self.get_prediction_function(surrogate, model, df_for_scan)

# Create Giskard dataset and model, and get scan results
if list_categories is None:
list_categories = []
giskard_dataset = Dataset(
df=df_for_scan, target=f"target_{surrogate.name}", cat_columns=list_categories + ["index"]
)
Expand All @@ -140,7 +143,9 @@ def get_giskard_results_from_surrogate(self, surrogate, model, df_for_scan, list
feature_names=list_metadata + ["index"],
classification_labels=model.classification_labels if self.type_task == "classification" else None,
)
results = scan(giskard_model, giskard_dataset, max_issues_per_detector=None, verbose=False)
results = scan(
giskard_model, giskard_dataset, max_issues_per_detector=None, verbose=False, raise_exceptions=True
)

return giskard_dataset, results

Expand Down Expand Up @@ -217,8 +222,9 @@ def get_df_for_scan(self, model: Any, dataset: Any, list_metadata: Sequence[str]
# we need the metadata, labels and image path on an individual basis,
# and sometimes the model may fail on an image.
# TODO: make this cleaner and more efficient with batch computations
from tqdm import tqdm

for i in range(len(dataset)):
for i in tqdm(range(len(dataset))):
try:
metadata = dataset.get_meta(i)

Expand Down
2 changes: 1 addition & 1 deletion giskard_vision/object_detection/dataloaders/loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ def get_meta(self, idx: int) -> MetaData | None:
MetaData | None: Metadata associated with the image.
"""
meta_list = ["domain", "country", "location", "development_stage"]
data = {self.ds[idx][elt] for elt in meta_list}
data = {elt: self.ds[idx][elt] for elt in meta_list}

return MetaData(data, categories=meta_list)

Expand Down
5 changes: 5 additions & 0 deletions giskard_vision/object_detection/detectors/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
from .metadata_detector import MetaDataScanDetectorObjectDetection

__all__ = [
"MetaDataScanDetectorObjectDetection",
]
46 changes: 46 additions & 0 deletions giskard_vision/object_detection/detectors/metadata_detector.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
from giskard_vision.core.detectors.metadata_scan_detector import MetaDataScanDetector
from giskard_vision.object_detection.detectors.surrogate_functions import (
SurrogateArea,
SurrogateAspectRatio,
SurrogateCenterMassX,
SurrogateCenterMassY,
SurrogateDistanceFromCenter,
SurrogateMeanIntensity,
SurrogateNormalizedHeight,
SurrogateNormalizedPerimeter,
SurrogateNormalizedWidth,
SurrogateRelativeBottomRightX,
SurrogateRelativeBottomRightY,
SurrogateRelativeTopLeftX,
SurrogateRelativeTopLeftY,
SurrogateStdIntensity,
)
from giskard_vision.object_detection.tests.performance import IoU

from ...core.detectors.decorator import maybe_detector


@maybe_detector("metadata_object_detection", tags=["vision", "object_detection", "metadata"])
class MetaDataScanDetectorObjectDetection(MetaDataScanDetector):
surrogates = [
SurrogateCenterMassX,
SurrogateCenterMassY,
SurrogateArea,
SurrogateAspectRatio,
SurrogateMeanIntensity,
SurrogateStdIntensity,
SurrogateNormalizedHeight,
SurrogateNormalizedWidth,
SurrogateDistanceFromCenter,
SurrogateRelativeBottomRightX,
SurrogateRelativeBottomRightY,
SurrogateRelativeTopLeftX,
SurrogateRelativeTopLeftY,
SurrogateNormalizedPerimeter,
]
metric = IoU
type_task = "regression"
metric_type = "absolute"
metric_direction = "better_higher"
deviation_threshold = 0.10
issue_level_threshold = 0.05
159 changes: 159 additions & 0 deletions giskard_vision/object_detection/detectors/surrogate_functions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
import numpy as np

from giskard_vision.core.detectors.metadata_scan_detector import Surrogate


@staticmethod
def center_mass_x(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
center_x = (x_min + x_max) / 2
return center_x / image.shape[0]


SurrogateCenterMassX = Surrogate("center_mass_x", center_mass_x)


@staticmethod
def center_mass_y(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
center_y = (y_min + y_max) / 2
return center_y / image.shape[1]


SurrogateCenterMassY = Surrogate("center_mass_y", center_mass_y)


@staticmethod
def area(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
area = (x_max - x_min) * (y_max - y_min)
return area / (image.shape[0] * image.shape[1])


SurrogateArea = Surrogate("area", area)


@staticmethod
def aspect_ratio(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
width = x_max - x_min
height = y_max - y_min
return width / height


SurrogateAspectRatio = Surrogate("aspect_ratio", aspect_ratio)


@staticmethod
def normalized_width(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
width = x_max - x_min
normalized_width = width / image.shape[1]
return normalized_width


SurrogateNormalizedWidth = Surrogate("normalized_width", normalized_width)


@staticmethod
def normalized_height(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
height = y_max - y_min
normalized_height = height / image.shape[0]
return normalized_height


SurrogateNormalizedHeight = Surrogate("normalized_height", normalized_height)


@staticmethod
def normalized_perimeter(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
width = x_max - x_min
height = y_max - y_min
perimeter = 2 * (width + height)
normalized_perimeter = perimeter / (2 * (image.shape[0] + image.shape[1]))
return normalized_perimeter


SurrogateNormalizedPerimeter = Surrogate("normalized_perimeter", normalized_perimeter)


@staticmethod
def relative_top_left_x(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
relative_x = x_min / float(image.shape[0])
return relative_x


SurrogateRelativeTopLeftX = Surrogate("relative_top_left_x", relative_top_left_x)


@staticmethod
def relative_top_left_y(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
relative_y = y_min / float(image.shape[1])
return relative_y


SurrogateRelativeTopLeftY = Surrogate("relative_top_left_y", relative_top_left_y)


@staticmethod
def relative_bottom_right_x(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
relative_x = x_max / float(image.shape[0])
return relative_x


SurrogateRelativeBottomRightX = Surrogate("relative_bottom_right_x", relative_bottom_right_x)


@staticmethod
def relative_bottom_right_y(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
relative_y = y_max / float(image.shape[1])
return relative_y


SurrogateRelativeBottomRightY = Surrogate("relative_bottom_right_y", relative_bottom_right_y)


@staticmethod
def distance_from_center(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
center_x = (x_min + x_max) / 2
center_y = (y_min + y_max) / 2
image_center_x = image.shape[1] / 2
image_center_y = image.shape[0] / 2
distance = np.sqrt((center_x - image_center_x) ** 2 + (center_y - image_center_y) ** 2)
return distance


SurrogateDistanceFromCenter = Surrogate("distance_from_center", distance_from_center)


@staticmethod
def mean_intensity(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
y_min = max(0, y_min)
x_min = max(0, x_min)
roi = image[int(y_min) : int(y_max), int(x_min) : int(x_max)]
mean_intensity = roi.mean()
return mean_intensity


SurrogateMeanIntensity = Surrogate("mean_intensity", mean_intensity)


@staticmethod
def std_intensity(result, image):
x_min, y_min, x_max, y_max = result[0]["boxes"]
y_min = max(0, y_min)
x_min = max(0, x_min)
roi = image[int(y_min) : int(y_max), int(x_min) : int(x_max)]
std_intensity = roi.std()
return std_intensity


SurrogateStdIntensity = Surrogate("std_intensity", std_intensity)
1 change: 1 addition & 0 deletions giskard_vision/object_detection/models/wrappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ def preprocessing(self, image):

class RacoonDetection(ModelBase):
model_weights: str = "racoon_detection.h5"
model_type: str = "object_detection"
image_size: int = 128
alpha: float = 1.0

Expand Down
22 changes: 22 additions & 0 deletions giskard_vision/object_detection/tests/base.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
from dataclasses import dataclass

from giskard_vision.core.tests.base import MetricBase

from ..types import Types


@dataclass
class Metric(MetricBase):
@classmethod
def validation(cls, prediction_result: Types.prediction_result, ground_truth: Types.label, **kwargs) -> None:
"""Validate the input types for the metric calculation.

Args:
prediction_result (Types.prediction_result): The prediction result to evaluate.
labels (Dict[str, Iterable[float]]): Ground truth for object detection.

Raises:
ValueError: If the input types are incorrect.

"""
pass
Comment on lines +11 to +22
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tbd

48 changes: 48 additions & 0 deletions giskard_vision/object_detection/tests/performance.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
from dataclasses import dataclass

from ..types import Types
from .base import Metric


@dataclass
class IoU(Metric):
"""Intersection over Union distance between a prediction and a ground truth"""

name = "IoU"
description = "Intersection over Union"

@staticmethod
def definition(prediction_result: Types.prediction_result, ground_truth: Types.label):

# if prediction_result.prediction.item().get("labels") != ground_truth.item().get("labels"):
# return 0

gt_box = prediction_result.prediction.item().get("boxes")
pred_box = ground_truth.item().get("boxes")

x1_min, y1_min, x1_max, y1_max = gt_box
x2_min, y2_min, x2_max, y2_max = pred_box

# Calculate the coordinates of the intersection rectangle
x_inter_min = max(x1_min, x2_min)
y_inter_min = max(y1_min, y2_min)
x_inter_max = min(x1_max, x2_max)
y_inter_max = min(y1_max, y2_max)

# Compute the area of the intersection rectangle
if x_inter_max < x_inter_min or y_inter_max < y_inter_min:
inter_area = 0
else:
inter_area = (x_inter_max - x_inter_min) * (y_inter_max - y_inter_min)

# Compute the area of both the prediction and ground-truth rectangles
box1_area = (x1_max - x1_min) * (y1_max - y1_min)
box2_area = (x2_max - x2_min) * (y2_max - y2_min)

# Compute the union area
union_area = box1_area + box2_area - inter_area

# Compute the IoU
iou = inter_area / union_area

return iou
Loading