Get basic image metadata from image properties (#49)

* Add utilities to get image metadata

* Add meta auto based on basic meta data of images

* Refactor HF dataset to always have numpy array instead of PIL image

* Refactor HF ppl model to convert numpy array to PIL image

* Allow to set global mode for an HF ppl model for PIL conversion

* Correct height and width in np array

* Add brightness

* Add average color and contrast

* Add entropy

* Fix test due to missing raw HF image in PIL format

* Adapt object detection models with image mode to convert PIL image

* Rewrite `get_meta` to concatenate basic metadata

* Fix import of issues

* Fix name masking in ffhq

* Expand dicts in ffhq face detection

* Should database be integer instead of float?

* Use uint8 for image numpy test data

* Convert and assume RGB uint8 image as input

* Fix cached dataloader counter after calling `get_image` in `get_meta`

* Unify issue groups

* Save images from `get_raw_hf_image` to get image path in HF

* Recover issue meta groups as in `main`

* Clarify the dataloader access time changes

* Use AttributesIssueMeta for basic meta

* Use AttributesIssueMeta for depth basic meta

* Fix format

* Fix the issue group of brightness

---------

Co-authored-by: Rabah Khalek <[email protected]>

Author: Inokinoki

giskard_vision/core/dataloaders/base.py

@@ -2,9 +2,16 @@
 from abc import ABC, abstractmethod
 from typing import List, Optional
 
+import cv2
 import numpy as np
 
-from giskard_vision.core.dataloaders.meta import MetaData
+from giskard_vision.core.dataloaders.meta import (
+    MetaData,
+    get_brightness,
+    get_entropy,
+    get_image_channel_number,
+    get_image_size,
+)
 from giskard_vision.core.detectors.base import IssueGroup
 
 from ..types import TypesBase
@@ -17,6 +24,10 @@
     "Performance",
     description="The data are filtered by metadata like emotion, head pose, or exposure value to detect performance issues.",
 )
+AttributesIssueMeta = IssueGroup(
+    "Attributes",
+    description="The data are filtered by the image attributes like width, height, or brightness value to detect issues.",
+)
 
 
 class DataIteratorBase(ABC):
@@ -120,7 +131,7 @@ def meta_none(self) -> Optional[TypesBase.meta]:
         Returns default for meta data if it is None.
 
         Returns:
-            Optional[np.ndarray]: Default for meta data.
+            Optional[TypesBase.meta]: Default for meta data.
         """
         return None
 
@@ -146,7 +157,41 @@ def get_meta(self, idx: int) -> Optional[TypesBase.meta]:
         Returns:
             Optional[TypesBase.meta]: Meta information for the given index.
         """
-        return None
+        img = self.get_image(idx)
+        if img.dtype != np.uint8:
+            # Normalize image to 0-255 range with uint8
+            img = (img * 255 % 255).astype(np.uint8)
+
+        gray_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
+        size = get_image_size(img)
+        nb_channels = get_image_channel_number(img)
+        avg_color = np.mean(img, axis=(0, 1))
+
+        return MetaData(
+            data={
+                "height": size[0],
+                "width": size[1],
+                "nb_channels": nb_channels,
+                "brightness": get_brightness(img),
+                "average_color_r": avg_color[0],
+                "average_color_g": avg_color[1] if avg_color.shape[0] > 0 else avg_color[0],
+                "average_color_b": avg_color[2] if avg_color.shape[0] > 0 else avg_color[0],
+                "contrast": np.max(gray_img) - np.min(gray_img),
+                "entropy": get_entropy(img),
+            },
+            categories=["nb_channels"],
+            issue_groups={
+                "width": AttributesIssueMeta,
+                "height": AttributesIssueMeta,
+                "nb_channels": AttributesIssueMeta,
+                "brightness": AttributesIssueMeta,
+                "average_color_r": AttributesIssueMeta,
+                "average_color_g": AttributesIssueMeta,
+                "average_color_b": AttributesIssueMeta,
+                "contrast": AttributesIssueMeta,
+                "entropy": AttributesIssueMeta,
+            },
+        )
 
     def get_labels_with_default(self, idx: int) -> np.ndarray:
         """

giskard_vision/core/dataloaders/hf.py

@@ -2,9 +2,13 @@
 import os
 import shutil
 import tempfile
+from abc import abstractmethod
 from typing import Optional
 
-from giskard_vision.core.dataloaders.base import DataIteratorBase
+from PIL.Image import Image as PILImage
+
+from giskard_vision.core.dataloaders.base import AttributesIssueMeta, DataIteratorBase
+from giskard_vision.core.dataloaders.meta import MetaData, get_pil_image_depth
 from giskard_vision.utils.errors import GiskardError, GiskardImportError
 
 
@@ -31,7 +35,11 @@ class HFDataLoader(DataIteratorBase):
     """
 
     def __init__(
-        self, hf_id: str, hf_config: Optional[str] = None, hf_split: str = "test", name: Optional[str] = None
+        self,
+        hf_id: str,
+        hf_config: Optional[str] = None,
+        hf_split: str = "test",
+        name: Optional[str] = None,
     ) -> None:
         """
         Initializes the general HuggingFace Datasets instance.
@@ -90,7 +98,7 @@ def get_image_path(self, idx: int) -> str:
             str: Image path
         """
 
-        image = self.ds[idx]["image"]
+        image = self.get_raw_hf_image(idx)
         image_path = os.path.join(self.temp_folder, f"image_{idx}.png")
         image.save(image_path)
 
@@ -101,3 +109,31 @@ def cleanup(self):
         Clean the temporary folder
         """
         shutil.rmtree(self.temp_folder)
+
+    @abstractmethod
+    def get_raw_hf_image(self, idx: int) -> PILImage:
+        """
+        Retrieves the raw image at the specified index in the HF dataset.
+        Args:
+            idx (int): Index of the image
+
+        Returns:
+            PIL.Image.Image: The image instance.
+        """
+        ...
+
+    def get_meta(self, idx: int) -> MetaData:
+        meta = super().get_meta(idx)
+        img = self.get_raw_hf_image(idx)
+
+        return MetaData(
+            data={
+                **meta.data,
+                "depth": get_pil_image_depth(img),
+            },
+            categories=["depth"] + meta.categories,
+            issue_groups={
+                **meta.issue_groups,
+                "depth": AttributesIssueMeta,
+            },
+        )

giskard_vision/core/dataloaders/meta.py

@@ -1,4 +1,8 @@
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
+
+import cv2
+import numpy as np
+from PIL.Image import Image as PILImage
 
 from giskard_vision.core.detectors.base import IssueGroup
 
@@ -140,3 +144,96 @@ def get_categories(self) -> Optional[List[str]]:
             Optional[List[str]]: The categories of the metadata, or None if no categories were provided.
         """
         return self.categories
+
+
+def get_image_size(image: np.ndarray) -> Tuple[int, int]:
+    """
+    Utitlity to create metadata with image size.
+
+    Args:
+        image (np.ndarray): The numpy ndarray representation of an image.
+
+    Returns:
+        Tuple[int, int]: The image size, width and height.
+    """
+    return image.shape[:2]
+
+
+def get_image_channel_number(image: np.ndarray) -> int:
+    """
+    Utitlity to create metadata with image channel number.
+
+    Args:
+        image (np.ndarray): The numpy ndarray representation of an image.
+
+    Returns:
+        int: The image channel number.
+    """
+    shape = image.shape
+    return shape[2] if len(shape) > 2 else 1
+
+
+def get_pil_image_depth(image: PILImage) -> int:
+    """
+    Utitlity to create metadata with image depth.
+
+    Args:
+        image (PILImage): The PIL Image object.
+
+    Returns:
+        int: The image depth.
+    """
+    mode = image.mode
+    if mode == "1":
+        return 1
+    elif mode == "L":
+        return 8
+    elif mode == "P":
+        return 8
+    elif mode == "RGB":
+        return 24
+    elif mode == "RGBA":
+        return 32
+    elif mode == "CMYK":
+        return 32
+    elif mode == "YCbCr":
+        return 24
+    elif mode == "LAB":
+        return 24
+    elif mode == "HSV":
+        return 24
+    elif mode == "I":
+        return 32
+    elif mode == "F":
+        return 32
+    return 0
+
+
+def get_brightness(image: np.ndarray) -> float:
+    """
+    Utitlity to create metadata with image brightness.
+
+    Args:
+        image (np.ndarray): The numpy ndarray representation of an image.
+
+    Returns:
+        float: The image brightness normalized to 1.
+    """
+    hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
+    return np.mean(hsv[:, :, 2]) / 255
+
+
+def get_entropy(image: np.ndarray) -> float:
+    """
+    Utitlity to create metadata with image entropy.
+
+    Args:
+        image (np.ndarray): The numpy ndarray representation of an image.
+
+    Returns:
+        float: The image entropy.
+    """
+    hist = cv2.calcHist([image], [0], None, [256], [0, 256])
+    hist /= hist.sum()
+    # Add eps to avoid log(0)
+    return -np.sum(hist * np.log2(hist + np.finfo(float).eps))

giskard_vision/core/detectors/metadata_scan_detector.py

@@ -4,11 +4,8 @@
 import numpy as np
 import pandas as pd
 
-from giskard_vision.core.detectors.base import (
-    DetectorVisionBase,
-    IssueGroup,
-    ScanResult,
-)
+from giskard_vision.core.dataloaders.base import PerformanceIssueMeta
+from giskard_vision.core.detectors.base import DetectorVisionBase, ScanResult
 from giskard_vision.core.tests.base import MetricBase
 from giskard_vision.utils.errors import GiskardImportError
 
@@ -46,9 +43,7 @@ class MetaDataScanDetector(DetectorVisionBase):
     metric: MetricBase = None
     metric_type: str = None
     metric_direction: str = "better_lower"
-    issue_group = IssueGroup(
-        name="Performance", description="The data are filtered by metadata to detect performance issues."
-    )
+    issue_group = PerformanceIssueMeta
 
     def __init__(self) -> None:
         super().__init__()

giskard_vision/core/models/hf_pipeline.py

@@ -19,9 +19,9 @@ def __init__(
         """init method that accepts a model object, number of landmarks and dimensions
 
         Args:
-            model_id (str): Hugging Face model ID
-            name (Optional[str]): name of the model
-            pipeline_task (HFPipelineTask): HuggingFace pipeline task
+            model_id (str): Hugging Face model ID.
+            name (Optional[str]): name of the model.
+            pipeline_task (HFPipelineTask): HuggingFace pipeline task.
 
         Raises:
             GiskardImportError: If there are missing Hugging Face dependencies.