1. updated the rule in the evaluation of different sized regions

nico-zck · nico-zck · commit a2a2b4ea357c · 2023-03-13T20:28:54.000+08:00
(🔥Make sure you are using the same version of the protocol when you compare detection performance with other methods on different size regions🔥);
2. updated sample threshold with pixel ratio;
3. updated configure of dataset path;
4. unified all metric names by "F1";
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
+saved_models/  # exclude model checkpoints
+
 # Created by .ignore support plugin (hsz.mobi)
 ### Python template
 # Byte-compiled / optimized / DLL files
diff --git a/baselines/libs/configurer/DEFAULT.yaml b/baselines/libs/configurer/DEFAULT.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: all
   normalization: zscore
   color_mode: rgb
diff --git a/baselines/libs/dataset/base_datasets.py b/baselines/libs/dataset/base_datasets.py
@@ -1,13 +1,12 @@
-from typing import List
-
 import numpy as np
 import torch
 from PIL import Image, ImageOps
 from numpy.lib.stride_tricks import as_strided
 from torch.utils.data import Dataset
 from torchvision import transforms
+from typing import List
 
-from ..dataset_api.zl_fabric import ZLImage
+from dataset_api.zl_fabric import ZLImage
 
 '''
 FabricFinal RGB
diff --git a/baselines/libs/evaluator/evaluator.py b/baselines/libs/evaluator/evaluator.py
@@ -5,12 +5,11 @@
 from collections import OrderedDict
 from datetime import datetime
 from math import isnan
-
 from pandas import DataFrame
 from tensorboardX import SummaryWriter
 
+from dataset_api import ZLEval
 from ..configurer import Configurer
-from ..dataset_api import ZLEval
 
 
 class _Evaluator(ABC):
diff --git a/baselines/models/seg_augment/unet_aug/unet-aug.yaml b/baselines/models/seg_augment/unet_aug/unet-aug.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: all
   normalization: '[0,1]'
   color_mode: rgb
diff --git a/baselines/models/seg_fully/unet/unet-defect-group.yaml b/baselines/models/seg_fully/unet/unet-defect-group.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: groups
   normalization: 'zscore'
   color_mode: rgb
diff --git a/baselines/models/seg_fully/unet/unet-defect-total.yaml b/baselines/models/seg_fully/unet/unet-defect-total.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: total
   normalization: 'zscore'
   color_mode: rgb
diff --git a/baselines/models/seg_fully/unet/unet-defect.yaml b/baselines/models/seg_fully/unet/unet-defect.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: all
   normalization: 'zscore'
   color_mode: rgb
diff --git a/baselines/models/seg_fully/unet/unet-small.yaml b/baselines/models/seg_fully/unet/unet-small.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: all
   normalization: 'zscore'
   color_mode: rgb
diff --git a/baselines/models/seg_one_class/cae/cae.yaml b/baselines/models/seg_one_class/cae/cae.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: all
   color_mode: gray
   normalization: '[-1,1]'
diff --git a/baselines/models/seg_one_class/ocsvm_feat/ocsvm_feat.yaml b/baselines/models/seg_one_class/ocsvm_feat/ocsvm_feat.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: all
   color_mode: gray
   normalization: '[-1,1]'
diff --git a/baselines/models/seg_one_class/sc/sc.yaml b/baselines/models/seg_one_class/sc/sc.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: all
   color_mode: gray
   normalization: histeq
diff --git a/baselines/models/seg_transfer/unet_unsup/unet-unsup.yaml b/baselines/models/seg_transfer/unet_unsup/unet-unsup.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: all
   normalization: 'zscore'
   color_mode: rgb
diff --git a/baselines/models/seg_weakly/bbox/unet_bbox.yaml b/baselines/models/seg_weakly/bbox/unet_bbox.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: all
   normalization: 'zscore'
   color_mode: rgb
diff --git a/baselines/models/seg_weakly/label/unet_label.yaml b/baselines/models/seg_weakly/label/unet_label.yaml
@@ -1,5 +1,5 @@
 DATASET:
-  dataset_dir: /home/nico/Dataset/FabricFinal
+  dataset_dir: /home/nico/Dataset/ZJU-Leaper
   fabrics: all
   normalization: 'zscore'
   color_mode: rgb
diff --git a/dataset_api/eval_pixel.py b/dataset_api/eval_pixel.py
@@ -3,9 +3,8 @@
 @Time   : 2019-05-07 23:53
 @Author : Nico
 """
-from collections import OrderedDict
-
 import numpy as np
+from collections import OrderedDict
 from typing import Tuple
 
 
@@ -86,6 +85,7 @@ def evaluation_pixel(binary_pixel_pred: np.ndarray, binary_pixel_target: np.ndar
         Pre=precision,
         Rec=recall,
         Dice=dice,
+        F1=dice,  # dice index is just the pixel F1-score
         # IoU=_iou(confusion),
         # nMCC=_mcc(confusion, normalized=True),
     )
diff --git a/dataset_api/eval_region.py b/dataset_api/eval_region.py
@@ -5,10 +5,9 @@
 """
 
 import itertools
-import warnings
-
 import numpy as np
 import pandas as pd
+import warnings
 from skimage.measure import label, regionprops
 from skimage.measure._regionprops import RegionProperties
 
@@ -170,21 +169,33 @@ def _precision_for_region_pred(info_pred_overlap: pd.DataFrame, iou_thresh: floa
     :param iou_thresh:
     :return:
     """
+    # if scale == 'all':
+    #     pass
+    # elif scale == 'small':
+    #     info_pred_overlap = info_pred_overlap[
+    #         (info_pred_overlap['area_target'] <= SMALL_TH)
+    #         | (info_pred_overlap['id_target'].isna() & (info_pred_overlap['area_pred'] <= SMALL_TH))]
+    # elif scale == 'medium':
+    #     info_pred_overlap = info_pred_overlap[
+    #         ((SMALL_TH < info_pred_overlap['area_target']) & (info_pred_overlap['area_target'] < MEDIUM_TH))
+    #         | (info_pred_overlap['id_target'].isna()
+    #            & ((SMALL_TH < info_pred_overlap['area_pred']) & (info_pred_overlap['area_pred'] < MEDIUM_TH)))]
+    # elif scale == 'large':
+    #     info_pred_overlap = info_pred_overlap[
+    #         (info_pred_overlap['area_target'] >= MEDIUM_TH)
+    #         | (info_pred_overlap['id_target'].isna() & (info_pred_overlap['area_pred'] >= MEDIUM_TH))]
+    # else:
+    #     raise NotImplementedError
+
+    # NEW: the size constrains only related to the ground-truth
     if scale == 'all':
         pass
     elif scale == 'small':
-        info_pred_overlap = info_pred_overlap[
-            (info_pred_overlap['area_target'] <= SMALL_TH)
-            | (info_pred_overlap['id_target'].isna() & (info_pred_overlap['area_pred'] <= SMALL_TH))]
+        info_pred_overlap = info_pred_overlap[info_pred_overlap['area_target'] < SMALL_TH]
     elif scale == 'medium':
-        info_pred_overlap = info_pred_overlap[
-            ((SMALL_TH < info_pred_overlap['area_target']) & (info_pred_overlap['area_target'] < MEDIUM_TH))
-            | (info_pred_overlap['id_target'].isna()
-               & ((SMALL_TH < info_pred_overlap['area_pred']) & (info_pred_overlap['area_pred'] < MEDIUM_TH)))]
+        info_pred_overlap = info_pred_overlap[((SMALL_TH <= info_pred_overlap['area_target']) & (info_pred_overlap['area_target'] <= MEDIUM_TH))]
     elif scale == 'large':
-        info_pred_overlap = info_pred_overlap[
-            (info_pred_overlap['area_target'] >= MEDIUM_TH)
-            | (info_pred_overlap['id_target'].isna() & (info_pred_overlap['area_pred'] >= MEDIUM_TH))]
+        info_pred_overlap = info_pred_overlap[info_pred_overlap['area_target'] > MEDIUM_TH]
     else:
         raise NotImplementedError
 
diff --git a/dataset_api/eval_sample.py b/dataset_api/eval_sample.py
@@ -9,7 +9,8 @@
 
 from .eval_pixel import _compute_confusion_matrix, _f1_score, _acc, _fpr
 
-MIN_AREA = 4
+# the size constrain is according to pixel ratio
+MIN_AREA_RATIO = 1. / 128.
 
 
 def evaluation_sample(binary_pixel_pred, binary_pixel_target, info_region):
@@ -19,10 +20,12 @@ def evaluation_sample(binary_pixel_pred, binary_pixel_target, info_region):
     info_overlap: pd.DataFrame = info_region["info_overlap"]
 
     num_img = len(binary_pixel_target)
+    _, h, w = binary_pixel_target.squeeze().shape
+    MIN_AREA_TH = (np.float_(h) * MIN_AREA_RATIO) * (np.float_(w) * MIN_AREA_RATIO)
 
     label_pred = np.zeros(num_img, dtype=np.bool)
     if not info_pred.empty:
-        img_pred_ids = info_pred[info_pred['area_pred'] > MIN_AREA]['id_img'].unique()
+        img_pred_ids = info_pred[info_pred['area_pred'] > MIN_AREA_TH]['id_img'].unique()
         label_pred[img_pred_ids] = 1
 
     label_target = np.zeros(num_img, dtype=np.bool)
diff --git a/dataset_api/zl_eval.py b/dataset_api/zl_eval.py
@@ -4,11 +4,10 @@
 @Author : Nico
 """
 import json
+import numpy as np
 import warnings
 from collections import OrderedDict
 
-import numpy as np
-
 from .eval_pixel import evaluation_pixel
 from .eval_region import evaluation_region
 from .eval_sample import evaluation_sample
@@ -56,12 +55,12 @@ def evaluate(self) -> dict:
                                                         diff_size=self.eval_diff_size, return_info=True)
         sample_metrics = evaluation_sample(binary_pixel_pred=self.binary_pixel_pred,
                                            binary_pixel_target=self.binary_pixel_target, info_region=info_region)
-        summary_score = 0.4 * pixel_metrics['Dice'] + 0.4 * region_metrics['F1'] + 0.2 * sample_metrics['F1']
+        summary_score = 0.4 * pixel_metrics['F1'] + 0.4 * region_metrics['F1'] + 0.2 * sample_metrics['F1']
 
         self.metrics_dict = OrderedDict(
             # Pix_Pre=pixel_metrics['Pre'],
             # Pix_Rec=pixel_metrics['Rec'],
-            F1_Pix=pixel_metrics['Dice'],
+            F1_Pix=pixel_metrics['F1'],
 
             # Reg_Pre=region_metrics['Pre'],
             # Reg_Rec=region_metrics['Rec'],

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,5 @@`
	`1`	`+saved_models/ # exclude model checkpoints`
	`2`	`+`
`1`	`3`	`# Created by .ignore support plugin (hsz.mobi)`
`2`	`4`	`### Python template`
`3`	`5`	`# Byte-compiled / optimized / DLL files`