Merge pull request #37 from tomas-gajarsky/feature/action_unit

Feature/action unit
tomas-gajarsky · Mar 4, 2023 · 45c8fa4 · 45c8fa4
2 parents fe3587d + af6640c
commit 45c8fa4
Show file tree

Hide file tree

Showing 34 changed files with 2,415 additions and 1,347 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,17 @@
 # Change Log
 
+
+## 0.3.0
+
+Released on February 9, 2023.
+
+### Added
+* predictor for facial action unit detection - OpenGraphAU from Luo et al.
+
+### Changed
+* facetorch version retrieval error handling to not fail the run
+
+
 ## 0.2.4
 
 Released on February 4, 2023.

diff --git a/README.md b/README.md
@@ -78,6 +78,7 @@ analyzer
  ├── embed
  ├── verify
  ├── fer
+ ├── au
  ├── deepfake
  └── align
  └── utilizer
@@ -150,6 +151,16 @@ analyzer
  * B0 [![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/facial-expression-and-attributes-recognition/facial-expression-recognition-on-affectnet)](https://paperswithcode.com/sota/facial-expression-recognition-on-affectnet?p=facial-expression-and-attributes-recognition)
  * B0 [![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/facial-expression-and-attributes-recognition/facial-expression-recognition-on-acted-facial)](https://paperswithcode.com/sota/facial-expression-recognition-on-acted-facial?p=facial-expression-and-attributes-recognition)
 
+#### Facial Action Unit Detection (au)
+
+ | model | source | params | license | version | 
+ | ------------------- | --------- | ------- | ------------------ | ------- |
+ | OpenGraph Swin Base | CVI-SZU | 94M | MIT License | 1 |
+
+1. CVI-SZU
+ * code: [ME-GraphAU](https://github.com/CVI-SZU/ME-GraphAU)
+ * paper: [Luo et al. - Learning Multi-dimensional Edge Feature-based AU Relation Graph for Facial Action Unit Recognition](https://arxiv.org/abs/2205.01782)
+ * [![PWC](https://img.shields.io/endpoint.svg?url=https://paperswithcode.com/badge/learning-multi-dimensional-edge-feature-based/facial-action-unit-detection-on-bp4d)](https://paperswithcode.com/sota/facial-action-unit-detection-on-bp4d?p=learning-multi-dimensional-edge-feature-based)
 
 #### Deepfake Detection (deepfake)
 
@@ -185,18 +196,19 @@ You can also download the models manually from a [public Google Drive folder](ht
 
 ### Execution time
 
-Image test.jpg (4 faces) is analyzed (including drawing boxes and landmarks, but not saving) in about 465ms and test3.jpg (25 faces) in about 1480ms (batch_size=8) on NVIDIA Tesla T4 GPU once the default configuration (*conf/config.yaml*) of models is initialized and pre heated to the initial image size 1080x1080 by the first run. One can monitor the execution times in logs using the DEBUG level.
+Image test.jpg (4 faces) is analyzed (including drawing boxes and landmarks, but not saving) in about 482ms and test3.jpg (25 faces) in about 1844ms (batch_size=8) on NVIDIA Tesla T4 GPU once the default configuration (*conf/config.yaml*) of models is initialized and pre heated to the initial image size 1080x1080 by the first run. One can monitor the execution times in logs using the DEBUG level.
 
 Detailed test.jpg execution times:
 ```
 analyzer
  ├── reader: 27 ms
- ├── detector: 230 ms
+ ├── detector: 193 ms
  ├── unifier: 1 ms
  └── predictor
  ├── embed: 8 ms
  ├── verify: 58 ms
  ├── fer: 28 ms
+ ├── au: 57 ms
  ├── deepfake: 117 ms
  └── align: 5 ms
  └── utilizer

diff --git a/conf/analyzer/predictor/au/open_graph_swin_base.yaml b/conf/analyzer/predictor/au/open_graph_swin_base.yaml
@@ -0,0 +1,38 @@
+_target_: facetorch.analyzer.predictor.FacePredictor
+
+downloader:
+ _target_: facetorch.downloader.DownloaderGDrive
+ file_id: 1uoVX9suSA5JVWTms3hEtJKzwO-CUR_jV
+ path_local: /opt/facetorch/models/torchscript/predictor/au/1/model.pt # str
+
+device:
+ _target_: torch.device
+ type: ${analyzer.device} # str
+
+preprocessor:
+ _target_: facetorch.analyzer.predictor.pre.PredictorPreProcessor
+ transform:
+ _target_: torchvision.transforms.Compose
+ transforms:
+ - _target_: torchvision.transforms.Resize
+ size: [224, 224] # List[int]
+ - _target_: torchvision.transforms.Normalize
+ mean: [0.485, 0.456, 0.406] # List[float]
+ std: [0.229, 0.224, 0.225] # List[float]
+ device: 
+ _target_: torch.device
+ type: ${analyzer.predictor.au.device.type}
+ optimize_transform: ${analyzer.optimize_transforms}
+ reverse_colors: False # bool
+
+postprocessor:
+ _target_: facetorch.analyzer.predictor.post.PostMultiLabel
+ transform: None
+ device: 
+ _target_: torch.device
+ type: ${analyzer.predictor.au.device.type}
+ optimize_transform: ${analyzer.optimize_transforms}
+ dim: 1
+ threshold: 0.5
+ labels: ["inner_brow_raiser", "outer_brow_raiser", "brow_lowerer", "upper_lid_raiser", "cheek_raiser", "lid_tightener", "nose_wrinkler", "upper_lip_raiser", "nasolabial_deepener", "lip_corner_puller", "sharp_lip_puller", "dimpler", "lip_corner_depressor", "lower_lip_depressor", "chin_raiser", "lip_pucker", "tongue_show", "lip_stretcher", "lip_funneler", "lip_tightener", "lip_pressor", "lips_part", "jaw_drop", "mouth_stretch", "lip_bite", "nostril_dilator", "nostril_compressor", "left_inner_brow_raiser", "right_inner_brow_raiser", "left_outer_brow_raiser", "right_outer_brow_raiser", "left_brow_lowerer", "right_brow_lowerer", "left_cheek_raiser", "right_cheek_raiser", "left_upper_lip_raiser", "right_upper_lip_raiser", "left_nasolabial_deepener", "right_nasolabial_deepener", "left_dimpler", "right_dimpler"] # List
+
diff --git a/conf/config.yaml b/conf/config.yaml
@@ -6,6 +6,7 @@ defaults:
  - analyzer/predictor/embed: r50_vggface_1m.yaml
  - analyzer/predictor/verify: adaface_ir101_webface12m.yaml
  - analyzer/predictor/fer: efficientnet_b2_8.yaml
+ - analyzer/predictor/au: open_graph_swin_base.yaml
  - analyzer/predictor/deepfake: efficientnet_b7.yaml
  - analyzer/predictor/align: synergy_mobilenet_v2.yaml
  - analyzer/utilizer/align: lmk3d_mesh_pose.yaml

diff --git a/conf/merged/gpu.merged.config.yaml b/conf/merged/gpu.merged.config.yaml
@@ -228,6 +228,81 @@ analyzer:
  - Neutral
  - Sadness
  - Surprise
+ au:
+ _target_: facetorch.analyzer.predictor.FacePredictor
+ downloader:
+ _target_: facetorch.downloader.DownloaderGDrive
+ file_id: 1uoVX9suSA5JVWTms3hEtJKzwO-CUR_jV
+ path_local: /opt/facetorch/models/torchscript/predictor/au/1/model.pt # str
+ device:
+ _target_: torch.device
+ type: ${analyzer.device} # str
+ preprocessor:
+ _target_: facetorch.analyzer.predictor.pre.PredictorPreProcessor
+ transform:
+ _target_: torchvision.transforms.Compose
+ transforms:
+ - _target_: torchvision.transforms.Resize
+ size: [224, 224] # List[int]
+ - _target_: torchvision.transforms.Normalize
+ mean: [0.485, 0.456, 0.406] # List[float]
+ std: [0.229, 0.224, 0.225] # List[float]
+ device: 
+ _target_: torch.device
+ type: ${analyzer.predictor.au.device.type}
+ optimize_transform: ${analyzer.optimize_transforms}
+ reverse_colors: False # bool
+ postprocessor:
+ _target_: facetorch.analyzer.predictor.post.PostMultiLabel
+ transform: None
+ device: 
+ _target_: torch.device
+ type: ${analyzer.predictor.au.device.type}
+ optimize_transform: ${analyzer.optimize_transforms}
+ dim: 1
+ threshold: 0.5
+ labels:
+ - inner_brow_raiser
+ - outer_brow_raiser
+ - brow_lowerer
+ - upper_lid_raiser
+ - cheek_raiser
+ - lid_tightener
+ - nose_wrinkler
+ - upper_lip_raiser
+ - nasolabial_deepener
+ - lip_corner_puller
+ - sharp_lip_puller
+ - dimpler
+ - lip_corner_depressor
+ - lower_lip_depressor
+ - chin_raiser
+ - lip_pucker
+ - tongue_show
+ - lip_stretcher
+ - lip_funneler
+ - lip_tightener
+ - lip_pressor
+ - lips_part
+ - jaw_drop
+ - mouth_stretch
+ - lip_bite
+ - nostril_dilator
+ - nostril_compressor
+ - left_inner_brow_raiser
+ - right_inner_brow_raiser
+ - left_outer_brow_raiser
+ - right_outer_brow_raiser
+ - left_brow_lowerer
+ - right_brow_lowerer
+ - left_cheek_raiser
+ - right_cheek_raiser
+ - left_upper_lip_raiser
+ - right_upper_lip_raiser
+ - left_nasolabial_deepener
+ - right_nasolabial_deepener
+ - left_dimpler
+ - right_dimpler
  deepfake:
  _target_: facetorch.analyzer.predictor.FacePredictor
  downloader:

diff --git a/conf/merged/merged.config.yaml b/conf/merged/merged.config.yaml
@@ -228,6 +228,81 @@ analyzer:
  - Neutral
  - Sadness
  - Surprise
+ au:
+ _target_: facetorch.analyzer.predictor.FacePredictor
+ downloader:
+ _target_: facetorch.downloader.DownloaderGDrive
+ file_id: 1uoVX9suSA5JVWTms3hEtJKzwO-CUR_jV
+ path_local: /opt/facetorch/models/torchscript/predictor/au/1/model.pt # str
+ device:
+ _target_: torch.device
+ type: ${analyzer.device} # str
+ preprocessor:
+ _target_: facetorch.analyzer.predictor.pre.PredictorPreProcessor
+ transform:
+ _target_: torchvision.transforms.Compose
+ transforms:
+ - _target_: torchvision.transforms.Resize
+ size: [224, 224] # List[int]
+ - _target_: torchvision.transforms.Normalize
+ mean: [0.485, 0.456, 0.406] # List[float]
+ std: [0.229, 0.224, 0.225] # List[float]
+ device: 
+ _target_: torch.device
+ type: ${analyzer.predictor.au.device.type}
+ optimize_transform: ${analyzer.optimize_transforms}
+ reverse_colors: False # bool
+ postprocessor:
+ _target_: facetorch.analyzer.predictor.post.PostMultiLabel
+ transform: None
+ device: 
+ _target_: torch.device
+ type: ${analyzer.predictor.au.device.type}
+ optimize_transform: ${analyzer.optimize_transforms}
+ dim: 1
+ threshold: 0.5
+ labels:
+ - inner_brow_raiser
+ - outer_brow_raiser
+ - brow_lowerer
+ - upper_lid_raiser
+ - cheek_raiser
+ - lid_tightener
+ - nose_wrinkler
+ - upper_lip_raiser
+ - nasolabial_deepener
+ - lip_corner_puller
+ - sharp_lip_puller
+ - dimpler
+ - lip_corner_depressor
+ - lower_lip_depressor
+ - chin_raiser
+ - lip_pucker
+ - tongue_show
+ - lip_stretcher
+ - lip_funneler
+ - lip_tightener
+ - lip_pressor
+ - lips_part
+ - jaw_drop
+ - mouth_stretch
+ - lip_bite
+ - nostril_dilator
+ - nostril_compressor
+ - left_inner_brow_raiser
+ - right_inner_brow_raiser
+ - left_outer_brow_raiser
+ - right_outer_brow_raiser
+ - left_brow_lowerer
+ - right_brow_lowerer
+ - left_cheek_raiser
+ - right_cheek_raiser
+ - left_upper_lip_raiser
+ - right_upper_lip_raiser
+ - left_nasolabial_deepener
+ - right_nasolabial_deepener
+ - left_dimpler
+ - right_dimpler
  deepfake:
  _target_: facetorch.analyzer.predictor.FacePredictor
  downloader:

diff --git a/conf/tests.config.1.yaml b/conf/tests.config.1.yaml
@@ -6,6 +6,7 @@ defaults:
  - analyzer/predictor/embed: r50_vggface_1m.yaml
  - analyzer/predictor/verify: adaface_ir101_webface12m.yaml
  - analyzer/predictor/fer: efficientnet_b2_8.yaml
+ - analyzer/predictor/au: open_graph_swin_base.yaml
  - analyzer/predictor/deepfake: efficientnet_b7.yaml
  - analyzer/predictor/align: synergy_mobilenet_v2.yaml
  - analyzer/utilizer/align: lmk3d_mesh_pose.yaml

diff --git a/conf/tests.config.2.yaml b/conf/tests.config.2.yaml
@@ -6,6 +6,7 @@ defaults:
  - analyzer/predictor/embed: r50_vggface_1m.yaml
  - analyzer/predictor/verify: r100_magface_unpg.yaml
  - analyzer/predictor/fer: efficientnet_b0_7.yaml
+ - analyzer/predictor/au: open_graph_swin_base.yaml
  - analyzer/predictor/deepfake: efficientnet_b7.yaml
  - analyzer/predictor/align: synergy_mobilenet_v2.yaml
  - analyzer/utilizer/align: lmk3d_mesh_pose.yaml

diff --git a/conf/tests.config.3.yaml b/conf/tests.config.3.yaml
@@ -6,6 +6,7 @@ defaults:
  - analyzer/predictor/embed: r50_vggface_1m.yaml
  - analyzer/predictor/verify: adaface_ir101_webface12m.yaml
  - analyzer/predictor/fer: efficientnet_b2_8.yaml
+ - analyzer/predictor/au: open_graph_swin_base.yaml
  - analyzer/predictor/deepfake: efficientnet_b7.yaml
  - analyzer/predictor/align: synergy_mobilenet_v2.yaml
  - analyzer/utilizer/align: lmk3d_mesh_pose.yaml

diff --git a/conf/tests.config.4.yaml b/conf/tests.config.4.yaml
@@ -6,6 +6,7 @@ defaults:
  - analyzer/predictor/embed: r50_vggface_1m.yaml
  - analyzer/predictor/verify: adaface_ir101_webface12m.yaml
  - analyzer/predictor/fer: efficientnet_b2_8.yaml
+ - analyzer/predictor/au: open_graph_swin_base.yaml
  - analyzer/predictor/deepfake: efficientnet_b7.yaml
  - analyzer/predictor/align: synergy_mobilenet_v2.yaml
  - analyzer/utilizer/align: lmk3d_mesh_pose.yaml

diff --git a/docker/Dockerfile.dev.gpu b/docker/Dockerfile.dev.gpu
@@ -1,4 +1,4 @@
-FROM nvidia/cuda:11.6.2-runtime-ubuntu20.04
+FROM nvidia/cuda:11.2.2-runtime-ubuntu20.04
 
 # Set working directory
 ENV WORKDIR=/opt/facetorch

diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu
@@ -1,4 +1,4 @@
-FROM nvidia/cuda:11.7.0-runtime-ubuntu20.04
+FROM nvidia/cuda:11.2.2-runtime-ubuntu20.04
 
 # Set working directory
 ENV WORKDIR=/opt/facetorch

diff --git a/docs/facetorch/analyzer/core.html b/docs/facetorch/analyzer/core.html
@@ -147,7 +147,11 @@ <h1 class="title">Module <code>facetorch.analyzer.core</code></h1>
  data = self.reader.run(path_image, fix_img_size=fix_img_size)
  path_output = None if path_output == &#34;None&#34; else path_output
  data.path_output = path_output
- data.version = pkg_resources.get_distribution(&#34;facetorch&#34;).version
+
+ try:
+ data.version = pkg_resources.get_distribution(&#34;facetorch&#34;).version
+ except Exception as e:
+ self.logger.warning(&#34;Could not get version number&#34;, extra={&#34;error&#34;: e})
 
  self.logger.info(&#34;Detecting faces&#34;)
  data = self.detector.run(data)
@@ -344,7 +348,11 @@ <h2 id="attributes">Attributes</h2>
  data = self.reader.run(path_image, fix_img_size=fix_img_size)
  path_output = None if path_output == &#34;None&#34; else path_output
  data.path_output = path_output
- data.version = pkg_resources.get_distribution(&#34;facetorch&#34;).version
+
+ try:
+ data.version = pkg_resources.get_distribution(&#34;facetorch&#34;).version
+ except Exception as e:
+ self.logger.warning(&#34;Could not get version number&#34;, extra={&#34;error&#34;: e})
 
  self.logger.info(&#34;Detecting faces&#34;)
  data = self.detector.run(data)
@@ -462,7 +470,11 @@ <h2 id="returns">Returns</h2>
  data = self.reader.run(path_image, fix_img_size=fix_img_size)
  path_output = None if path_output == &#34;None&#34; else path_output
  data.path_output = path_output
- data.version = pkg_resources.get_distribution(&#34;facetorch&#34;).version
+
+ try:
+ data.version = pkg_resources.get_distribution(&#34;facetorch&#34;).version
+ except Exception as e:
+ self.logger.warning(&#34;Could not get version number&#34;, extra={&#34;error&#34;: e})
 
  self.logger.info(&#34;Detecting faces&#34;)
  data = self.detector.run(data)