Skip to content

Commit 35adb52

Browse files
rabah-khalekrabah-khalek
authored
Merge pull request #9 from Giskard-AI/GSK-2248
[GSK-2248] TestResult to dict
2 parents 0f76a9f + 1821a0a commit 35adb52

File tree

7 files changed

+182
-155
lines changed

7 files changed

+182
-155
lines changed

‎examples/ex5_models_comparison.ipynb

+123-131
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
},
1212
{
1313
"cell_type": "code",
14-
"execution_count": 1,
14+
"execution_count": 2,
1515
"metadata": {},
1616
"outputs": [],
1717
"source": [
@@ -30,18 +30,18 @@
3030
},
3131
{
3232
"cell_type": "code",
33-
"execution_count": 9,
33+
"execution_count": 3,
3434
"metadata": {},
3535
"outputs": [],
3636
"source": [
3737
"dl = DataLoader300W(dir_path=\"300W/sample\")\n",
3838
"chosen_idx = 4\n",
39-
"image, ground_truth_landmarks, _ = ds[chosen_idx]"
39+
"image, ground_truth_landmarks, _ = dl[chosen_idx]"
4040
]
4141
},
4242
{
4343
"cell_type": "code",
44-
"execution_count": 10,
44+
"execution_count": 4,
4545
"metadata": {},
4646
"outputs": [
4747
{
@@ -50,7 +50,7 @@
5050
"'cpu'"
5151
]
5252
},
53-
"execution_count": 10,
53+
"execution_count": 4,
5454
"metadata": {},
5555
"output_type": "execute_result"
5656
}
@@ -64,15 +64,13 @@
6464
},
6565
{
6666
"cell_type": "code",
67-
"execution_count": 11,
67+
"execution_count": 5,
6868
"metadata": {},
6969
"outputs": [
7070
{
7171
"name": "stdout",
7272
"output_type": "stream",
7373
"text": [
74-
"File exists\n",
75-
"File exists\n",
7674
"loading data from : lbfmodel.yaml\n"
7775
]
7876
}
@@ -84,109 +82,60 @@
8482
},
8583
{
8684
"cell_type": "code",
87-
"execution_count": 12,
88-
"metadata": {},
89-
"outputs": [
90-
{
91-
"data": {
92-
"text/html": [
93-
"\n",
94-
" <h4><span style=\"color:green;\">✓</span> Test \"NME_mean\" succeeded</h4>\n",
95-
" <p>Description: Mean of normalised mean Euclidean distances across images</p>\n",
96-
" <p>Metric: <b>0.0414</b> (threshold = 1)</p>\n",
97-
" \n",
98-
" <p>Prediction time: 0.54 s.</p>\n",
99-
" "
100-
],
101-
"text/plain": [
102-
"\n",
103-
" Test \"NME_mean\" succeeded\n",
104-
" Description: Mean of normalised mean Euclidean distances across images\n",
105-
" Metric: 0.0414 (threshold = 1)\n",
106-
" \n",
107-
" Prediction time: 0.54 s.\n",
108-
" "
109-
]
110-
},
111-
"execution_count": 12,
112-
"metadata": {},
113-
"output_type": "execute_result"
114-
}
115-
],
116-
"source": [
117-
"Test(metric=NMEMean, threshold=1).run(model=opencv_model, dataloader=dl)"
118-
]
119-
},
120-
{
121-
"cell_type": "code",
122-
"execution_count": 14,
85+
"execution_count": 6,
12386
"metadata": {},
12487
"outputs": [
12588
{
12689
"data": {
127-
"text/html": [
128-
"\n",
129-
" <h4><span style=\"color:green;\">✓</span> Test \"NME_mean\" succeeded</h4>\n",
130-
" <p>Description: Mean of normalised mean Euclidean distances across images</p>\n",
131-
" <p>Metric: <b>0.0623</b> (threshold = 1)</p>\n",
132-
" \n",
133-
" <p>Prediction time: 37.83 s.</p>\n",
134-
" "
135-
],
13690
"text/plain": [
137-
"\n",
138-
" Test \"NME_mean\" succeeded\n",
139-
" Description: Mean of normalised mean Euclidean distances across images\n",
140-
" Metric: 0.0623 (threshold = 1)\n",
141-
" \n",
142-
" Prediction time: 37.83 s.\n",
143-
" "
91+
"{'test_name': 'Test',\n",
92+
" 'metric_name': 'NME_mean',\n",
93+
" 'metric_value': 0.04136279942306024,\n",
94+
" 'threshold': 1,\n",
95+
" 'passed': True,\n",
96+
" 'facial_part': 'entire face',\n",
97+
" 'model_name': 'OpenCV',\n",
98+
" 'dataloader_name': '300W'}"
14499
]
145100
},
146-
"execution_count": 14,
101+
"execution_count": 6,
147102
"metadata": {},
148103
"output_type": "execute_result"
149104
}
150105
],
151106
"source": [
152-
"Test(metric=NMEMean, threshold=1).run(model=facealignment_model, dataloader=dl)"
107+
"test = Test(metric=NMEMean, threshold=1).run(model=opencv_model, dataloader=dl)\n",
108+
"\n",
109+
"test.to_dict()"
153110
]
154111
},
155112
{
156113
"cell_type": "code",
157-
"execution_count": 8,
114+
"execution_count": 7,
158115
"metadata": {},
159116
"outputs": [
160117
{
161118
"name": "stderr",
162119
"output_type": "stream",
163120
"text": [
164-
"/home/bazire/loreal-poc/loreal_poc/tests/performance.py:47: RuntimeWarning: Mean of empty slice\n",
121+
"/Users/rak/Documents/loreal-poc/loreal_poc/tests/performance.py:47: RuntimeWarning: Mean of empty slice\n",
165122
" mes = np.nanmean(es, axis=1)\n"
166123
]
167124
},
168125
{
169126
"data": {
170-
"text/html": [
171-
"\n",
172-
" <h4><span style=\"color:green;\">✓</span> Test \"NME_mean\" succeeded</h4>\n",
173-
" <p>Description: Mean of normalised mean Euclidean distances across images</p>\n",
174-
" <p>Metric: <b>0.6623</b> (threshold = 1)</p>\n",
175-
" Prediction fail rate: 0.4\n",
176-
" <p>Prediction time: 0.54 s.</p>\n",
177-
" "
178-
],
179127
"text/plain": [
180-
"\n",
181-
" Test \"NME_mean\" succeeded\n",
182-
" Description: Mean of normalised mean Euclidean distances across images\n",
183-
" Metric: 0.6623 (threshold = 1)\n",
184-
" Prediction fail rate: 0.4\n",
185-
" Prediction time: 0.54 s.\n",
186-
" "
128+
"{'test_name': 'TestDiff',\n",
129+
" 'metric_name': 'NME_mean',\n",
130+
" 'metric_value': 0.3984345106345805,\n",
131+
" 'threshold': 1,\n",
132+
" 'passed': True,\n",
133+
" 'facial_part': 'left half',\n",
134+
" 'model_name': 'OpenCV',\n",
135+
" 'dataloader_name': '300W'}"
187136
]
188137
},
189-
"execution_count": 8,
138+
"execution_count": 7,
190139
"metadata": {},
191140
"output_type": "execute_result"
192141
}
@@ -196,87 +145,130 @@
196145
"\n",
197146
"facial_part = FacialParts.left_half\n",
198147
"\n",
199-
"TestDiff(metric=NMEMean, threshold=1).run(\n",
148+
"test_diff = TestDiff(metric=NMEMean, threshold=1).run(\n",
200149
" model=opencv_model, dataloader=dl, dataloader_ref=CroppedDataLoader(dl, part=facial_part), facial_part=facial_part\n",
201-
")"
150+
")\n",
151+
"test_diff.to_dict()"
202152
]
203153
},
204154
{
205155
"cell_type": "code",
206-
"execution_count": 9,
156+
"execution_count": 8,
207157
"metadata": {},
208158
"outputs": [
209159
{
210160
"name": "stderr",
211161
"output_type": "stream",
212162
"text": [
213-
"/home/bazire/loreal-poc/loreal_poc/tests/performance.py:85: RuntimeWarning: Mean of empty slice\n",
214-
" return np.nanmean(NMEs.get(prediction_result, marks))\n",
215-
"/home/bazire/loreal-poc/.venv/lib/python3.10/site-packages/face_alignment/api.py:147: UserWarning: No faces were detected.\n",
216-
" warnings.warn(\"No faces were detected.\")\n"
163+
"/Users/rak/Documents/loreal-poc/loreal_poc/tests/performance.py:85: RuntimeWarning: Mean of empty slice\n",
164+
" return np.nanmean(NMEs.get(prediction_result, marks))\n"
217165
]
218166
}
219167
],
220168
"source": [
221-
"from dataclasses import fields\n",
169+
"# from dataclasses import fields\n",
170+
"def report(models, dataloader, tests, facial_parts):\n",
171+
" results = []\n",
172+
" for model in models:\n",
173+
" # for __facial_part in fields(FacialParts):\n",
174+
" # _facial_part = __facial_part.default\n",
175+
" for facial_part in facial_parts:\n",
176+
" for test_cls in tests:\n",
177+
" test_result = test_cls(metric=NMEMean, threshold=1).run(\n",
178+
" model=model,\n",
179+
" dataloader=CroppedDataLoader(dataloader, part=facial_part),\n",
180+
" dataloader_ref=dl,\n",
181+
" facial_part=facial_part,\n",
182+
" )\n",
183+
" results.append(test_result.to_dict())\n",
184+
" return results\n",
185+
"\n",
222186
"\n",
223-
"results = {\"OpenCV\": {\"model\": opencv_model}, \"FaceAlignment\": {\"model\": facealignment_model}}\n",
224-
"for model_name, _model in results.items():\n",
225-
" model = _model[\"model\"]\n",
226-
" for __facial_part in fields(FacialParts):\n",
227-
" _facial_part = __facial_part.default\n",
228-
" test = TestDiff(metric=NMEMean, threshold=1).run(\n",
229-
" model=model,\n",
230-
" dataloader=dl,\n",
231-
" dataloader_ref=CroppedDataLoader(dl, part=_facial_part),\n",
232-
" facial_part=_facial_part,\n",
233-
" )\n",
234-
" results[model_name][_facial_part.name] = test.metric"
187+
"report = report([opencv_model], dl, [TestDiff], [FacialParts.bottom_half, FacialParts.upper_half])"
235188
]
236189
},
237190
{
238191
"cell_type": "code",
239-
"execution_count": 10,
192+
"execution_count": 9,
240193
"metadata": {},
241194
"outputs": [
242195
{
243196
"data": {
197+
"text/html": [
198+
"<div>\n",
199+
"<style scoped>\n",
200+
" .dataframe tbody tr th:only-of-type {\n",
201+
" vertical-align: middle;\n",
202+
" }\n",
203+
"\n",
204+
" .dataframe tbody tr th {\n",
205+
" vertical-align: top;\n",
206+
" }\n",
207+
"\n",
208+
" .dataframe thead th {\n",
209+
" text-align: right;\n",
210+
" }\n",
211+
"</style>\n",
212+
"<table border=\"1\" class=\"dataframe\">\n",
213+
" <thead>\n",
214+
" <tr style=\"text-align: right;\">\n",
215+
" <th></th>\n",
216+
" <th>test_name</th>\n",
217+
" <th>metric_name</th>\n",
218+
" <th>metric_value</th>\n",
219+
" <th>threshold</th>\n",
220+
" <th>passed</th>\n",
221+
" <th>facial_part</th>\n",
222+
" <th>model_name</th>\n",
223+
" <th>dataloader_name</th>\n",
224+
" </tr>\n",
225+
" </thead>\n",
226+
" <tbody>\n",
227+
" <tr>\n",
228+
" <th>0</th>\n",
229+
" <td>TestDiff</td>\n",
230+
" <td>NME_mean</td>\n",
231+
" <td>NaN</td>\n",
232+
" <td>1</td>\n",
233+
" <td>False</td>\n",
234+
" <td>bottom half</td>\n",
235+
" <td>OpenCV</td>\n",
236+
" <td>300W cropped on bottom half</td>\n",
237+
" </tr>\n",
238+
" <tr>\n",
239+
" <th>1</th>\n",
240+
" <td>TestDiff</td>\n",
241+
" <td>NME_mean</td>\n",
242+
" <td>0.040216</td>\n",
243+
" <td>1</td>\n",
244+
" <td>True</td>\n",
245+
" <td>upper half</td>\n",
246+
" <td>OpenCV</td>\n",
247+
" <td>300W cropped on upper half</td>\n",
248+
" </tr>\n",
249+
" </tbody>\n",
250+
"</table>\n",
251+
"</div>"
252+
],
244253
"text/plain": [
245-
"{'OpenCV': {'model': <loreal_poc.models.wrappers.OpenCVWrapper at 0x7f75e93bf910>,\n",
246-
" 'entire face': 0.031593074968095824,\n",
247-
" 'face contour': 0.0524161528719839,\n",
248-
" 'left eyebrow': nan,\n",
249-
" 'right eyebrow': nan,\n",
250-
" 'nose': nan,\n",
251-
" 'left eye': nan,\n",
252-
" 'right eye': nan,\n",
253-
" 'mouth': nan,\n",
254-
" 'bottom half': nan,\n",
255-
" 'upper half': 0.04021610184657019,\n",
256-
" 'left half': 0.662329401666445,\n",
257-
" 'right half': 0.9675771331137769},\n",
258-
" 'FaceAlignment': {'model': <loreal_poc.models.wrappers.FaceAlignmentWrapper at 0x7f75e93bc370>,\n",
259-
" 'entire face': 0.06824182183674955,\n",
260-
" 'face contour': 0.13120804359950317,\n",
261-
" 'left eyebrow': nan,\n",
262-
" 'right eyebrow': nan,\n",
263-
" 'nose': nan,\n",
264-
" 'left eye': 2.2704133968712634,\n",
265-
" 'right eye': nan,\n",
266-
" 'mouth': nan,\n",
267-
" 'bottom half': 8.182222629236117e-05,\n",
268-
" 'upper half': 0.22317472213175207,\n",
269-
" 'left half': 0.6372027526669263,\n",
270-
" 'right half': 0.3839864768052418}}"
254+
" test_name metric_name metric_value threshold passed facial_part \\\n",
255+
"0 TestDiff NME_mean NaN 1 False bottom half \n",
256+
"1 TestDiff NME_mean 0.040216 1 True upper half \n",
257+
"\n",
258+
" model_name dataloader_name \n",
259+
"0 OpenCV 300W cropped on bottom half \n",
260+
"1 OpenCV 300W cropped on upper half "
271261
]
272262
},
273-
"execution_count": 10,
263+
"execution_count": 9,
274264
"metadata": {},
275265
"output_type": "execute_result"
276266
}
277267
],
278268
"source": [
279-
"results"
269+
"import pandas as pd\n",
270+
"\n",
271+
"pd.DataFrame(report)"
280272
]
281273
}
282274
],
@@ -296,7 +288,7 @@
296288
"name": "python",
297289
"nbconvert_exporter": "python",
298290
"pygments_lexer": "ipython3",
299-
"version": "3.10.12"
291+
"version": "3.11.6"
300292
}
301293
},
302294
"nbformat": 4,

0 commit comments

Comments
 (0)