AC: input dumping POC

tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/__init__.py

@@ -0,0 +1,25 @@
+"""
+Copyright (c) 2018-2022 Intel Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from .bin_dumper import BinDataDumper
+from .numpy_dumper import NPZInputDumper
+from .input_dumper import InputDumper
+
+__all__ = [
+    'BinDataDumper',
+    'NPZInputDumper',
+    'InputDumper'
+]

tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/ark_dumper.py

@@ -0,0 +1,15 @@
+"""
+Copyright (c) 2018-2022 Intel Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/bin_dumper.py

@@ -0,0 +1,29 @@
+"""
+Copyright (c) 2018-2022 Intel Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import numpy as np
+
+from .input_dumper import InputDumper
+
+
+class BinDataDumper(InputDumper):
+    __provider__ = 'bin'
+
+    def dump_sample(self, data, data_id):
+        suffix = f'{data_id}.bin'
+        out_file_template = '{}_{}'
+        for key, value in data.items():
+            out_file_name = out_file_template.format(key, suffix)
+            value.tofile(str(self.out_dir / out_file_name))

tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/dump_inputs.py

@@ -0,0 +1,85 @@
+"""
+Copyright (c) 2018-2022 Intel Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from argparse import ArgumentParser
+from functools import partial
+from ..argparser import add_common_args, add_config_filtration_args, add_dataset_related_args
+from ..config import ConfigReader
+from ..utils import extract_image_representations, get_path
+from ..evaluators.quantization_model_evaluator import create_dataset_attributes
+from ..launcher import create_launcher, InputFeeder
+from .input_dumper import InputDumper
+
+
+def build_argparser():
+    parser = ArgumentParser('dump_inputs')
+    add_common_args(parser)
+    add_config_filtration_args(parser)
+    add_dataset_related_args(parser)
+    parser.add_argument(
+        '-o', '--output_dir', required=True, type=partial(get_path, is_directory=True),
+        help='Output directory for dumping')
+    parser.add_argument(
+        '--output_format', required=False, default='npz',
+        help='Format for output mapping. Supported: {}'.join(['bin', 'npz', 'npy', 'ark', 'pickle']))
+    parser.add_argument('-n', '--num_samples', type=int, required=False)
+    return parser
+
+
+def dump_inputs(output_dir, data_loader, preprocessor, input_feeder=None, num_samples=None, dump_format='bin'):
+    def _get_batch_input(batch_annotation, batch_input):
+        batch_input = preprocessor.process(batch_input, batch_annotation)
+        batch_data, batch_meta = extract_image_representations(batch_input)
+        if input_feeder is None:
+            return batch_data, batch_meta
+        filled_inputs = input_feeder.fill_inputs(batch_input)
+        return filled_inputs, batch_meta
+
+    sample_dumper = InputDumper.provide(dump_format, {'type': dump_format}, output_dir)
+
+    for batch_id, (_, batch_annotation, batch_input, _) in enumerate(data_loader):
+        filled_inputs, batch_meta = _get_batch_input(batch_annotation, batch_input)
+        sample_dumper(filled_inputs[0], batch_id)
+        if num_samples is not None and batch_id == num_samples:
+            break
+
+
+def main():
+    args = build_argparser().parse_args()
+    config, mode = ConfigReader.merge(args)
+    if mode != 'models':
+        raise ValueError(f'Unsupported mode {mode}')
+    for config_entry in config[mode]:
+        data_loader, _, preprocessor, _ = create_dataset_attributes(config_entry['datasets'], '')
+        try:
+            launcher_config = config_entry['launchers'][0]
+            launcher = create_launcher(launcher_config)
+            launcher_inputs = launcher.inputs
+            input_precision = launcher_config.get('_input_precision', [])
+            input_layouts = launcher_config.get('_input_layout', '')
+            input_feeder = InputFeeder(
+                launcher.config.get('inputs', []), launcher_inputs, launcher.input_shape, launcher.fit_to_input,
+                launcher.default_layout, launcher_config['framework'] == 'dummy',
+                input_precision,
+                input_layouts
+            )
+        except:
+            input_feeder = None
+        dump_inputs(args.output_dir, data_loader, preprocessor, input_feeder, args.num_samples, args.output_format)
+
+
+if __name__ == '__main__':
+    main()

tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/input_dumper.py

@@ -0,0 +1,35 @@
+"""
+Copyright (c) 2018-2022 Intel Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from pathlib import Path
+from ..dependency import ClassProvider
+
+
+class InputDumper(ClassProvider):
+    __provider_type__ = 'input_dumper'
+
+    def __init__(self, config, out_dir):
+        self.config = config
+        self.out_dir = Path(out_dir)
+        if not self.out_dir.exists():
+            self.out_dir.mkdir(parents=True)
+
+    def dump_sample(self, data, data_id):
+        raise NotImplementedError
+
+    def __call__(self, data, data_id):
+        self.dump_sample(data, data_id)
+

tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/numpy_dumper.py

@@ -0,0 +1,26 @@
+"""
+Copyright (c) 2018-2022 Intel Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import numpy as np
+from .input_dumper import InputDumper
+
+
+class NPZInputDumper(InputDumper):
+    __provider__ = 'npz'
+
+    def dump_sample(self, data, data_id):
+        out_file = f'{data_id}.npz'
+        np.savez(self.out_dir / out_file, **data)

tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/pickle_dumper.py

@@ -0,0 +1,15 @@
+"""
+Copyright (c) 2018-2022 Intel Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""

tools/accuracy_checker/setup.py

@@ -122,7 +122,9 @@ def prepare_requirements():
     entry_points={
         "console_scripts": [
             "accuracy_check=openvino.tools.accuracy_checker.main:main",
-            "convert_annotation=openvino.tools.accuracy_checker.annotation_converters.convert:main"]},
+            "convert_annotation=openvino.tools.accuracy_checker.annotation_converters.convert:main",
+            "dump_inputs=openvino.tools.accuracy_checker.input_damping.dump_inputs:main"
+        ]},
     zip_safe=False,
     python_requires='>=3.5',
     install_requires=_requirements if not is_arm else '',