diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/__init__.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/__init__.py new file mode 100644 index 00000000000..3044ce190f0 --- /dev/null +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/__init__.py @@ -0,0 +1,25 @@ +""" +Copyright (c) 2018-2022 Intel Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from .bin_dumper import BinDataDumper +from .numpy_dumper import NPZInputDumper +from .input_dumper import InputDumper + +__all__ = [ + 'BinDataDumper', + 'NPZInputDumper', + 'InputDumper' +] diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/ark_dumper.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/ark_dumper.py new file mode 100644 index 00000000000..2f96414a53d --- /dev/null +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/ark_dumper.py @@ -0,0 +1,15 @@ +""" +Copyright (c) 2018-2022 Intel Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/bin_dumper.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/bin_dumper.py new file mode 100644 index 00000000000..dbfae1cda0a --- /dev/null +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/bin_dumper.py @@ -0,0 +1,29 @@ +""" +Copyright (c) 2018-2022 Intel Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import numpy as np + +from .input_dumper import InputDumper + + +class BinDataDumper(InputDumper): + __provider__ = 'bin' + + def dump_sample(self, data, data_id): + suffix = f'{data_id}.bin' + out_file_template = '{}_{}' + for key, value in data.items(): + out_file_name = out_file_template.format(key, suffix) + value.tofile(str(self.out_dir / out_file_name)) diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/dump_inputs.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/dump_inputs.py new file mode 100644 index 00000000000..41a795d7c6c --- /dev/null +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/dump_inputs.py @@ -0,0 +1,85 @@ +""" +Copyright (c) 2018-2022 Intel Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from argparse import ArgumentParser +from functools import partial +from ..argparser import add_common_args, add_config_filtration_args, add_dataset_related_args +from ..config import ConfigReader +from ..utils import extract_image_representations, get_path +from ..evaluators.quantization_model_evaluator import create_dataset_attributes +from ..launcher import create_launcher, InputFeeder +from .input_dumper import InputDumper + + +def build_argparser(): + parser = ArgumentParser('dump_inputs') + add_common_args(parser) + add_config_filtration_args(parser) + add_dataset_related_args(parser) + parser.add_argument( + '-o', '--output_dir', required=True, type=partial(get_path, is_directory=True), + help='Output directory for dumping') + parser.add_argument( + '--output_format', required=False, default='npz', + help='Format for output mapping. Supported: {}'.join(['bin', 'npz', 'npy', 'ark', 'pickle'])) + parser.add_argument('-n', '--num_samples', type=int, required=False) + return parser + + +def dump_inputs(output_dir, data_loader, preprocessor, input_feeder=None, num_samples=None, dump_format='bin'): + def _get_batch_input(batch_annotation, batch_input): + batch_input = preprocessor.process(batch_input, batch_annotation) + batch_data, batch_meta = extract_image_representations(batch_input) + if input_feeder is None: + return batch_data, batch_meta + filled_inputs = input_feeder.fill_inputs(batch_input) + return filled_inputs, batch_meta + + sample_dumper = InputDumper.provide(dump_format, {'type': dump_format}, output_dir) + + for batch_id, (_, batch_annotation, batch_input, _) in enumerate(data_loader): + filled_inputs, batch_meta = _get_batch_input(batch_annotation, batch_input) + sample_dumper(filled_inputs[0], batch_id) + if num_samples is not None and batch_id == num_samples: + break + + +def main(): + args = build_argparser().parse_args() + config, mode = ConfigReader.merge(args) + if mode != 'models': + raise ValueError(f'Unsupported mode {mode}') + for config_entry in config[mode]: + data_loader, _, preprocessor, _ = create_dataset_attributes(config_entry['datasets'], '') + try: + launcher_config = config_entry['launchers'][0] + launcher = create_launcher(launcher_config) + launcher_inputs = launcher.inputs + input_precision = launcher_config.get('_input_precision', []) + input_layouts = launcher_config.get('_input_layout', '') + input_feeder = InputFeeder( + launcher.config.get('inputs', []), launcher_inputs, launcher.input_shape, launcher.fit_to_input, + launcher.default_layout, launcher_config['framework'] == 'dummy', + input_precision, + input_layouts + ) + except: + input_feeder = None + dump_inputs(args.output_dir, data_loader, preprocessor, input_feeder, args.num_samples, args.output_format) + + +if __name__ == '__main__': + main() diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/input_dumper.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/input_dumper.py new file mode 100644 index 00000000000..660552573ff --- /dev/null +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/input_dumper.py @@ -0,0 +1,35 @@ +""" +Copyright (c) 2018-2022 Intel Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from pathlib import Path +from ..dependency import ClassProvider + + +class InputDumper(ClassProvider): + __provider_type__ = 'input_dumper' + + def __init__(self, config, out_dir): + self.config = config + self.out_dir = Path(out_dir) + if not self.out_dir.exists(): + self.out_dir.mkdir(parents=True) + + def dump_sample(self, data, data_id): + raise NotImplementedError + + def __call__(self, data, data_id): + self.dump_sample(data, data_id) + diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/numpy_dumper.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/numpy_dumper.py new file mode 100644 index 00000000000..b43f5de91db --- /dev/null +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/numpy_dumper.py @@ -0,0 +1,26 @@ +""" +Copyright (c) 2018-2022 Intel Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np +from .input_dumper import InputDumper + + +class NPZInputDumper(InputDumper): + __provider__ = 'npz' + + def dump_sample(self, data, data_id): + out_file = f'{data_id}.npz' + np.savez(self.out_dir / out_file, **data) diff --git a/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/pickle_dumper.py b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/pickle_dumper.py new file mode 100644 index 00000000000..2f96414a53d --- /dev/null +++ b/tools/accuracy_checker/openvino/tools/accuracy_checker/input_damping/pickle_dumper.py @@ -0,0 +1,15 @@ +""" +Copyright (c) 2018-2022 Intel Corporation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/tools/accuracy_checker/setup.py b/tools/accuracy_checker/setup.py index 26762e01bfa..7c9b1a60093 100644 --- a/tools/accuracy_checker/setup.py +++ b/tools/accuracy_checker/setup.py @@ -122,7 +122,9 @@ def prepare_requirements(): entry_points={ "console_scripts": [ "accuracy_check=openvino.tools.accuracy_checker.main:main", - "convert_annotation=openvino.tools.accuracy_checker.annotation_converters.convert:main"]}, + "convert_annotation=openvino.tools.accuracy_checker.annotation_converters.convert:main", + "dump_inputs=openvino.tools.accuracy_checker.input_damping.dump_inputs:main" + ]}, zip_safe=False, python_requires='>=3.5', install_requires=_requirements if not is_arm else '',