From fb98103ede489447f4e5c0032db75a2d741d654d Mon Sep 17 00:00:00 2001 From: Fredrik Knutsson Date: Fri, 8 Nov 2024 14:09:39 +0100 Subject: [PATCH] Revert "Run tosa_reference_model using python binding (#6658)" This reverts commit 4bbe9945b7c221f7b687dbb6754ce4e650c93c05. --- backends/arm/arm_backend.py | 9 ++- backends/arm/test/common.py | 12 ++-- backends/arm/test/misc/test_debug_feats.py | 5 +- backends/arm/test/ops/test_cat.py | 2 +- backends/arm/test/ops/test_select.py | 4 +- backends/arm/test/runner_utils.py | 81 ++++------------------ backends/arm/test/tester/arm_tester.py | 15 ++-- examples/arm/setup.sh | 27 ++++++-- 8 files changed, 59 insertions(+), 96 deletions(-) diff --git a/backends/arm/arm_backend.py b/backends/arm/arm_backend.py index db3b368115..28af583106 100644 --- a/backends/arm/arm_backend.py +++ b/backends/arm/arm_backend.py @@ -13,7 +13,7 @@ import logging import os -from typing import cast, final, List, Optional +from typing import final, List, Optional import serializer.tosa_serializer as ts from executorch.backends.arm.arm_vela import vela_compile @@ -31,7 +31,6 @@ from executorch.exir.backend.backend_details import BackendDetails, PreprocessResult from executorch.exir.backend.compile_spec_schema import CompileSpec from torch.export.exported_program import ExportedProgram -from torch.fx import Node # TOSA backend debug functionality logger = logging.getLogger(__name__) @@ -226,7 +225,6 @@ def preprocess( # noqa: C901 node_visitors = get_node_visitors(edge_program) for node in graph_module.graph.nodes: - node = cast(Node, node) if node.op == "call_function": process_call_function(node, tosa_graph, node_visitors) elif node.op == "placeholder": @@ -238,6 +236,9 @@ def preprocess( # noqa: C901 # any checking of compatibility. dbg_fail(node, tosa_graph, artifact_path) + # TODO: It would be awesome if this dump could somehow be done on top level and not here. + # Problem is that the desc.json has to be created on the tosa_graph object, which we can't + # access from top level. if artifact_path: tag = _get_first_delegation_tag(graph_module) dbg_tosa_dump( @@ -258,4 +259,6 @@ def preprocess( # noqa: C901 else: raise RuntimeError(f"Unknown format {output_format}") + # Continueing from above. Can I put tosa_graph into this function? + # debug_handle_map = ... return PreprocessResult(processed_bytes=binary) diff --git a/backends/arm/test/common.py b/backends/arm/test/common.py index 1a155c0323..b0e2a7f0bb 100644 --- a/backends/arm/test/common.py +++ b/backends/arm/test/common.py @@ -192,15 +192,19 @@ def get_tosa_compile_spec_unbuilt( the compile spec before calling .build() to finalize it. """ if not custom_path: - custom_path = maybe_get_tosa_collate_path() + intermediate_path = maybe_get_tosa_collate_path() or tempfile.mkdtemp( + prefix="arm_tosa_" + ) + else: + intermediate_path = custom_path - if custom_path is not None and not os.path.exists(custom_path): - os.makedirs(custom_path, exist_ok=True) + if not os.path.exists(intermediate_path): + os.makedirs(intermediate_path, exist_ok=True) compile_spec_builder = ( ArmCompileSpecBuilder() .tosa_compile_spec() .set_permute_memory_format(permute_memory_to_nhwc) - .dump_intermediate_artifacts_to(custom_path) + .dump_intermediate_artifacts_to(intermediate_path) ) return compile_spec_builder diff --git a/backends/arm/test/misc/test_debug_feats.py b/backends/arm/test/misc/test_debug_feats.py index 1aa3e82c76..7d9a18a80e 100644 --- a/backends/arm/test/misc/test_debug_feats.py +++ b/backends/arm/test/misc/test_debug_feats.py @@ -107,10 +107,7 @@ def test_numerical_diff_prints(self): ArmTester( model, example_inputs=model.get_inputs(), - compile_spec=common.get_tosa_compile_spec( - permute_memory_to_nhwc=True, - custom_path=tempfile.mkdtemp("diff_print_test"), - ), + compile_spec=common.get_tosa_compile_spec(permute_memory_to_nhwc=False), ) .export() .to_edge() diff --git a/backends/arm/test/ops/test_cat.py b/backends/arm/test/ops/test_cat.py index b0a38ce198..9723ba0f0c 100644 --- a/backends/arm/test/ops/test_cat.py +++ b/backends/arm/test/ops/test_cat.py @@ -121,7 +121,7 @@ def test_cat_tosa_MI(self, operands: tuple[torch.Tensor, ...], dim: int): def test_cat_4d_tosa_MI(self): square = torch.ones((2, 2, 2, 2)) for dim in range(-3, 3): - test_data = ((square, square.clone()), dim) + test_data = ((square, square), dim) self._test_cat_tosa_MI_pipeline(self.Cat(), test_data) @parameterized.expand(Cat.test_parameters) diff --git a/backends/arm/test/ops/test_select.py b/backends/arm/test/ops/test_select.py index 6a47c2e66b..fdb2fa1463 100644 --- a/backends/arm/test/ops/test_select.py +++ b/backends/arm/test/ops/test_select.py @@ -93,6 +93,8 @@ def _test_select_tosa_BI_pipeline( .check(["torch.ops.quantized_decomposed"]) .to_edge() .partition() + .dump_artifact() + .dump_operator_distribution() .check_count({"torch.ops.higher_order.executorch_call_delegate": 1}) .to_executorch() .run_method_and_compare_outputs(inputs=test_data) @@ -160,14 +162,12 @@ def test_select_int_tosa_MI(self, test_data: test_data_t): ) @parameterized.expand(test_data_suite) - @unittest.skip def test_select_copy_tosa_BI(self, test_data: test_data_t): self._test_select_tosa_BI_pipeline( self.SelectCopy(), test_data, export_target="torch.ops.aten.select_copy.int" ) @parameterized.expand(test_data_suite) - @unittest.skip def test_select_int_tosa_BI(self, test_data: test_data_t): self._test_select_tosa_BI_pipeline( self.SelectInt(), test_data, export_target="torch.ops.aten.select.int" diff --git a/backends/arm/test/runner_utils.py b/backends/arm/test/runner_utils.py index f3c90eda83..d2ee113a5d 100644 --- a/backends/arm/test/runner_utils.py +++ b/backends/arm/test/runner_utils.py @@ -17,14 +17,11 @@ import numpy as np import torch -import tosa_reference_model - from torch.export import ExportedProgram from torch.fx.node import Node -from tosa import TosaGraph logger = logging.getLogger(__name__) -logger.setLevel(logging.CRITICAL) +logger.setLevel(logging.WARNING) class QuantizationParams: @@ -170,7 +167,7 @@ def __init__( ): self.intermediate_path = intermediate_path self.tosa_ref_model_path = tosa_ref_model_path or "tosa_reference_model" - assert self.intermediate_path is None or os.path.exists( + assert os.path.exists( self.intermediate_path ), f"TOSA artifact path don't exist! Path: {self.intermediate_path}" @@ -326,46 +323,7 @@ def run_corstone( tosa_ref_output = np.fromfile(out_path_with_suffix, dtype=np.float32) output_shape = self.output_node.args[0][0].meta["val"].shape tosa_ref_output = torch.from_numpy(tosa_ref_output).reshape(output_shape) - return tosa_ref_output - - def run_tosa_graph( - self, graph: TosaGraph, inputs: list[np.ndarray] | list[torch.Tensor] - ) -> torch.Tensor: - """Runs the TOSA reference model with inputs and returns the result.""" - data_np = [ - prep_data_for_save( - input, self.is_quantized, self.input_names[i], self.qp_input[i] - ) - for i, input in enumerate(inputs) - ] - # tosa_profile: 0 = Base Inference, 1 = Main Inference, 2 = Main Training. - tosa_profile = 0 if self.is_quantized else 1 - debug_mode = "ALL" if logger.level <= logging.DEBUG else None - outputs, status = tosa_reference_model.run( - graph, - data_np, - verbosity=_tosa_refmodel_loglevel(logger.level), - tosa_profile=tosa_profile, - initialize_variable_tensor_from_numpy=1, # True - debug_mode=debug_mode, - ) - - assert ( - status == tosa_reference_model.GraphStatus.TOSA_VALID - ), "Non-valid TOSA given to reference model." - - outputs_torch = [] - for output in outputs: - output = output.astype(np.float32) - if self.is_quantized: - # Need to dequant back to FP32 for comparison with torch output - quant_param = self.qp_output - assert ( - quant_param is not None - ), "There are no quantization parameters, check output parameters" - output = (output - quant_param.zp) * quant_param.scale - outputs_torch.append(torch.from_numpy(output)) - return tuple(outputs_torch) + return [tosa_ref_output] def run_tosa_ref_model( self, @@ -450,13 +408,21 @@ def run_tosa_ref_model( assert ( shutil.which(self.tosa_ref_model_path) is not None ), f"tosa_reference_model tool not found, did you run examples/arm/setup.sh? Path: {self.tosa_ref_model_path}" - + loglevel_map = { + logging.INFO: "INFO", + logging.CRITICAL: "LOW", + logging.ERROR: "LOW", + logging.WARNING: "MED", + logging.DEBUG: "HIGH", + logging.NOTSET: "MED", + } + clamped_logging_level = max(min(logger.level // 10 * 10, 50), 0) cmd_ref_model = [ self.tosa_ref_model_path, "--test_desc", desc_file_path, "-l", - _tosa_refmodel_loglevel(logger.level), + loglevel_map[clamped_logging_level], ] _run_cmd(cmd_ref_model) @@ -492,10 +458,7 @@ def run_tosa_ref_model( def prep_data_for_save( - data: torch.Tensor, - is_quantized: bool, - input_name: str, - quant_param: QuantizationParams, + data, is_quantized: bool, input_name: str, quant_param: QuantizationParams ): data_np = np.array(data.detach(), order="C").astype( f"{data.dtype}".replace("torch.", "") @@ -639,19 +602,3 @@ def dbg_tosa_fb_to_json(tosa_fb: bytes) -> Dict: pass return json_out - - -def _tosa_refmodel_loglevel(loglevel: int) -> str: - """Converts a logging loglevel to tosa_reference_model logginglevel, - returned as string. - """ - loglevel_map = { - logging.INFO: "INFO", - logging.CRITICAL: "LOW", - logging.ERROR: "LOW", - logging.WARNING: "MED", - logging.DEBUG: "HIGH", - logging.NOTSET: "MED", - } - clamped_logging_level = max(min(loglevel // 10 * 10, 50), 0) - return loglevel_map[clamped_logging_level] diff --git a/backends/arm/test/tester/arm_tester.py b/backends/arm/test/tester/arm_tester.py index 834e177b7d..096bc2b22f 100644 --- a/backends/arm/test/tester/arm_tester.py +++ b/backends/arm/test/tester/arm_tester.py @@ -39,7 +39,7 @@ from executorch.backends.xnnpack.test.tester import Tester from executorch.devtools.backend_debug import get_delegation_info -from executorch.exir import EdgeCompileConfig, EdgeProgramManager +from executorch.exir import EdgeCompileConfig from executorch.exir.backend.compile_spec_schema import CompileSpec from executorch.exir.lowered_backend_module import LoweredBackendModule @@ -120,15 +120,10 @@ def __init__( super().__init__(dynamic_shapes) self.tosa_test_util = tosa_test_util - def run(self, artifact: EdgeProgramManager, inputs=None): - self.executorch_program = artifact.to_executorch(self.config) - if module := getattr( - artifact.exported_program().graph_module, "lowered_module_0", None - ): - self.buffer = module.processed_bytes - def run_artifact(self, inputs): - tosa_output = self.tosa_test_util.run_tosa_graph(self.buffer, inputs) + tosa_output = self.tosa_test_util.run_tosa_ref_model( + inputs=inputs, + ) return tosa_output @@ -321,7 +316,7 @@ def run_method_and_compare_outputs( logger.info(f"Run #{run_iteration}, input shapes: {input_shape_str}") reference_output = reference_stage.run_artifact(reference_input) - test_output = test_stage.run_artifact(test_input) + test_output = tuple(test_stage.run_artifact(test_input)) if ( is_nhwc and test_stage == self.stages[self.stage_name(tester.ToExecutorch)] diff --git a/examples/arm/setup.sh b/examples/arm/setup.sh index 43f7d48b83..583237729d 100755 --- a/examples/arm/setup.sh +++ b/examples/arm/setup.sh @@ -88,7 +88,7 @@ ethos_u_base_rev="24.08" # tosa reference model tosa_reference_model_url="https://review.mlplatform.org/tosa/reference_model" -tosa_reference_model_rev="ef31e7222e99cb1c24b2aff9fc52b2d609612283" +tosa_reference_model_rev="f9ea4ab7da19318fe36b1c34d68a3e40fd6e56c5" ######## ### Mandatory user args @@ -227,13 +227,30 @@ function setup_tosa_reference_model() { cd reference_model git checkout ${tosa_reference_model_rev} git submodule update --init --recursive + cd .. + fi + cd reference_model + mkdir -p build + cd build + cmake .. + + # make use of half the cores for building + if [[ "${OS}" == "Linux" ]]; then + n=$(( $(nproc) / 2 )) + elif [[ "${OS}" == "Darwin" ]]; then + n=$(( $(sysctl -n hw.logicalcpu) / 2 )) + else + n=1 fi - echo "pip installing reference_model..." - repo_dir="${root_dir}/reference_model" - cd $repo_dir - pip install . + if [[ "$n" -lt 1 ]]; then + n=1 + fi + make -j"${n}" + cd reference_model + tosa_bin_path=`pwd` + echo "export PATH=\${PATH}:${tosa_bin_path}" >> "${setup_path_script}" } function setup_vela() {