[TVMC][Relay] Introduce executor and runtime parameters (#9352)

* [TVMC][Relay] Introduce executor and runtime parameters

This introduces `executor` and `runtime` into the various entrypoints but also into `tvmc` as `--executor` and `--runtime`. This touchs a lot of files and I've tried to update anywhere as necessary.

Notable, executor code generators now accept the initial `IRModule` rather than creating
it themselves so it can be annotated once.

Validated the demo application continues to classify the tabby cat with
new CLI options.

* Correct Graph Executor Python API

Author: Mousius

apps/bundle_deploy/build_model.py

@@ -20,15 +20,15 @@
 import os
 from tvm import relay
 import tvm
-from tvm import te, runtime
+from tvm import runtime as tvm_runtime
 import logging
-import json
+from tvm.relay.backend import Runtime
 from tvm.contrib import cc as _cc
 
-RUNTIMES = {
-    "c": "{name}_c.{ext}",
-    "c++": "{name}_cpp.{ext}",
-}
+RUNTIMES = [
+    (Runtime("crt", {"system-lib": True}), "{name}_c.{ext}"),
+    (Runtime("cpp", {"system-lib": True}), "{name}_cpp.{ext}"),
+]
 
 
 def build_module(opts):
@@ -43,18 +43,16 @@ def build_module(opts):
         func.params, relay.nn.softmax(func.body), None, func.type_params, func.attrs
     )
 
-    for runtime_name, file_format_str in RUNTIMES.items():
+    for runtime, file_format_str in RUNTIMES:
         with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}):
-            graph, lib, params = relay.build(
-                func, f"llvm --runtime={runtime_name} --system-lib", params=params
-            )
+            graph, lib, params = relay.build(func, "llvm", runtime=runtime, params=params)
 
         build_dir = os.path.abspath(opts.out_dir)
         if not os.path.isdir(build_dir):
             os.makedirs(build_dir)
-        ext = "tar" if runtime_name == "c" else "o"
+        ext = "tar" if str(runtime) == "crt" else "o"
         lib_file_name = os.path.join(build_dir, file_format_str.format(name="model", ext=ext))
-        if runtime_name == "c":
+        if str(runtime) == "crt":
             lib.export_library(lib_file_name)
         else:
             # NOTE: at present, export_libarary will always create _another_ shared object, and you
@@ -70,7 +68,7 @@ def build_module(opts):
         with open(
             os.path.join(build_dir, file_format_str.format(name="params", ext="bin")), "wb"
         ) as f_params:
-            f_params.write(runtime.save_param_dict(params))
+            f_params.write(tvm_runtime.save_param_dict(params))
 
 
 def build_test_module(opts):
@@ -84,20 +82,21 @@ def build_test_module(opts):
     y_data = np.random.rand(1, 5).astype("float32")
     params = {"y": y_data}
 
-    for runtime_name, file_format_str in RUNTIMES.items():
+    for runtime, file_format_str in RUNTIMES:
         with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}):
             graph, lib, lowered_params = relay.build(
                 tvm.IRModule.from_expr(func),
-                f"llvm --runtime={runtime_name} --system-lib",
+                "llvm",
+                runtime=runtime,
                 params=params,
             )
 
         build_dir = os.path.abspath(opts.out_dir)
         if not os.path.isdir(build_dir):
             os.makedirs(build_dir)
-        ext = "tar" if runtime_name == "c" else "o"
+        ext = "tar" if str(runtime) == "crt" else "o"
         lib_file_name = os.path.join(build_dir, file_format_str.format(name="test_model", ext=ext))
-        if runtime_name == "c":
+        if str(runtime) == "crt":
             lib.export_library(lib_file_name)
         else:
             # NOTE: at present, export_libarary will always create _another_ shared object, and you
@@ -113,7 +112,7 @@ def build_test_module(opts):
         with open(
             os.path.join(build_dir, file_format_str.format(name="test_params", ext="bin")), "wb"
         ) as f_params:
-            f_params.write(runtime.save_param_dict(lowered_params))
+            f_params.write(tvm_runtime.save_param_dict(lowered_params))
         with open(
             os.path.join(build_dir, file_format_str.format(name="test_data", ext="bin")), "wb"
         ) as fp:

apps/microtvm/ethosu/run_demo.sh

@@ -130,8 +130,12 @@ curl --retry 64 -sSL ${mobilenet_url} | gunzip | tar -xvf - ./mobilenet_v1_1.0_2
 # Compile model for Arm(R) Cortex(R)-M55 CPU and Ethos(TM)-U55 NPU
 # An alternative to using "python3 -m tvm.driver.tvmc" is to call
 # "tvmc" directly once TVM has been pip installed.
-python3 -m tvm.driver.tvmc compile --target="ethos-u -accelerator_config=ethos-u55-256, \
-    c -runtime=c --link-params -mcpu=cortex-m55 -executor=aot -interface-api=c -unpacked-api=1" \
+python3 -m tvm.driver.tvmc compile --target="ethos-u -accelerator_config=ethos-u55-256, c" \
+    --target-c-mcpu=cortex-m55 \
+    --runtime=crt \
+    --executor=aot \
+    --executor-aot-interface-api=c \
+    --executor-aot-unpacked-api=1 \
     --pass-config tir.disable_vectorize=1 ./mobilenet_v1_1.0_224_quant.tflite --output-format=mlf
 tar -xvf module.tar
 

docs/arch/microtvm_design.rst

@@ -127,18 +127,24 @@ logs use it to rank measured performance (but see Future Work).
 Targets are currently represented as strings structured similarly to command-line arguments. An
 example target is shown below:
 
-    ``c -keys=arm_cpu -mcpu=cortex-m7 -link-params -model=stm32f746xx -runtime=c -system-lib=1``
+    ``c -keys=arm_cpu -mcpu=cortex-m7 -model=stm32f746xx``
 
 The relevant parts to microTVM are:
 
  * Code generator (``llvm`` or ``c``)
  * ``-mcpu=cortex-m7``: used by TOPI to enable Cortex-M schedules, and, when the C source code
    generator is selected, included in the output as a comment to help identify the code and
    configure the downstream C compiler.
- * ``-link-params``: include parameters as global constants to load from flash.
- * ``-runtime=c``: build glue code to allow operators to work with the C runtime
- * ``-system-lib=1``: emit a system library (i.e. which can be loaded by calling the PackedFunc
-   ``runtime.SystemLib``.
+
+Runtime and Executor configuration for microTVM
+-----------------------------------------------
+
+When using microTVM, it's important to use the C Runtime (``Runtime('crt')``), which is the runtime that works best on micro devices rather than the more dynamic C++ Runtime. Alongside this, there are two executors which you could use in combination with the C runtime:
+
+* ``Executor("aot")`` - The Ahead of Time (AOT) executor precompiles the network into a runnable function which you can add directly into your micro application
+* ``Executor("graph", {"link-params": True})`` - The Graph executor provides a JSON representation of your network and requires the C Runtime's system library to be generated to find functions in the function registry (``Runtime("crt", {"system-lib": True})``). ``{"link-params":True}`` enables parameters to be linked into the generated files rather than provided externally.
+
+These are specified when building a runtime module: ``relay.build(..., runtime=..., executor=...)``.
 
 Writing Schedules for microTVM
 ------------------------------

gallery/how_to/work_with_microtvm/micro_autotune.py

@@ -32,6 +32,7 @@
 import pathlib
 
 import tvm
+from tvm.relay.backend import Executor, Runtime
 
 ####################
 # Defining the model
@@ -69,13 +70,15 @@
 # Defining the target #
 #######################
 # Now we define the TVM target that describes the execution environment. This looks very similar
-# to target definitions from other microTVM tutorials.
+# to target definitions from other microTVM tutorials. Alongside this we pick the C Runtime to code
+# generate our model against.
 #
 # When running on physical hardware, choose a target and a board that
 # describe the hardware. There are multiple hardware targets that could be selected from
 # PLATFORM list in this tutorial. You can chose the platform by passing --platform argument when running
 # this tutorial.
 #
+RUNTIME = Runtime("crt", {"system-lib": True})
 TARGET = tvm.target.target.micro("host")
 
 # Compiling for physical hardware
@@ -123,6 +126,7 @@
     build_kwargs={"build_option": {"tir.disable_vectorize": True}},
     do_fork=True,
     build_func=tvm.micro.autotvm_build_func,
+    runtime=RUNTIME,
 )
 runner = tvm.autotvm.LocalRunner(number=1, repeat=1, timeout=100, module_loader=module_loader)
 
@@ -175,7 +179,7 @@
 # the tuned operator.
 
 with pass_context:
-    lowered = tvm.relay.build(relay_mod, target=TARGET, params=params)
+    lowered = tvm.relay.build(relay_mod, target=TARGET, runtime=RUNTIME, params=params)
 
 temp_dir = tvm.contrib.utils.tempdir()
 
@@ -218,7 +222,7 @@
 
 with tvm.autotvm.apply_history_best("microtvm_autotune.log.txt"):
     with pass_context:
-        lowered_tuned = tvm.relay.build(relay_mod, target=TARGET, params=params)
+        lowered_tuned = tvm.relay.build(relay_mod, target=TARGET, runtime=RUNTIME, params=params)
 
 temp_dir = tvm.contrib.utils.tempdir()
 

gallery/how_to/work_with_microtvm/micro_tflite.py

@@ -124,12 +124,9 @@
 
 import os
 import numpy as np
-import logging
 
 import tvm
-import tvm.micro as micro
 from tvm.contrib.download import download_testdata
-from tvm.contrib import graph_executor, utils
 from tvm import relay
 
 model_url = "https://people.linaro.org/~tom.gall/sine_model.tflite"
@@ -179,9 +176,10 @@
 # Now we create a build config for relay, turning off two options and then calling relay.build which
 # will result in a C source file for the selected TARGET. When running on a simulated target of the
 # same architecture as the host (where this Python script is executed) choose "host" below for the
-# TARGET and a proper board/VM to run it (Zephyr will create the right QEMU VM based on BOARD. In
-# the example below the x86 arch is selected and a x86 VM is picked up accordingly:
+# TARGET, the C Runtime as the RUNTIME and a proper board/VM to run it (Zephyr will create the right
+# QEMU VM based on BOARD. In the example below the x86 arch is selected and a x86 VM is picked up accordingly:
 #
+RUNTIME = tvm.relay.backend.Runtime("crt", {"system-lib": True})
 TARGET = tvm.target.target.micro("host")
 BOARD = "qemu_x86"
 #
@@ -210,7 +208,7 @@
 with tvm.transform.PassContext(
     opt_level=3, config={"tir.disable_vectorize": True}, disabled_pass=["AlterOpLayout"]
 ):
-    module = relay.build(mod, target=TARGET, params=params)
+    module = relay.build(mod, target=TARGET, runtime=RUNTIME, params=params)
 
 
 # Inspecting the compilation output

include/tvm/ir/module.h

@@ -301,6 +301,12 @@ class IRModuleNode : public Object {
    */
   TVM_DLL void ImportFromStd(const String& path);
 
+  /*!
+   * \brief Should Link Parameters into the module
+   * \return Whether the Executor is configured to execute with linked parameters (Default: false)
+   */
+  TVM_DLL Bool ShouldLinkParameters() const;
+
   /*!
    * \brief The set of imported files.
    */
@@ -468,5 +474,27 @@ TVM_DLL String PrettyPrint(const ObjectRef& node);
  */
 TVM_DLL String AsText(const ObjectRef& node, bool show_meta_data = true,
                       runtime::TypedPackedFunc<String(ObjectRef)> annotate = nullptr);
+
+namespace attr {
+
+/*!
+ * \brief Executor targetted by the module
+ *
+ * Type: Executor
+ *
+ * \sa tvm::relay::Executor
+ */
+constexpr const char* kExecutor = "executor";
+
+/*!
+ * \brief Runtime target of the module
+ *
+ * Type: Runtime
+ *
+ * \sa tvm::relay::Runtime
+ */
+constexpr const char* kRuntime = "runtime";
+
+}  // namespace attr
 }  // namespace tvm
 #endif  // TVM_IR_MODULE_H_

include/tvm/relay/executor.h

@@ -59,6 +59,14 @@ class ExecutorNode : public Object {
   /* \brief Additional attributes storing meta-data about the Executor. */
   DictAttrs attrs;
 
+  /*!
+   * \brief Should Link Parameters into the module
+   * \return Whether the Executor is configured to execute modules with linked parameters
+   */
+  Bool ShouldLinkParameters() const {
+    return name == "aot" || GetAttr<Bool>("link-params").value_or(Bool(false));
+  }
+
   /*!
    * \brief Get an attribute.
    *
@@ -114,14 +122,16 @@ class ExecutorNode : public Object {
  */
 class Executor : public ObjectRef {
  public:
+  Executor() = default;
+
   /*!
    * \brief Create a new Executor object using the registry
    * \throws Error if name is not registered
    * \param name The name of the executor.
    * \param attrs Attributes for the executor.
    * \return the new Executor object.
    */
-  TVM_DLL static Executor Create(String name, Map<String, ObjectRef> attrs);
+  TVM_DLL static Executor Create(String name, Map<String, ObjectRef> attrs = {});
 
   /*!
    * \brief List all registered Executors

include/tvm/relay/runtime.h

@@ -114,14 +114,16 @@ class RuntimeNode : public Object {
  */
 class Runtime : public ObjectRef {
  public:
+  Runtime() = default;
+
   /*!
    * \brief Create a new Runtime object using the registry
    * \throws Error if name is not registered
    * \param name The name of the Runtime.
    * \param attrs Attributes for the Runtime.
    * \return the new Runtime object.
    */
-  TVM_DLL static Runtime Create(String name, Map<String, ObjectRef> attrs);
+  TVM_DLL static Runtime Create(String name, Map<String, ObjectRef> attrs = {});
 
   /*!
    * \brief List all registered Runtimes

python/tvm/autotvm/graph_tuner/utils/traverse_graph.py

@@ -144,9 +144,7 @@ def _traverse_expr(node):
                 mod = tvm.IRModule.from_expr(relay.Function(params, call))
                 relay.backend.te_compiler.get().clear()
                 tracing_target = _replace_device_with_tracing(tvm_target)
-                build_thread = threading.Thread(
-                    target=relay.build, args=(mod, tracing_target, None, None)
-                )
+                build_thread = threading.Thread(target=relay.build, args=(mod, tracing_target))
                 build_thread.start()
                 build_thread.join()
         elif isinstance(node, Var):