diff --git a/.github/styles/config/vocabularies/PSDK/accept.txt b/.github/styles/config/vocabularies/PSDK/accept.txt index 9611e6dcb..ec68dd68d 100644 --- a/.github/styles/config/vocabularies/PSDK/accept.txt +++ b/.github/styles/config/vocabularies/PSDK/accept.txt @@ -33,9 +33,13 @@ Zink [Mm]ulticast [Rr]enderers? [Ss]carthgap +[Ss]ubgraphs? [Tt]oolchain balenaEtcher bdebstrap fdisk +logits? mmdebstrap +perfsim umount +venv diff --git a/.rstcheck.cfg b/.rstcheck.cfg index 3c3e4dfd8..e867f2893 100644 --- a/.rstcheck.cfg +++ b/.rstcheck.cfg @@ -3,5 +3,5 @@ ignore_directives = ifconfig,graphviz,grid,grid-item,grid-item-card,card,dropdow # ignore the following languages because they require external compilers and # snippets in this repo depend on external code ignore_languages = c,cpp,c++ -ignore_substitutions = __PART_FAMILY_NAME__,__PART_FAMILY_DEVICE_NAMES__,__PRODUCT_LINE_NAME__,__SDK_BUILD_MACHINE__,__SDK_FULL_NAME__,__SDK_SHORT_NAME__,__SDK_INSTALL_FILE__,__SDK_INSTALL_DIR__,__SDK_DOWNLOAD_URL__,__LINUX_UBUNTU_VERSION_LONG__,__LINUX_UBUNTU_VERSION_SHORT__,__OPTEE_PLATFORM_FLAVOR__,__RTOS_UBUNTU_VERSION_LONG__,__WINDOWS_SUPPORTED_LONG__,__FEATURINGMATRIX__,__SYSFW_CORE_NAME__,__PCIE_BASE_ADDRESS__,__PCIE_DEVICE_ID__,__IMAGE_TYPE__,__SDK_OS_NAME__ +ignore_substitutions = __PART_FAMILY_NAME__,__PART_FAMILY_DEVICE_NAMES__,__PRODUCT_LINE_NAME__,__SDK_BUILD_MACHINE__,__SDK_FULL_NAME__,__SDK_SHORT_NAME__,__SDK_INSTALL_FILE__,__SDK_INSTALL_DIR__,__SDK_DOWNLOAD_URL__,__LINUX_UBUNTU_VERSION_LONG__,__LINUX_UBUNTU_VERSION_SHORT__,__OPTEE_PLATFORM_FLAVOR__,__RTOS_UBUNTU_VERSION_LONG__,__WINDOWS_SUPPORTED_LONG__,__FEATURINGMATRIX__,__SYSFW_CORE_NAME__,__PCIE_BASE_ADDRESS__,__PCIE_DEVICE_ID__,__IMAGE_TYPE__,__SDK_OS_NAME__,__AUDIOAI_SDK_VERSION__,trade ignore_messages = (Hyperlink target.*is not referenced\.) diff --git a/conf.py b/conf.py index c84b2b8c9..5aa8837cf 100644 --- a/conf.py +++ b/conf.py @@ -261,6 +261,12 @@ FAMILY = os.environ.get("DEVFAMILY", "") OS = os.environ.get("OS", "") + +# Audio AI docs prefix every shell command with a "$ " prompt symbol; strip it +# from the copy-button output so copied text is directly runnable. +if OS == "edgeai" and FAMILY == "AM62DX": + copybutton_prompt_text = "$ " + try: globals().update(importlib.import_module(f"configs.{FAMILY}.{FAMILY}_{OS}_tags").__dict__) except ModuleNotFoundError as exc: diff --git a/configs/AM62DX/AM62DX_edgeai_config.txt b/configs/AM62DX/AM62DX_edgeai_config.txt new file mode 100644 index 000000000..9520fbd15 --- /dev/null +++ b/configs/AM62DX/AM62DX_edgeai_config.txt @@ -0,0 +1,30 @@ +# General family replacement variables and configuration values for AM62D Audio AI build + +# This file is processed by Python scripts to define both replacement +# variable and configuration values for a device family build. +# For replacement variables, all variables should be listed on a single +# line, and all the variables should be listed on consecutive lines +# (i.e. no additional blank lines between the replacement variables). +# Similarly, all the configuration values should be listed on individual, +# consecutive lines (with no blank lines between the configuration values). +# +# Note that neither replacement variables nor configuration values may contain +# a colon (i.e. ":") in their name. However, values may contain colons. + +Replacement Variables +--------------------- +'__PART_FAMILY_NAME__' : 'AM62Dx' +'__PART_FAMILY_DEVICE_NAMES__' : 'AM62Dx' +'__PRODUCT_LINE_NAME__' : 'Sitara MPU' +'__SDK_FULL_NAME__' : 'Processor SDK Linux Audio AI AM62Dx' +'__SDK_SHORT_NAME__' : 'PSDK Audio AI' +'__SDK_DOWNLOAD_URL__' : '`AM62D-SDK-Download-page `__' +'__AUDIOAI_SDK_VERSION__' : '12.1' +'__TIDL_VERSION__' : '11_02_18_00' +'__TIDL_TOOLS_VERSION__' : '11.2.x' + +Configuration Values +-------------------- +'CONFIG_part_family' : 'AM62DX_family' +'CONFIG_part_variant' : 'AM62DX' +'CONFIG_sdk' : 'SITARA' diff --git a/configs/AM62DX/AM62DX_edgeai_tags.py b/configs/AM62DX/AM62DX_edgeai_tags.py new file mode 100644 index 000000000..a6627eba2 --- /dev/null +++ b/configs/AM62DX/AM62DX_edgeai_tags.py @@ -0,0 +1,6 @@ +# Project name and HTML title +project = u'Audio AI for AM62D' +html_title = 'Audio AI for AM62D Documentation' + +# The master toctree document. +master_doc = 'devices/AM62DX/edgeai/index' diff --git a/configs/AM62DX/AM62DX_edgeai_toc.txt b/configs/AM62DX/AM62DX_edgeai_toc.txt new file mode 100644 index 000000000..0faa09f81 --- /dev/null +++ b/configs/AM62DX/AM62DX_edgeai_toc.txt @@ -0,0 +1,6 @@ +devices/AM62DX/edgeai/index +devices/AM62DX/edgeai/introduction +devices/AM62DX/edgeai/model_zoo +devices/AM62DX/edgeai/model_compile_evaluate +devices/AM62DX/edgeai/model_inspect +devices/AM62DX/edgeai/performance diff --git a/source/devices/AM62DX/edgeai/index.rst b/source/devices/AM62DX/edgeai/index.rst new file mode 100644 index 000000000..1617e815e --- /dev/null +++ b/source/devices/AM62DX/edgeai/index.rst @@ -0,0 +1,21 @@ +.. _audioai-index: + +######################################### +AM62D Audio AI Software Developer's Guide +######################################### + +.. rubric:: Welcome to the AM62D Audio AI/DL Software Developer's Guide + +.. toctree:: + :maxdepth: 5 + :numbered: + + introduction + model_zoo + model_compile_evaluate + model_inspect + performance + ++--------------------------------+--------------------------------------------------------------------------------------------------+ +| .. Image:: /images/E2e.jpg | For technical support please post your questions at `http://e2e.ti.com `__. | ++--------------------------------+--------------------------------------------------------------------------------------------------+ diff --git a/source/devices/AM62DX/edgeai/introduction.rst b/source/devices/AM62DX/edgeai/introduction.rst new file mode 100644 index 000000000..ec75cc579 --- /dev/null +++ b/source/devices/AM62DX/edgeai/introduction.rst @@ -0,0 +1,131 @@ +.. include:: + +.. _audioai-introduction: + +####################### +Introduction & Overview +####################### + +This guide walks through the workflow for developing and deploying audio +deep-learning (Audio AI/DL) models on the |__PART_FAMILY_NAME__| platform, using +|__SDK_FULL_NAME__| |__AUDIOAI_SDK_VERSION__|. It is written for embedded +developers who want to run, compile, evaluate, and benchmark audio models on the +AM62D - either out of the box on the target with pre-compiled artifacts, or +through the x86 compile/evaluate workflow. + +Four **reference models** serve as worked examples throughout the guide, but the +workflow is not limited to them: the same compile/evaluate/inspect flow is a +bring-your-own-model (BYOM) flow that accepts any supported `ONNX +`__ model. More reference models will be added over time. + +The remaining sections cover: + +- :ref:`Audio AI Model Zoo ` - out-of-box demos on the AM62D + target (inference scripts and Jupyter notebooks) using pre-trained models and + pre-compiled artifacts. +- :ref:`Model Compile and Evaluate ` - the + x86 BYOM workflow built on ``edgeai-tidlrunner``: compile a model for the AM62D + and evaluate accuracy on a dataset. +- :ref:`Model Inspect ` - analyze the compiled subgraphs + and per-layer behavior with Model Inspector. +- :ref:`Performance benchmark ` - latency and memory + methodology with per-model tables. + +******************** +Development workflow +******************** + +The end-to-end path is the same for a reference model or your own model: + +#. **Obtain an ONNX model** - a reference graph shipped with the AudioAI ModelZoo, + or your own exported ONNX. +#. **Compile** it for the C7\ |trade| NPU on an x86 host (:ref:`Model Compile and + Evaluate `). +#. **Evaluate** accuracy or audio-quality metrics on a dataset. +#. **Inspect** the compiled subgraphs and per-layer analysis (:ref:`Model + Inspect `). +#. **Run** on the AM62D target using the compiled artifact + (:ref:`Audio AI Model Zoo `). +#. **Benchmark** latency and memory (:ref:`Performance `). + +**************** +Reference models +**************** + +Four reference models are provided across two task families, as worked examples +of the workflow. Each is delivered as an `ONNX `__ graph and, +except for GTCRN, is offloaded to the C7\ |trade| NPU through the TVM runtime +(TVM-RT); GTCRN is the one that runs ARM-only. See :ref:`the framework overview +` below for what that means. The same flow accepts your own +ONNX model, and more reference models will be added over time. + +Audio analysis and classification +================================= + +**VGGish11** (model ID ``ac-0101``) + Audio-clip classification. Input is a log-mel patch of shape ``(1, 1, 64, 126)``; + output is ``(1, 10)`` class logits plus a ``(1, 512)`` embedding. Runs at 8-bit, + fully offloaded to the C7\ |trade| NPU via TVM-RT (a single TIDL subgraph). + +**YAMNet** (model ID ``ac-0201``) + Audio-event classification. Input is a log-mel patch of shape ``(1, 1, 96, 64)``; + output is ``(1, 521)`` class scores. Runs at 8-bit, fully offloaded to the + C7\ |trade| NPU via TVM-RT (a single TIDL subgraph). + +Both classification models are evaluated on the UrbanSound8K dataset. + +Speech enhancement +================== + +**GCRN** (model ID ``ase-0201``) + Convolutional-recurrent speech enhancement (noise suppression) over a fixed 4-second + window. Input and output are complex spectrograms of shape ``(1, 2, 401, 161)``. + Runs at 16-bit and is offloaded to the C7\ |trade| NPU via TVM-RT + (``tidl_offload: true``). + +**GTCRN** (model ID ``ase-0100``) + Grouped temporal convolutional-recurrent speech enhancement. Input and output + are complex spectrograms of shape ``(1, 257, T, 2)`` with a dynamic time + dimension ``T``. Runs FP32 on the Arm core through the ONNX runtime + (``tidl_offload: false``) - it is the only supported model that is **not** + offloaded to the C7\ |trade| NPU. + +Both speech-enhancement models are evaluated on the VoiceBank-DEMAND-16k +dataset. + +.. _audioai-tvm-tidl: + +******************** +TVM + TIDL framework +******************** + +The AM62D audio models are compiled and deployed with **TI TVM**, TI's fork of +the `Apache TVM `__ deep-learning compiler. The target +accelerator is the **C7**\ |trade| **NPU** - TI's Neural Processing Unit, which +combines the **C7x DSP** with the **Matrix Multiplication Accelerator (MMA)** +(referred to as *C7x+MMA* in older documentation). + +TI TVM integrates **TIDL** (TI Deep Learning) as an accelerator backend: +TIDL-supported layers are offloaded to the MMA, while TVM generates **C7x DSP** +code for any remaining layers. Because the MMA and the C7x DSP together make up +the C7\ |trade| NPU, the result is a single deployable artifact in which the +**entire model runs on the C7**\ |trade| **NPU**. + +Compilation runs on an x86 host and proceeds in stages: + +#. **Model import** - the ONNX graph is imported into TVM. +#. **Partitioning** - using the TIDL import API, the graph is split into + TIDL-supported subgraphs (compiled by the TIDL network compiler) and the + remaining operators (handled by TVM C7x DSP code generation). +#. **Artifact generation** - both parts are packaged into one TVM-RT deployable + artifact (``deploy_lib.so`` + ``deploy_graph.json`` + ``deploy_param.params``). + +At run time on the AM62D, the TVM runtime loads the artifact and dispatches the +offloaded subgraphs to TIDL-RT on the C7\ |trade| NPU. + +.. figure:: ../../../images/audioai/tvm_tidl_flow.svg + :width: 80 % + :align: center + + TVM + TIDL compile-and-deploy flow for the AM62D audio models. Green blocks + are handled by TIDL; white blocks are handled by TVM. diff --git a/source/devices/AM62DX/edgeai/model_compile_evaluate.rst b/source/devices/AM62DX/edgeai/model_compile_evaluate.rst new file mode 100644 index 000000000..d3a04f26c --- /dev/null +++ b/source/devices/AM62DX/edgeai/model_compile_evaluate.rst @@ -0,0 +1,251 @@ +.. include:: + +.. _audioai-compile-evaluate: + +########################## +Model Compile and Evaluate +########################## + +The :ref:`Audio AI Model Zoo ` runs *pre-compiled* artifacts +on the |__PART_FAMILY_NAME__| target. This section covers the **x86 host +workflow** that produces those artifacts: how to compile an audio model for +the C7\ |trade| NPU, run host-emulation inference, and measure accuracy against a +dataset - using **edgeai-tidlrunner**. It runs entirely on an Ubuntu x86 PC; +nothing here touches the target. + +This is the workflow for *any* audio model, not just the reference models. The +four reference models below are worked examples. To bring your own model you +point the same ``tidlrunner-cli`` at your ONNX graph and a per-model YAML config, +using a reference config as a starting template. Which operators offload to the +C7\ |trade| NPU (vs. fall back to the Arm core) is determined by TIDL - see +`edgeai-tidl-tools `__ for +the authoritative list of supported operators. + +***************** +edgeai-tidlrunner +***************** + +edgeai-tidlrunner is a bring-your-own-model (BYOM) command-line wrapper over the +core `edgeai-tidl-tools `__ +compilation stack. A single YAML config per model drives every stage, and one +CLI - ``tidlrunner-cli`` - exposes the stages this section uses: + +.. list-table:: + :header-rows: 1 + :widths: 18 82 + + * - Command + - What it does + * - ``compile`` + - Imports the ONNX graph, quantizes and partitions it, and writes the + deployable TIDL/TVM-RT artifacts. + * - ``infer`` + - Runs the compiled model on the x86 host (PC emulation) to check output + correctness before deploying to the target. + * - ``evaluate`` + - Runs the compiled model over a dataset and reports accuracy or + speech-enhancement metrics. + * - ``inspect`` + - One-shot compile + inference + reference comparison that emits an + interactive HTML analysis report (see :ref:`Model Inspect `). + +All commands are run from the repository root (``edgeai-tidlrunner/``) and take +``--config_path `` pointing at the per-model YAML. The same config file is +reused across ``compile``, ``infer``, and ``evaluate``. + +.. _audioai-am62d-setup: + +***** +Setup +***** + +Model compilation runs on an x86 PC (Ubuntu Linux recommended). The AM62D audio +flow is self-contained: a single setup script installs the whole TVM toolchain +into a dedicated Python 3.10 environment. The TIDL tools target Python 3.10. +These steps assume `pyenv `__. + +#. **Clone the repository** (https://github.com/TexasInstruments/edgeai-tidlrunner): + + .. code-block:: console + + $ git clone https://github.com/TexasInstruments/edgeai-tidlrunner.git + $ cd edgeai-tidlrunner + +#. **Create and activate a dedicated Python 3.10 environment.** The AM62D flow + uses a TI TVM wheel; keep it in its own venv (``tidlrunner-am62d``) + so it never clobbers a standard ``tidlrunner`` setup: + + .. code-block:: console + + $ pyenv install 3.10 + $ pyenv virtualenv 3.10 tidlrunner-am62d + $ pyenv activate tidlrunner-am62d + +#. **Run the AM62D setup script.** With the venv active, one script installs + everything this flow needs: + + .. code-block:: console + + $ ./devices/setup_am62d.sh + + It downloads the ARM GCC 15.2 and C7000 CGT 5.0.0.LTS cross-toolchains into + ``tools/tidl_tools_package/bin/`` (skipped if already present), installs the + x86 TI TVM wheel - which bundles the AM62D x86 TIDL tools *inside* + the package - and installs ``tidlrunner[pc,audio]``, ``tools``, ``onnxruntime``, + and ``tidl_onnx_model_optimizer``. The ``[audio]`` extra pulls in the audio + packages (``librosa``, ``soundfile``, ``scipy``, ``pesq``, ``pystoi``, + ``scikit-learn``). No separate ``pip install`` is needed. + +#. **Source the env script once per shell.** TVM ``dlopen``\ s the TIDL runtime + (``libvx_tidl_rt.so``) before Python's own import runs, so the tool paths must be + live in the shell *before* ``tidlrunner-cli`` starts. Source (do **not** execute) + ``devices/am62d_env.sh`` once; every ``tidlrunner-cli`` command run afterward in + that shell picks the vars up: + + .. code-block:: console + + $ source devices/am62d_env.sh + + It exports: + + .. list-table:: + :header-rows: 1 + :widths: 26 74 + + * - Variable + - Value + * - ``TIDL_TOOLS_PATH`` + - ``/3rdparty/x86_tidl_tools/AM62D`` (resolved from the + active venv's installed ``tvm``) + * - ``LD_LIBRARY_PATH`` + - prepends ``TIDL_TOOLS_PATH`` so ``libvx_tidl_rt.so`` resolves + * - ``ARM64_GCC_PATH`` + - the aarch64 GNU toolchain installed by ``setup_am62d.sh`` under + ``tools/tidl_tools_package/bin/`` (override by exporting before sourcing) + * - ``CGT7X_ROOT`` + - the C7000 code-generation tools installed by ``setup_am62d.sh`` under + ``tools/tidl_tools_package/bin/`` (override by exporting before sourcing) + * - ``SOC`` + - ``am62d`` + +Datasets +======== + +Accuracy evaluation needs the reference datasets. Compilation alone can run on +random data (for latency-only measurement), but ``evaluate`` requires the real +sets. Download them from the repository root: + +.. code-block:: console + + $ # UrbanSound8K (~5.6 GB) - sound classification (VGGish11, YAMNet) + $ bash examples/audio/scripts/download_urbansound8k.sh + + $ # VoiceBank-DEMAND-16k (~2 GB) - speech enhancement (GCRN, GTCRN) + $ python3 examples/audio/scripts/download_voicebank_demand.py + +They land under ``data/datasets/UrbanSound8K/`` and +``data/datasets/VoiceBank-DEMAND-16k/`` - the paths the model configs expect. +The classification configs use UrbanSound8K **fold 10** (837 samples) as the +test set; the speech-enhancement configs use the VoiceBank-DEMAND-16k **test** set +(824 files). + +The reference ONNX models auto-download at compile time via ``.link`` files, so +fetching them ahead of time is **optional**. To pre-download all reference ONNX models: + +.. code-block:: console + + $ bash examples/audio/scripts/download_audio_models.sh + +******************** +Compile and evaluate +******************** + +Each model has one config that drives all stages. All four run under the +``tidlrunner-am62d`` environment with ``devices/am62d_env.sh`` already sourced +(see `Setup`_). Three models compile to the C7\ |trade| NPU through the TVM runtime; +GTCRN runs ARM-only through ONNX Runtime (no C7\ |trade| NPU offload) but uses the +same environment. The AM62D config paths are: + +.. list-table:: + :header-rows: 1 + :widths: 12 10 12 18 54 + + * - Model + - Model ID + - Runtime + - Tensor Bits + - Config path (relative to ``edgeai-tidlrunner/``) + * - VGGish11 + - ``ac-0101`` + - TVM-RT + - 8 + - ``data/configs/samples/models/audio/audio_classification/urbansound8k/vggish11_tvmrt_config.yaml`` + * - YAMNet + - ``ac-0201`` + - TVM-RT + - 8 + - ``data/configs/samples/models/audio/audio_classification/urbansound8k/yamnet_tvmrt_config.yaml`` + * - GCRN + - ``ase-0201`` + - TVM-RT + - 16 (LSTM), FP32 (Enc/Dec) + - ``data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gcrn_fixed_4sec_tvmrt_config.yaml`` + * - GTCRN + - ``ase-0100`` + - ONNX-RT + - FP32 + - ``data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gtcrn_dns3_config.yaml`` + +VGGish11 (TVM-RT, 8-bit) +======================== + +Runs on the C7\ |trade| NPU (``tidl_offload: true``): + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/audio_classification/urbansound8k/vggish11_tvmrt_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG + +``evaluate`` reports top-1 / top-5 / macro-F1 over UrbanSound8K fold 10. + +YAMNet (TVM-RT, 8-bit) +====================== + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/audio_classification/urbansound8k/yamnet_tvmrt_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG + +GCRN (TVM-RT, 16-bit) +===================== + +GCRN is C7\ |trade| NPU-offloaded at 16-bit (``tidl_offload: true``, fixed 4-second +input): + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gcrn_fixed_4sec_tvmrt_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG + +``evaluate`` reports PESQ / STOI / SI-SDR over the VoiceBank-DEMAND-16k test set. + +GTCRN (ONNX-RT, ARM-only) +========================= + +GTCRN has a dynamic time axis and TIDL-unsupported operators, so it runs on the +Arm Cortex-A cores through ONNX Runtime (``tidl_offload: false``) - no C7\ |trade| NPU +offload. It uses the same ``tidlrunner-am62d`` environment as the other models; +it simply does not offload to the DSP: + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gtcrn_dns3_config.yaml + $ tidlrunner-cli compile --config_path $CFG + $ tidlrunner-cli infer --config_path $CFG + $ tidlrunner-cli evaluate --config_path $CFG diff --git a/source/devices/AM62DX/edgeai/model_inspect.rst b/source/devices/AM62DX/edgeai/model_inspect.rst new file mode 100644 index 000000000..67b5fd91e --- /dev/null +++ b/source/devices/AM62DX/edgeai/model_inspect.rst @@ -0,0 +1,65 @@ +.. _audioai-model-inspect: + +############# +Model Inspect +############# + +Model Inspector analyzes a model on the x86 host after (or instead of) the +:ref:`compile and evaluate ` flow. It uses the same +edgeai-tidlrunner installation and environment described in +:ref:`Setup `. + +*************** +Model Inspector +*************** + +``tidlrunner-cli inspect`` is a one-shot analysis pipeline: it compiles the +model for TIDL, runs inference, re-runs a non-TIDL reference for comparison, +computes per-layer signal-to-noise ratios, and emits a **self-contained +interactive HTML report** - no need to run ``compile`` and ``infer`` +separately. The report opens in any browser and lands at: + +.. code-block:: text + + work_dirs/compile////inspect/modelinspector.html + +```` is the model's work directory, named after its model ID, +runtime, task, dataset, and model, e.g. +``ase-0201_tvmrt_speech_enhancement_voicebank_demand_16k_gcrn_fixed_4sec_onnx``. + +The report has three tabs: + +- **Model Summary** - acceleration ratio (layers on the C7x DSP vs. Arm + fallback), input/output specs, and an interactive ONNX graph. +- **Subgraphs** - the TIDL subgraph(s) offloaded to the DSP, with per-layer + identity and a quantization analysis (INT8 vs. FP32 activation histograms and + scatter plots). +- **Performance** - *estimated* per-layer processing time, DSP cycle breakdown + (total / IO / kernel), memory usage (DDR / L2 / MSMC), and SNR metrics. + +.. note:: + + All Performance-tab numbers are **estimates** from the compiler, not + on-target measurements. Model Inspector requires TIDL tools **11.02 or + later** (earlier versions emit SVG subgraph files it cannot read). + +GCRN showcase - an offloaded DSP subgraph +========================================= + +Because GCRN now compiles with ``tidl_offload: true`` on AM62D, its inspector +report shows a **real C7x DSP subgraph** rather than an all-Arm fallback: +the Subgraphs tab lists a single offloaded subgraph (16-bit) with its layer +mapping, and the Performance tab breaks that subgraph down by cycles and memory. +Generate it with the AM62D env sourced (see :ref:`Setup `): + +.. code-block:: console + + $ CFG=data/configs/samples/models/audio/speech_enhancement/voicebank_demand_16k/gcrn_fixed_4sec_tvmrt_config.yaml + $ tidlrunner-cli inspect --config_path $CFG + +For contrast, running ``inspect`` on the GTCRN (ONNX-RT) config produces a +report with **no** DSP subgraph - every layer is Arm fallback - which is what a +non-offloaded model looks like in the inspector. GTCRN also runs with +``tidl_offload: false``, so its work directory uses a ``notidl`` segment in +place of the ```` shown in the path template above, e.g. +``work_dirs/compile/AM62D/notidl//...``. diff --git a/source/devices/AM62DX/edgeai/model_zoo.rst b/source/devices/AM62DX/edgeai/model_zoo.rst new file mode 100644 index 000000000..a95d7a725 --- /dev/null +++ b/source/devices/AM62DX/edgeai/model_zoo.rst @@ -0,0 +1,314 @@ +.. include:: + +.. _audioai-model-zoo: + +################## +Audio AI Model Zoo +################## + +The AudioAI ModelZoo delivers out-of-box audio deep-learning demos that run +directly on the |__PART_FAMILY_NAME__| target. Each reference model ships as an +`ONNX `__ graph plus, where applicable, a pre-compiled +TVM-RT + TIDL artifact, so you can reproduce these examples on the AM62D without +compiling anything yourself. Two demo front-ends are provided: standalone +Python inference scripts and Jupyter notebooks. The same on-target runtime hosts +your own compiled artifacts once you build them through the x86 flow. This +section runs entirely **on the target**; the x86 compile/evaluate flow is covered +in :ref:`Model Compile and Evaluate `. + +.. note:: + + The models in this repository are made available for experimentation and + development. They are not intended for deployment in production. + +********** +What ships +********** + +.. list-table:: + :header-rows: 1 + :widths: 18 26 14 14 28 + + * - Model + - Task + - Model ID + - On target + - Demo + * - VGGish11 + - Sound classification (UrbanSound8K) + - ``ac-0101`` + - C7\ |trade| NPU (TVM-RT + TIDL, 8-bit) + - Script + notebook + * - YAMNet + - Sound classification (UrbanSound8K) + - ``ac-0201`` + - C7\ |trade| NPU (TVM-RT + TIDL, 8-bit) + - Script + notebook + * - GCRN + - Speech enhancement (VoiceBank-DEMAND-16k) + - ``ase-0201`` + - C7\ |trade| NPU (TVM-RT + TIDL, 16-bit) + - Script + notebook + * - GTCRN + - Speech enhancement (VoiceBank-DEMAND-16k) + - ``ase-0100`` + - Arm core (ONNX runtime, FP32) + - Script + notebook + +VGGish11, YAMNet, and GCRN each ship a pre-compiled TVM-RT + TIDL artifact and +run offloaded to the C7\ |trade| NPU. GTCRN ships **no** compiled artifact - it +runs FP32 on the Arm core through the ONNX runtime, packaged as a plain ONNX +Runtime artifact. + +*************** +On-target setup +*************** + +The steps below run on the AM62D target (aarch64), on the Linux command line of +the Processor SDK rootfs. The notebooks and scripts run in a Python virtual +environment that reuses the TIDL-enabled TVM runtime already present in the +rootfs. + +#. **Clone the repository.** Clone the AudioAI ModelZoo repository + (https://github.com/TexasInstruments/audioai-modelzoo): + + .. code-block:: console + + $ mkdir -p ~/tidl && cd ~/tidl + $ git clone https://github.com/TexasInstruments/audioai-modelzoo.git + $ cd audioai-modelzoo + +#. **Download the model artifacts.** Downloads the three pre-compiled + TVM-RT + TIDL artifacts (VGGish11, YAMNet, GCRN) plus the GTCRN ONNX Runtime + artifact into ``model_artifacts//am62d/``: + + .. code-block:: console + + $ ./download_models.sh -y + + The script defaults to an interactive selection menu; the ``-y`` flag + downloads everything non-interactively. Add ``-l`` to print the source URLs + without downloading. + +#. **Create the virtual environment.** Creates the venv at + ~/venv/modelzoo with ``--system-site-packages`` so the TIDL-enabled + runtime from the rootfs is visible: + + .. code-block:: console + + $ ./setup_venv.sh + +#. **Activate the venv.** Activate it in every new shell before running the + demos: + + .. code-block:: console + + $ source ~/venv/modelzoo/bin/activate + +********************** +Inference script demos +********************** + +Run each script from its own model directory in the activated venv. All four +models ship a command-line script. + +YAMNet (sound classification) +============================= + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/yamnet_sc + $ python3 yamnet_infer_audio.py --audio-file samples/miaow_16k.wav + +The script converts the audio into log-mel patches, runs each patch through +the TVM-RT + TIDL session, and averages the per-patch scores. For the bundled +``miaow_16k.wav`` clip the top classes are cat/animal sounds. Expected output +(abridged): + +.. code-block:: text + + ====================================================================== + YAMNET INFERENCE SUMMARY + ====================================================================== + + Audio File: miaow_16k.wav + Artifacts: ac-0201_tvmrt_audio_classification_urbansound8k_yamnet_onnx + Audio Duration: 6.73 seconds + Number of Patches: 7 + + ====================================================================== + TOP 10 PREDICTIONS (averaged across all patches): + ====================================================================== + Rank Score Class + ---------------------------------------------------------------------- + 1 ... Cat + 2 ... Animal + ... + ====================================================================== + PERFORMANCE METRICS: + ====================================================================== + Inference Time (all patches): 17.53 ms total | 7 patches + RTF = 0.0026 (0.018s / 6.73s) + ====================================================================== + +A second sample, ``samples/speech_whistling2.wav``, is also included. + +VGGish11 (sound classification) +=============================== + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/vggish11_sc + $ python3 vggish_infer_audio.py --audio-file sample_wav/139951-9-0-9.wav + +The bundled clip is UrbanSound8K class 9 (street music). Expected +output (abridged): + +.. code-block:: text + + ====================================================================== + VGGISH11 INFERENCE SUMMARY + ====================================================================== + + Audio File: 139951-9-0-9.wav + Artifacts: ac-0101_tvmrt_audio_classification_urbansound8k_vggish11_onnx + Audio Duration: 4.00 seconds + Top Class: Street music (Class 9) - Score: ... + + Performance: + Inference: 8.88 ms + Throughput: RTF = 0.0022 (0.009s / 4.00s) + ====================================================================== + +GCRN (speech enhancement) +========================= + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/gcrn_se + $ python3 gcrn_infer_audio.py --input sample_wav/noisy.wav + +The script writes the enhanced audio to ``enhanced.wav`` beside the input and a +per-block timing report to ``gcrn-benchmark.md``. Pass ``--output`` to change +the destination wav. Expected output (abridged): + +.. code-block:: text + + Machine: aarch64 SOC: am62d TIDL_VER: ... + Loading TVM + TIDL session... + Running inference on noisy.wav (9.77s)... + Wrote enhanced audio to sample_wav/enhanced.wav + + GCRN TVM+TIDL benchmark (3 chunk(s), 9.77s audio) + + block total (ms) per-chunk (ms) + ... + Real-Time Factor (inference / audio) = 3.9500 + + Wrote benchmark report to sample_wav/gcrn-benchmark.md + +GTCRN (speech enhancement) +========================== + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo/inference/gtcrn_se + $ python3 gtcrn_infer_audio.py --input sample_wav/mix.wav + +Unlike the other three models, GTCRN has no TVM-RT + TIDL artifact - the script +loads the ONNX graph directly and runs it on ``CPUExecutionProvider`` (Arm +core, FP32). It writes the enhanced audio to ``enh_onnx.wav`` beside the input; +pass ``--output`` to change the destination wav. Expected output (abridged): + +.. code-block:: text + + Machine: aarch64 + Model: .../ase-0100_onnxrt_speech_enhancement_voicebank_demand_16k_gtcrn_dns3_onnx/model/gtcrn_dns3.onnx + Loading ONNX Runtime session... + Running inference on mix.wav (9.77s)... + Wrote enhanced audio to sample_wav/enh_onnx.wav + + GTCRN ONNX Runtime (CPU) benchmark (9.77s audio) + + block total (ms) + ------------------------ ---------- + pre-processing (STFT) ... + model inference ... + post-processing (iSTFT) ... + end-to-end ... + + Real-Time Factor (inference / audio) = 0.0700 + +GTCRN is already faster than real time on the Arm core, so it needs no NPU +offload. + +.. _audioai-model-zoo-notebooks: + +********************** +Jupyter notebook demos +********************** + +All four reference models have a Jupyter notebook demo. Start Jupyter Lab from +the activated venv: + +.. code-block:: console + + $ cd ~/tidl/audioai-modelzoo + $ ./jupyter_lab_venv.sh + +The launcher prints a highlighted access URL. Open the URL in a browser on a +machine that can reach the target and enter the token ``tidl`` when prompted: + +.. code-block:: text + + Access URL: http://:8888/lab?token=tidl + +Jupyter Lab opens with the GCRN notebook loaded. To run a different model, pick +its notebook from the file browser in the left sidebar. The notebooks are: + +.. list-table:: + :header-rows: 1 + :widths: 22 40 + + * - Model + - Notebook + * - GCRN + - ``inference/gcrn_se/gcrn_inference.ipynb`` + * - GTCRN + - ``inference/gtcrn_se/gtcrn_inference.ipynb`` + * - VGGish11 + - ``inference/vggish11_sc/vggish_inference.ipynb`` + * - YAMNet + - ``inference/yamnet_sc/yamnet_inference.ipynb`` + +****************** +Compiled artifacts +****************** + +``download_models.sh`` places the artifacts under +``model_artifacts//am62d/``. Each model gets its own directory, +named after its model ID, containing a ``model/`` folder (the trained ONNX +model) and, where applicable, an ``artifacts/`` folder (the compiled model +artifacts). VGGish11, YAMNet, and GCRN each get a pre-compiled TVM-RT + TIDL +artifact: + +- ``ac-0101`` - ``ac-0101_tvmrt_audio_classification_urbansound8k_vggish11_onnx`` (VGGish11) +- ``ac-0201`` - ``ac-0201_tvmrt_audio_classification_urbansound8k_yamnet_onnx`` (YAMNet) +- ``ase-0201`` - ``ase-0201_tvmrt_speech_enhancement_voicebank_demand_16k_gcrn_fixed_4sec_onnx`` (GCRN) + +Each of those artifact directories' ``artifacts/`` folder holds the TVM-RT +deployment artifacts (``deploy_lib.so``, ``deploy_graph.json``, ``deploy_param.params``) +in both target (``.evm``, aarch64) and host (``.pc``, x86) variants. The demo +scripts and notebooks select the ``.evm`` variant automatically on the target. + +GTCRN (``ase-0100``) gets an ONNX Runtime artifact instead of a +compiled one: + +- ``ase-0100`` - ``ase-0100_onnxrt_speech_enhancement_voicebank_demand_16k_gtcrn_dns3_onnx`` (GTCRN) + +Its directory holds only the ``model/`` folder, with the ``gtcrn_dns3.onnx`` +graph - there is no ``artifacts/`` folder since no TIDL compilation is +involved. The demo script and notebook load the ONNX model directly with +``CPUExecutionProvider`` and run FP32 on the Arm core. Compiled TVM-RT + TIDL +artifacts for your own models are produced by the x86 workflow in +:ref:`Model Compile and Evaluate `. diff --git a/source/devices/AM62DX/edgeai/performance.rst b/source/devices/AM62DX/edgeai/performance.rst new file mode 100644 index 000000000..2a1471f27 --- /dev/null +++ b/source/devices/AM62DX/edgeai/performance.rst @@ -0,0 +1,136 @@ +.. include:: + +.. _audioai-performance: + +##################### +Performance Benchmark +##################### + +This section reports **inference performance** - latency and memory - for the +reference |__PART_FAMILY_NAME__| Audio AI models. The same methodology (perfsim +estimate + on-target measured) applies to any model you compile through the +workflow. It is **not** an accuracy or audio-quality benchmark: model quality +(classification accuracy, PESQ / STOI / SI-SDR) is measured by the ``evaluate`` +step in :ref:`Model Compile and Evaluate `. + +*********** +Methodology +*********** + +Two independent figures are reported for each model, and they must not be +confused: + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Figure + - How it is obtained + * - **perfsim estimate** + - Produced by the TIDL performance simulator during ``compile``. The + compiler writes a ``result.yaml`` per model under its work directory; + the ``perfsim`` latency, GMAC count, and estimated DDR transfer are + read directly from it. This is a *static estimate* for the C7\ |trade| + NPU subgraph - it does not include Arm-side pre/post-processing or + runtime overhead. + * - **on-target measured** + - Wall-clock latency timed on the AM62D target while running the + :ref:`Model Zoo ` inference scripts. For the + streaming speech-enhancement models it is also expressed as a + **real-time factor (RTF)** = processing time / audio duration; RTF < 1 + means the model keeps up with real-time audio. + +Only models compiled for the C7\ |trade| NPU (TVM-RT + TIDL) have a perfsim +estimate. GTCRN runs FP32 on the Arm core through the ONNX runtime and has no +TIDL subgraph, so perfsim does not apply to it. + +.. note:: + + The perfsim values below are read from ``result.yaml`` and are labelled + *estimated (perfsim)*. On-target measured latency and RTF are marked **TBD** + until measured on the AM62D EVM. + +***************** +Per-model results +***************** + +.. list-table:: + :header-rows: 1 + :widths: 16 12 12 20 14 26 + + * - Model + - Model ID + - Precision + - perfsim latency *(estimated)* + - GMACs + - DDR transfer *(estimated)* + * - VGGish11 + - ``ac-0101`` + - 8-bit + - 5.235 ms + - 1.1515 + - 10.73 MB + * - YAMNet + - ``ac-0201`` + - 8-bit + - 1.990 ms + - 0.0692 + - 3.55 MB + * - GCRN + - ``ase-0201`` + - 16-bit + - TBD + - TBD + - TBD + * - GTCRN + - ``ase-0100`` + - FP32 (Arm) + - N/A + - N/A + - N/A + +VGGish11 and YAMNet are compiled and have ``result.yaml`` perfsim values (shown +above). GCRN is compiled at 16-bit but its ``result.yaml`` is not yet captured, +so its perfsim figures are **TBD**. GTCRN is Arm-only (no TIDL subgraph), so +perfsim is **N/A**. + +On-target measured latency and RTF for all four models are **TBD** - to be +filled in from the Model Zoo inference scripts on the AM62D EVM: + +.. list-table:: + :header-rows: 1 + :widths: 20 20 30 30 + + * - Model + - On target + - Measured latency + - RTF + * - VGGish11 + - C7\ |trade| NPU + - TBD + - - (clip classifier) + * - YAMNet + - C7\ |trade| NPU + - TBD + - - (clip classifier) + * - GCRN + - C7\ |trade| NPU + - TBD + - TBD + * - GTCRN + - Arm core + - TBD + - TBD + +RTF is reported only for the streaming speech-enhancement models (GCRN, GTCRN); +VGGish11 and YAMNet classify a whole clip, so a real-time factor is not +meaningful for them. + +******************* +Scope & Limitations +******************* + +This section does **not** assess model quality. Classification accuracy and +speech-enhancement quality metrics are produced by ``tidlrunner-cli evaluate`` +and documented in +:ref:`Model Compile and Evaluate `. diff --git a/source/devices/AM62DX/edgeai/version.txt b/source/devices/AM62DX/edgeai/version.txt new file mode 100644 index 000000000..9df24b162 --- /dev/null +++ b/source/devices/AM62DX/edgeai/version.txt @@ -0,0 +1 @@ +12_01_00_00 diff --git a/source/images/audioai/tvm_tidl_flow.svg b/source/images/audioai/tvm_tidl_flow.svg new file mode 100644 index 000000000..7d5920868 --- /dev/null +++ b/source/images/audioai/tvm_tidl_flow.svg @@ -0,0 +1,4 @@ + + + +
ONNX audio model
Model import into TVM
Partitioning
TIDL import API
TVM C7™ DSP
code generation
TIDL network compiler
TVM-RT deployable artifact
(deploy_lib.so + graph.json + params)
compile time (x86)
run time (AM62D)
TVM runtime
TIDL-RT
C7™ NPU
TIDL unsupported ops
TIDL supported ops
Text is not SVG - cannot display \ No newline at end of file