From 39724d54c52a6d55fabbb66a62712b42c84df571 Mon Sep 17 00:00:00 2001 From: Irina Korchakova Date: Fri, 3 Jul 2026 11:42:58 +0200 Subject: [PATCH 1/6] [EIEX-976] Add e2e test which creates etdump, etrecord and generate final profiling table with inspector --- backends/nxp/runtime/NeutronBackend.cpp | 2 +- .../nxp/tests/generic_tests/test_profiling.py | 114 +++++++++++++++++- backends/nxp/tests/nsys_testing.py | 9 ++ examples/nxp/executor_runner/CMakeLists.txt | 9 +- .../executor_runner/nxp_executor_runner.cpp | 46 ++++++- 5 files changed, 171 insertions(+), 9 deletions(-) diff --git a/backends/nxp/runtime/NeutronBackend.cpp b/backends/nxp/runtime/NeutronBackend.cpp index c8a1003243a..17c5066f146 100644 --- a/backends/nxp/runtime/NeutronBackend.cpp +++ b/backends/nxp/runtime/NeutronBackend.cpp @@ -586,7 +586,7 @@ class NeutronBackend final : public PyTorchBackendInterface { char* profile_info = static_cast(cfg->dcfg.outputs[profiling_index]); NeutronFullProfilingEvent* neutron_events = - reinterpreter_cast(profile_info); + reinterpret_cast(profile_info); executorch::runtime::EventTracer* tracer = context.event_tracer(); uint32_t start_time = 0; int index = 0; diff --git a/backends/nxp/tests/generic_tests/test_profiling.py b/backends/nxp/tests/generic_tests/test_profiling.py index d25330cae54..ccd03a81639 100644 --- a/backends/nxp/tests/generic_tests/test_profiling.py +++ b/backends/nxp/tests/generic_tests/test_profiling.py @@ -4,7 +4,9 @@ # LICENSE file in the root directory of this source tree. import ast import logging +import os import re +from typing import Any, Union import numpy as np import pytest @@ -13,17 +15,20 @@ from executorch.backends.nxp.tests.model_output_comparator import ( NumericalStatsOutputComparator, ) - from executorch.backends.nxp.tests.models import AvgPool2dModule, SoftmaxModule -from executorch.backends.nxp.tests.nsys_testing import lower_run_compare +from executorch.backends.nxp.tests.nsys_testing import ( + get_test_name, + lower_run_compare, + OUTPUTS_DIR, +) +from executorch.devtools.inspector._inspector import Inspector from executorch.examples.models.mlperf_tiny import ( DeepAutoEncoder, DSCNNKWS, MobileNetV1025, ResNet8, ) - from executorch.examples.nxp.experimental.cifar_net.cifar_net import CifarNetModel @@ -46,6 +51,103 @@ def extract_map_from_logs(caplog): return None +def inspector_check(test_name: str) -> None: + """ + Validate ExecuTorch Inspector profiling output. + + Checks: + 1. Required profiling artifacts (etrecord.bin, trace.etdump) exist. + 2. Inspector can be created and profiling data can be parsed. + 3. All numeric delegate events except the last contain + "Neutron kernel" metadata. + 4. The last numeric delegate event contains + "Profiling dump" metadata. + 5. The profiling dump event does not have associated op types. + """ + + def parse_delegate_metadata( + delegate_metadatas: list[bytes], + ) -> Union[list[str], dict[str, Any]]: + """Metadata parser for Neutron Backend metadata. + + The parser is a callable that deserializes the data and returns neutron kernel number. + The deserialized data is then added back to the corresponding event in the event block for user consumption. + """ + + metadata_list = [] + for metadata_bytes in delegate_metadatas: + if len(metadata_bytes) == 1: + function_code = metadata_bytes[0] + if function_code == 0: + metadata_list.append("Profiling dump") + else: + metadata_list.append("Neutron kernel " + str(function_code)) + else: + metadata_list.append("Invalid metadata size") + return metadata_list + + npu_results_path = os.path.join(OUTPUTS_DIR, test_name, "results_npu") + etrecord_path = os.path.join(npu_results_path, "etrecord.bin") + etdump_path = os.path.join(npu_results_path, "trace.etdump") + + # Verify profiling artifacts were generated. + for file_path in (etrecord_path, etdump_path): + assert os.path.isfile( + file_path + ), f"Required profiling file does not exist: {file_path}" + + # Create Inspector and parse profiling data. + try: + inspector = Inspector( + etdump_path=etdump_path, + etrecord=etrecord_path, + delegate_metadata_parser=parse_delegate_metadata, + ) + inspector.print_data_tabular(include_delegate_debug_data=True) + + except Exception as e: + raise RuntimeError( + "Failed to create or run Inspector for " + f"etdump='{etdump_path}', " + f"etrecord='{etrecord_path}'" + ) from e + + # Collect delegated profiling events whose names are numeric + # (0, 1, 2, ..., N). These events are emitted by the Neutron backend. + numeric_events = [ + event + for event_block in inspector.event_blocks + for event in event_block.events + if str(event.name).isdigit() + ] + + assert numeric_events, "No numeric delegate profiling events found" + + # All delegate events except the last one should describe + # individual Neutron kernels. + for event in numeric_events[:-1]: + metadata = str(event.delegate_debug_metadatas) + + assert "Neutron kernel" in metadata, ( + f"Event {event.name}: expected 'Neutron kernel', " f"got {metadata}" + ) + + # The final numeric event should represent the profiling dump. + profiling_dump_event = numeric_events[-1] + profiling_metadata = str(profiling_dump_event.delegate_debug_metadatas) + + assert "Profiling dump" in profiling_metadata, ( + f"Event {profiling_dump_event.name}: " + f"expected 'Profiling dump', got {profiling_metadata}" + ) + + # Profiling dump event is expected to have no associated operators. + assert profiling_dump_event.op_types == [], ( + f"Event {profiling_dump_event.name}: expected empty op_types, " + f"got {profiling_dump_event.op_types}" + ) + + class SimpleParallelPoolModel(torch.nn.Module): def __init__(self, channels: int): super().__init__() @@ -84,7 +186,10 @@ def forward(self, x): class TestProfiling: - @pytest.mark.xfail(reason="SoftMax support PR is not merged so far.", strict=True) + @pytest.mark.xfail( + reason="Profiling support for cmodel and SoftMax fix will be available in Neutron SW 3.2.", + strict=True, + ) def test__softmax(self, caplog, request): caplog.set_level(logging.INFO) model = SoftmaxModule(-1) @@ -164,6 +269,7 @@ def test__cifar(self, caplog, request): 10: (21,), # Slice 11: (), # Neutron Dump } + inspector_check(get_test_name(request)) def test__avg_pool(self, caplog, request): caplog.set_level(logging.INFO) diff --git a/backends/nxp/tests/nsys_testing.py b/backends/nxp/tests/nsys_testing.py index ef6fe9c864c..78a8f473024 100644 --- a/backends/nxp/tests/nsys_testing.py +++ b/backends/nxp/tests/nsys_testing.py @@ -153,6 +153,15 @@ def wrapper(*args, **kwargs): save_pte_program(delegated_program, test_name + "_delegated", test_dir) + # Generate ETRecord if profiling flag is set. + if use_profiling: + etrecord_path = os.path.join(npu_results_dir, "etrecord.bin") + # Create directory if it doesn't exist + os.makedirs(os.path.dirname(etrecord_path), exist_ok=True) + # Save ETRecord + delegated_program.get_etrecord().save(etrecord_path) + logging.info(f"The ETRecord for the model was saved to {etrecord_path}.") + # Preparation of quantized dataset, requires quantization parameters from converted delegated model if remove_quant_io_ops: dataset_dir_quant = os.path.join(test_dir, "dataset_quant") diff --git a/examples/nxp/executor_runner/CMakeLists.txt b/examples/nxp/executor_runner/CMakeLists.txt index 6a0c79f3402..ebe8b7befd9 100644 --- a/examples/nxp/executor_runner/CMakeLists.txt +++ b/examples/nxp/executor_runner/CMakeLists.txt @@ -60,6 +60,7 @@ if(NOT TARGET nd) ) add_library(nd STATIC IMPORTED GLOBAL) + add_library(flatccrt STATIC IMPORTED GLOBAL) set_target_properties( nd PROPERTIES IMPORTED_LOCATION ${EIQ_NEUTRON_TARGET_DRIVER} ) @@ -108,11 +109,15 @@ add_executable( ${EXECUTORCH_PATH}/extension/data_loader/file_data_loader.cpp ) -target_compile_options(nxp_executor_runner PRIVATE -DNEUTRON_CMODEL) +target_compile_options( + nxp_executor_runner PRIVATE -DNEUTRON_CMODEL -DEXECUTORCH_BUILD_DEVTOOLS=ON + -DEXECUTORCH_ENABLE_EVENT_TRACER=ON +) target_link_libraries( nxp_executor_runner - PRIVATE executorch + PRIVATE etdump + executorch executorch_kernels nd gflags::gflags diff --git a/examples/nxp/executor_runner/nxp_executor_runner.cpp b/examples/nxp/executor_runner/nxp_executor_runner.cpp index cfe2e5db3b2..4c677f9c8a3 100644 --- a/examples/nxp/executor_runner/nxp_executor_runner.cpp +++ b/examples/nxp/executor_runner/nxp_executor_runner.cpp @@ -13,6 +13,7 @@ * Neutron Backend. */ +#include #include #include #include @@ -204,6 +205,26 @@ Error saveOutputs( return Error::Ok; } +void saveETDump( + executorch::etdump::ETDumpGen* etdump_gen, + const std::string& outputPath) { + struct stat st; + const auto trace = etdump_gen->get_etdump_data(); + if (trace.buf && trace.size > 0) { + if (stat(outputPath.c_str(), &st) == -1) { + mkdir(outputPath.c_str(), 0700); + } + std::string fileName = outputPath + "/trace.etdump"; + printf("Saving file %s\n", fileName.c_str()); + FILE* f = fopen(fileName.c_str(), "w+"); + if (f != NULL) { + fwrite(static_cast(trace.buf), 1, trace.size, f); + fclose(f); + } + free(trace.buf); + } +} + template Error printClassificationOutput( const torch::executor::EValue& value, @@ -438,8 +459,23 @@ int main(int argc, char* argv[]) { &method_allocator, &planned_memory, &tmp_allocator); { - Result method = - program->load_method(method_name, &memory_manager); +#ifdef ET_EVENT_TRACER_ENABLED + // Create ETDumpGen before inference + auto etdump_gen_ptr = std::make_unique(); + executorch::etdump::ETDumpGen* etdump_gen = etdump_gen_ptr.get(); + printf("ET_EVENT_TRACER_ENABLED\n"); +#else + printf(" NO ET_EVENT_TRACER_ENABLED\n"); +#endif + Result method = program->load_method( + method_name, + &memory_manager, +#ifdef ET_EVENT_TRACER_ENABLED + etdump_gen +#else + nullptr +#endif + ); if (!method.ok()) { fprintf( stderr, @@ -572,6 +608,12 @@ int main(int argc, char* argv[]) { } } } + +#ifdef ET_EVENT_TRACER_ENABLED + // Save ETDump. + saveETDump(etdump_gen, FLAGS_output); +#endif + } // Destruct the method object before destroying the Neutron Device. printf("Finished...\n"); From 23bed6c77fdccf799a911e32aa450f790d8c49ee Mon Sep 17 00:00:00 2001 From: Irina Korchakova Date: Wed, 8 Jul 2026 14:32:03 +0200 Subject: [PATCH 2/6] Fix cmake --- examples/nxp/executor_runner/CMakeLists.txt | 27 ++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/examples/nxp/executor_runner/CMakeLists.txt b/examples/nxp/executor_runner/CMakeLists.txt index ebe8b7befd9..f9b777e71a3 100644 --- a/examples/nxp/executor_runner/CMakeLists.txt +++ b/examples/nxp/executor_runner/CMakeLists.txt @@ -60,7 +60,6 @@ if(NOT TARGET nd) ) add_library(nd STATIC IMPORTED GLOBAL) - add_library(flatccrt STATIC IMPORTED GLOBAL) set_target_properties( nd PROPERTIES IMPORTED_LOCATION ${EIQ_NEUTRON_TARGET_DRIVER} ) @@ -91,6 +90,31 @@ else() message(STATUS "Looking for executorch target -- found") endif() +message(STATUS "Looking for flatccrt") +if(NOT TARGET flatccrt) + message(STATUS "Looking for flatccrt -- not found, adding") + add_subdirectory( + "${EXECUTORCH_PATH}/third-party/flatcc" + "${CMAKE_CURRENT_BINARY_DIR}/flatcc" + EXCLUDE_FROM_ALL + ) +else() + message(STATUS "Looking for flatcc --found") +endif() + +message(STATUS "Looking for etdump") +if(NOT TARGET etdump) + message(STATUS "Looking for etdump -- not found, adding") + include_directories("${EXECUTORCH_PATH}/third-party/flatcc/include") + add_subdirectory( + "${EXECUTORCH_PATH}/devtools" + "${CMAKE_CURRENT_BINARY_DIR}/devtools" + EXCLUDE_FROM_ALL + ) +else() + message(STATUS "Looking for etdump --found") +endif() + message(STATUS "Looking for gflags") if(NOT TARGET gflags::gflags) message(STATUS "Looking for gflags -- not found, adding") @@ -120,6 +144,7 @@ target_link_libraries( executorch executorch_kernels nd + flatccrt gflags::gflags "-Wl,--whole-archive" executorch_delegate_neutron From 90ede512b13ab8de358a04453b2a5de763b439f4 Mon Sep 17 00:00:00 2001 From: Irina Korchakova Date: Wed, 8 Jul 2026 19:57:18 +0200 Subject: [PATCH 3/6] Remove temporary debug logging --- examples/nxp/executor_runner/nxp_executor_runner.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/nxp/executor_runner/nxp_executor_runner.cpp b/examples/nxp/executor_runner/nxp_executor_runner.cpp index 4c677f9c8a3..ed52ad84495 100644 --- a/examples/nxp/executor_runner/nxp_executor_runner.cpp +++ b/examples/nxp/executor_runner/nxp_executor_runner.cpp @@ -463,9 +463,6 @@ int main(int argc, char* argv[]) { // Create ETDumpGen before inference auto etdump_gen_ptr = std::make_unique(); executorch::etdump::ETDumpGen* etdump_gen = etdump_gen_ptr.get(); - printf("ET_EVENT_TRACER_ENABLED\n"); -#else - printf(" NO ET_EVENT_TRACER_ENABLED\n"); #endif Result method = program->load_method( method_name, From f399f055e8dbae276c3c504ee18f6a496dc294b2 Mon Sep 17 00:00:00 2001 From: Irina Korchakova Date: Wed, 8 Jul 2026 20:29:16 +0200 Subject: [PATCH 4/6] Fix lintrunner --- examples/nxp/executor_runner/CMakeLists.txt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/examples/nxp/executor_runner/CMakeLists.txt b/examples/nxp/executor_runner/CMakeLists.txt index f9b777e71a3..073f95d64a8 100644 --- a/examples/nxp/executor_runner/CMakeLists.txt +++ b/examples/nxp/executor_runner/CMakeLists.txt @@ -94,9 +94,8 @@ message(STATUS "Looking for flatccrt") if(NOT TARGET flatccrt) message(STATUS "Looking for flatccrt -- not found, adding") add_subdirectory( - "${EXECUTORCH_PATH}/third-party/flatcc" - "${CMAKE_CURRENT_BINARY_DIR}/flatcc" - EXCLUDE_FROM_ALL + "${EXECUTORCH_PATH}/third-party/flatcc" + "${CMAKE_CURRENT_BINARY_DIR}/flatcc" EXCLUDE_FROM_ALL ) else() message(STATUS "Looking for flatcc --found") @@ -107,9 +106,8 @@ if(NOT TARGET etdump) message(STATUS "Looking for etdump -- not found, adding") include_directories("${EXECUTORCH_PATH}/third-party/flatcc/include") add_subdirectory( - "${EXECUTORCH_PATH}/devtools" - "${CMAKE_CURRENT_BINARY_DIR}/devtools" - EXCLUDE_FROM_ALL + "${EXECUTORCH_PATH}/devtools" "${CMAKE_CURRENT_BINARY_DIR}/devtools" + EXCLUDE_FROM_ALL ) else() message(STATUS "Looking for etdump --found") From e3896717509f79f2897af04a2d070b8b964b74cc Mon Sep 17 00:00:00 2001 From: Irina Korchakova Date: Thu, 9 Jul 2026 12:11:24 +0200 Subject: [PATCH 5/6] Fix cmake --- examples/nxp/executor_runner/CMakeLists.txt | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/examples/nxp/executor_runner/CMakeLists.txt b/examples/nxp/executor_runner/CMakeLists.txt index 073f95d64a8..76e55b89d6f 100644 --- a/examples/nxp/executor_runner/CMakeLists.txt +++ b/examples/nxp/executor_runner/CMakeLists.txt @@ -74,7 +74,7 @@ else() message(STATUS "Neutron Driver -- found") endif() -# Check if the ExecuTorch Target is aleady defined, what indicate this +# Check if the ExecuTorch Target is already defined, what indicate this # CMakeLists.txt is involved as subproject: message(STATUS "Looking for executorch target") if(NOT TARGET executorch) @@ -82,6 +82,8 @@ if(NOT TARGET executorch) set(EXECUTORCH_BUILD_NXP_NEUTRON ON) set(EXECUTORCH_BUILD_KERNELS_QUANTIZED ON) set(EXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT ON) + set(EXECUTORCH_BUILD_DEVTOOLS ON) + set(EXECUTORCH_ENABLE_EVENT_TRACER ON) add_subdirectory( "${EXECUTORCH_PATH}" "${CMAKE_CURRENT_BINARY_DIR}/executorch" EXCLUDE_FROM_ALL @@ -101,18 +103,6 @@ else() message(STATUS "Looking for flatcc --found") endif() -message(STATUS "Looking for etdump") -if(NOT TARGET etdump) - message(STATUS "Looking for etdump -- not found, adding") - include_directories("${EXECUTORCH_PATH}/third-party/flatcc/include") - add_subdirectory( - "${EXECUTORCH_PATH}/devtools" "${CMAKE_CURRENT_BINARY_DIR}/devtools" - EXCLUDE_FROM_ALL - ) -else() - message(STATUS "Looking for etdump --found") -endif() - message(STATUS "Looking for gflags") if(NOT TARGET gflags::gflags) message(STATUS "Looking for gflags -- not found, adding") @@ -132,8 +122,7 @@ add_executable( ) target_compile_options( - nxp_executor_runner PRIVATE -DNEUTRON_CMODEL -DEXECUTORCH_BUILD_DEVTOOLS=ON - -DEXECUTORCH_ENABLE_EVENT_TRACER=ON + nxp_executor_runner PRIVATE -DNEUTRON_CMODEL -DET_EVENT_TRACER_ENABLED ) target_link_libraries( From 3c109ebbb56d208e557b079121f7738e4672d099 Mon Sep 17 00:00:00 2001 From: Irina Korchakova Date: Fri, 10 Jul 2026 11:43:33 +0200 Subject: [PATCH 6/6] Make event tracing optional in executor_runner --- examples/nxp/executor_runner/CMakeLists.txt | 43 +++++++++++-------- .../executor_runner/nxp_executor_runner.cpp | 2 + 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/examples/nxp/executor_runner/CMakeLists.txt b/examples/nxp/executor_runner/CMakeLists.txt index 76e55b89d6f..eb1d74b2206 100644 --- a/examples/nxp/executor_runner/CMakeLists.txt +++ b/examples/nxp/executor_runner/CMakeLists.txt @@ -6,6 +6,11 @@ cmake_minimum_required(VERSION 3.16) project(nxp_executor_runner C CXX) +option(EVENT_TRACER "Build with event tracing support" ON) + +set(NXP_RUNNER_DEFINES NEUTRON_CMODEL) +set(NXP_RUNNER_LIBS) + set(EIQ_NEUTRON_TARGET "imxrt700" CACHE STRING "Neutron Target to build the executor_runner" @@ -82,8 +87,10 @@ if(NOT TARGET executorch) set(EXECUTORCH_BUILD_NXP_NEUTRON ON) set(EXECUTORCH_BUILD_KERNELS_QUANTIZED ON) set(EXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT ON) - set(EXECUTORCH_BUILD_DEVTOOLS ON) - set(EXECUTORCH_ENABLE_EVENT_TRACER ON) + if(EVENT_TRACER) + set(EXECUTORCH_BUILD_DEVTOOLS ON) + set(EXECUTORCH_ENABLE_EVENT_TRACER ON) + endif() add_subdirectory( "${EXECUTORCH_PATH}" "${CMAKE_CURRENT_BINARY_DIR}/executorch" EXCLUDE_FROM_ALL @@ -92,15 +99,20 @@ else() message(STATUS "Looking for executorch target -- found") endif() -message(STATUS "Looking for flatccrt") -if(NOT TARGET flatccrt) - message(STATUS "Looking for flatccrt -- not found, adding") - add_subdirectory( - "${EXECUTORCH_PATH}/third-party/flatcc" - "${CMAKE_CURRENT_BINARY_DIR}/flatcc" EXCLUDE_FROM_ALL - ) -else() - message(STATUS "Looking for flatcc --found") +if(EVENT_TRACER) + list(APPEND NXP_RUNNER_DEFINES ET_EVENT_TRACER_ENABLED) + list(APPEND NXP_RUNNER_LIBS etdump flatccrt) + + message(STATUS "Looking for flatccrt") + if(NOT TARGET flatccrt) + message(STATUS "Looking for flatccrt -- not found, adding") + add_subdirectory( + "${EXECUTORCH_PATH}/third-party/flatcc" + "${CMAKE_CURRENT_BINARY_DIR}/flatcc" EXCLUDE_FROM_ALL + ) + else() + message(STATUS "Looking for flatcc -- found") + endif() endif() message(STATUS "Looking for gflags") @@ -121,19 +133,16 @@ add_executable( ${EXECUTORCH_PATH}/extension/data_loader/file_data_loader.cpp ) -target_compile_options( - nxp_executor_runner PRIVATE -DNEUTRON_CMODEL -DET_EVENT_TRACER_ENABLED -) +target_compile_definitions(nxp_executor_runner PRIVATE ${NXP_RUNNER_DEFINES}) target_link_libraries( nxp_executor_runner - PRIVATE etdump - executorch + PRIVATE executorch executorch_kernels nd - flatccrt gflags::gflags "-Wl,--whole-archive" executorch_delegate_neutron "-Wl,--no-whole-archive" + ${NXP_RUNNER_LIBS} ) diff --git a/examples/nxp/executor_runner/nxp_executor_runner.cpp b/examples/nxp/executor_runner/nxp_executor_runner.cpp index ed52ad84495..f3ee8aeb479 100644 --- a/examples/nxp/executor_runner/nxp_executor_runner.cpp +++ b/examples/nxp/executor_runner/nxp_executor_runner.cpp @@ -205,6 +205,7 @@ Error saveOutputs( return Error::Ok; } +#ifdef ET_EVENT_TRACER_ENABLED void saveETDump( executorch::etdump::ETDumpGen* etdump_gen, const std::string& outputPath) { @@ -224,6 +225,7 @@ void saveETDump( free(trace.buf); } } +#endif template Error printClassificationOutput(