Arm backend: support embedded PTE semihosting in executorch runner

- Allow `arm_executor_runner` to reuse `model_pte.h` in
  semihosting mode via `ET_COMPILED_PTE`
- Keep semihosting available for host-side prompt and tensor
  I/O while exercising the embedded-PTE execution path
- Make the semihosting file size configurable
- update the runner argument handling for semihosted embedded-PTE flows

Change-Id: Ia1b5596decd4350eaf243c36dc4da4d8df406f04
Signed-off-by: Xingguo Li <xingguo.li@arm.com>

Author: xingguo01

examples/arm/executor_runner/CMakeLists.txt

@@ -152,10 +152,7 @@ if(NOT ET_MODEL_PTE_ADDR
   )
 endif()
 
-if(NOT SEMIHOSTING
-   AND NOT ET_MODEL_PTE_ADDR
-   AND NOT "${ET_PTE_FILE_PATH}" STREQUAL ""
-)
+if(NOT ET_MODEL_PTE_ADDR AND NOT "${ET_PTE_FILE_PATH}" STREQUAL "")
   if(NOT EXISTS "${ET_PTE_FILE_PATH}")
     message(
       FATAL_ERROR
@@ -228,7 +225,7 @@ if(NOT CMAKE_SKIP_INSTALL_RULES AND TARGET ethosu_core_driver)
 endif()
 
 # Convert pte to header
-if(NOT "${ET_MODEL_PTE_ADDR}" AND NOT SEMIHOSTING)
+if(NOT ET_MODEL_PTE_ADDR AND NOT "${ET_PTE_FILE_PATH}" STREQUAL "")
   add_custom_target(
     gen_model_header DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/model_pte.h
   )
@@ -319,8 +316,7 @@ list(
 # EXECUTORCH_SELECT_OPS_MODEL to include ops automatically. If the pte contains
 # no undelegated ops, use neither.
 set(FOUND_OPS_IN_FILE FALSE)
-if(NOT SEMIHOSTING
-   AND NOT ET_MODEL_PTE_ADDR
+if(NOT ET_MODEL_PTE_ADDR
    AND NOT "${ET_PTE_FILE_PATH}" STREQUAL ""
    AND EXISTS "${ET_PTE_FILE_PATH}"
 )
@@ -337,7 +333,7 @@ if(NOT SEMIHOSTING
   endif()
 endif()
 
-if(SEMIHOSTING)
+if(SEMIHOSTING AND "${ET_PTE_FILE_PATH}" STREQUAL "")
   set(EXECUTORCH_SELECT_OPS_MODEL "")
   message(
     "gen_oplist: Building with semihosting, no model is used to auto generate ops from will use EXECUTORCH_SELECT_OPS_LIST=${EXECUTORCH_SELECT_OPS_LIST}"
@@ -506,7 +502,7 @@ target_compile_definitions(
   arm_executor_runner PRIVATE C10_USING_CUSTOM_GENERATED_MACROS
 )
 
-if(NOT "${ET_MODEL_PTE_ADDR}" AND NOT SEMIHOSTING)
+if(NOT ET_MODEL_PTE_ADDR AND NOT "${ET_PTE_FILE_PATH}" STREQUAL "")
   add_dependencies(arm_executor_runner gen_model_header)
 endif()
 
@@ -569,6 +565,10 @@ if(SEMIHOSTING)
   target_compile_definitions(arm_executor_runner PUBLIC SEMIHOSTING)
 endif()
 
+if(ET_PTE_FILE_PATH)
+  target_compile_definitions(arm_executor_runner PUBLIC ET_COMPILED_PTE)
+endif()
+
 # Memory buffer sizes for Executorch flow
 
 if(ET_ARM_BAREMETAL_METHOD_ALLOCATOR_POOL_SIZE)
@@ -579,6 +579,14 @@ if(ET_ARM_BAREMETAL_METHOD_ALLOCATOR_POOL_SIZE)
   )
 endif()
 
+if(DEFINED ET_ARM_BAREMETAL_SEMIHOSTING_FILE_ALLOCATOR_POOL_SIZE)
+  target_compile_definitions(
+    arm_executor_runner
+    PUBLIC
+      ET_ARM_BAREMETAL_SEMIHOSTING_FILE_ALLOCATOR_POOL_SIZE=${ET_ARM_BAREMETAL_SEMIHOSTING_FILE_ALLOCATOR_POOL_SIZE}
+  )
+endif()
+
 target_compile_definitions(
   arm_executor_runner
   PUBLIC

examples/arm/executor_runner/arm_executor_runner.cpp

@@ -18,8 +18,12 @@
  *                      a c-array named model_pte and put into model_pte.h
  *                      this is placed in network_model_sec linker section
  *                      that is controlled by your memory mode via the
- *                      ETHOSU_MODEL cmake parameter.
- *                      If SEMIHOSTING is define this is not used
+ *                      ETHOSU_MODEL cmake parameter. This is not used by the
+ *                      semihosting path, which either loads the model from a
+ *                      file or can reuse an embedded model with
+ *                      ET_COMPILED_PTE.
+ * ET_COMPILED_PTE    - In SEMIHOSTING mode, reuse the model embedded in
+ *                      model_pte.h instead of passing the PTE as a host file.
  * ET_NUM_INFERENCES  - Numbers of times to run the inference
  * ET_LOG_DUMP_INPUT  - Control if you want input to be dumped to the log.
  * ET_LOG_DUMP_OUTPUT     - Control if you want output to be dumped to the log.
@@ -61,12 +65,12 @@
  * as guidance if timeing adaptor values are set correctly.
  *
  * SEMIHOSTING - When using the FVP simulator it can be built to access your dev
- *               machines filesystem, this is used for testing models in
- *               unittest/pytest and a special version of the runner is built
- *               to read model and input as files and output is saved to the
- *               filesystem. The backends/arm/test/setup_testing.sh script will
- *               build this for you so you can use it from pytest to test with
- *               the FVP simulator.
+ *               machines filesystem. This is used both for unit-test style
+ *               flows that load model and input files from the host and for
+ *               host-driven prompt/input/output exchange while still reusing an
+ *               embedded PTE via ET_COMPILED_PTE. The
+ *               backends/arm/test/setup_testing.sh script builds the unittest
+ *               configuration used with the FVP simulator.
  *
  * Memory areas used:
  *    You might want to configure this differently on your HW, like maybe all
@@ -79,6 +83,14 @@
  * ET_ARM_BAREMETAL_METHOD_ALLOCATOR_POOL_SIZE            - Size of memory area
  *                                                          used when setting up
  *                                                          the model
+ * ET_ARM_BAREMETAL_SEMIHOSTING_FILE_ALLOCATOR_POOL_SIZE
+ *                                                        - Size of memory area
+ *                                                          used to hold
+ *                                                          semihosted files,
+ *                                                          including input
+ *                                                          tensors and, when
+ *                                                          applicable, an
+ *                                                          external PTE file
  * ET_ARM_BAREMETAL_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE - Size of memory area
  *                                                          used when running
  *                                                          inferences
@@ -140,24 +152,27 @@
  * files/memory
  */
 
-const size_t input_file_allocation_pool_size = 60 * 1024 * 1024;
+#if !defined(ET_ARM_BAREMETAL_SEMIHOSTING_FILE_ALLOCATOR_POOL_SIZE)
+#define ET_ARM_BAREMETAL_SEMIHOSTING_FILE_ALLOCATOR_POOL_SIZE (60 * 1024 * 1024)
+#endif
+const size_t input_file_allocation_pool_size =
+    ET_ARM_BAREMETAL_SEMIHOSTING_FILE_ALLOCATOR_POOL_SIZE;
 unsigned char __attribute__((
     section("input_data_sec"),
     aligned(16))) input_file_allocation_pool[input_file_allocation_pool_size];
-char* model_pte = nullptr;
+#endif
 
-#else
-#if defined(ET_MODEL_PTE_ADDR)
+#if defined(ET_MODEL_PTE_ADDR) && defined(ET_COMPILED_PTE)
+#error "ET_MODEL_PTE_ADDR and ET_COMPILED_PTE are mutually exclusive"
+#endif
 
-/**
- * Set ET_MODEL_PTE_ADDR to the memory address where your PTE is placed
- * e.g. if you for example flash it to 0x7000000 set
- * -DET_MODEL_PTE_ADDR=0x7000000 You can run the Corstone FVP with the --data
- * flag to place it on a address if you use the FVP.
- */
-char* model_pte = reinterpret_cast<char*>(ET_MODEL_PTE_ADDR);
+#if !defined(ET_MODEL_PTE_ADDR) && !defined(ET_COMPILED_PTE) && \
+    !defined(SEMIHOSTING)
+#error \
+    "One of ET_MODEL_PTE_ADDR, ET_COMPILED_PTE, or SEMIHOSTING must be defined"
+#endif
 
-#else
+#if !defined(ET_MODEL_PTE_ADDR) && defined(ET_COMPILED_PTE)
 /**
  * This header file is generated by the build process based on the .pte file
  * specified in the ET_PTE_FILE_PATH variable to the cmake build.
@@ -169,7 +184,6 @@ char* model_pte = reinterpret_cast<char*>(ET_MODEL_PTE_ADDR);
  */
 #include "model_pte.h"
 #endif
-#endif
 
 using executorch::aten::ScalarType;
 using executorch::aten::Tensor;
@@ -543,6 +557,7 @@ std::pair<char*, size_t> read_binary_file(
   fclose(fp);
   return std::make_pair(buffer, read_size);
 }
+
 #endif
 
 /// Holds all state needed for setup and run phases
@@ -557,7 +572,7 @@ struct RunnerContext {
   size_t executor_membase = 0;
   size_t program_data_len = 0;
   size_t input_memsize = 0;
-  size_t pte_size = 0;
+  size_t model_data_size = 0;
   bool bundle_io = false;
   Box<BufferDataLoader> loader;
   Box<Program> program;
@@ -581,21 +596,22 @@ struct RunnerContext {
 
 void runner_init(
     RunnerContext& ctx,
-    std::vector<std::pair<char*, size_t>> input_buffers,
-    size_t pte_size) {
+    const uint8_t* model_data,
+    size_t model_size,
+    std::vector<std::pair<char*, size_t>> input_buffers) {
   // Find the offset to the embedded Program.
-  const void* program_data = model_pte;
-  ctx.program_data_len = pte_size;
-  ctx.pte_size = pte_size;
+  const void* program_data = model_data;
+  ctx.program_data_len = model_size;
+  ctx.model_data_size = model_size;
 
 #if defined(ET_BUNDLE_IO)
   ctx.bundle_io = executorch::bundled_program::is_bundled_program(
-      reinterpret_cast<void*>(model_pte), ctx.pte_size);
+      const_cast<uint8_t*>(model_data), ctx.model_data_size);
   if (ctx.bundle_io) {
     // BundleIO bpte is provided, dig out the actual model from the data area
     Error status = executorch::bundled_program::get_program_data(
-        reinterpret_cast<void*>(model_pte),
-        ctx.pte_size,
+        const_cast<uint8_t*>(model_data),
+        ctx.model_data_size,
         &program_data,
         &ctx.program_data_len);
 
@@ -780,7 +796,7 @@ void runner_init(
     // Useful for testing
     ET_LOG(Info, "Input testset[%d] from bundled bpte", testset_idx);
     Error status = executorch::bundled_program::load_bundled_input(
-        *ctx.method.value(), model_pte, testset_idx);
+        *ctx.method.value(), model_data, testset_idx);
     ET_CHECK_MSG(
         status == Error::Ok,
         "load_bundled_input failed with status 0x%" PRIx32,
@@ -857,7 +873,7 @@ void log_mem_status(RunnerContext& ctx) {
   ET_LOG(
       Info,
       "model_pte_loaded_size:      %lu bytes. (pte size unknown when not baked into elf)",
-      static_cast<unsigned long>(ctx.pte_size));
+      static_cast<unsigned long>(ctx.model_data_size));
 #else
   ET_LOG(
       Info,
@@ -866,7 +882,7 @@ void log_mem_status(RunnerContext& ctx) {
   ET_LOG(
       Info,
       "model_pte_loaded_size:      %lu bytes.",
-      static_cast<unsigned long>(ctx.pte_size));
+      static_cast<unsigned long>(ctx.model_data_size));
 #endif
 
 #if defined(SEMIHOSTING)
@@ -1149,13 +1165,13 @@ void write_etdump(RunnerContext& ctx) {
 // cppcheck-suppress constParameterReference
 // ET_BUNDLE_IO verification passes ctx.method into devtools/bundled_program
 // helpers, which currently require a non-const Method&.
-bool verify_result(RunnerContext& ctx, const void* model_pte) {
+bool verify_result(RunnerContext& ctx, const void* model_data) {
   bool model_ok = false;
 #if defined(ET_BUNDLE_IO)
   if (ctx.bundle_io) {
     // Check result
     ErrorStats stats = compute_method_output_error_stats(
-        *ctx.method.value(), model_pte, testset_idx);
+        *ctx.method.value(), model_data, testset_idx);
     if (stats.status == Error::Ok) {
       ET_LOG(Info, "=== Error stats for testset %d ===", testset_idx);
       ET_LOG(Info, " mean_absolute_error: %f", stats.mean_abs_error);
@@ -1172,7 +1188,7 @@ bool verify_result(RunnerContext& ctx, const void* model_pte) {
 
     // Verify the result.
     Error status = verify_method_outputs(
-        *ctx.method.value(), model_pte, testset_idx, et_rtol, et_atol);
+        *ctx.method.value(), model_data, testset_idx, et_rtol, et_atol);
     if (status == Error::Ok) {
       ET_LOG(Info, "Model output match expected BundleIO bpte ref data.");
       ET_LOG(Info, "TEST: BundleIO index[%d] Test_result: PASS", testset_idx);
@@ -1194,14 +1210,14 @@ bool verify_result(RunnerContext& ctx, const void* model_pte) {
   }
 #else // defined(ET_BUNDLE_IO)
   (void)ctx;
-  (void)model_pte;
+  (void)model_data;
   // No checking done, assume true
   model_ok = true;
 #endif // defined(ET_BUNDLE_IO)
   return model_ok;
 }
 
-bool run_model(RunnerContext& ctx, const void* model_pte) {
+bool run_model(RunnerContext& ctx, const void* model_data) {
   Error status;
   ET_LOG(Info, "Starting running %d inferences...", num_inferences);
   int n = 0;
@@ -1229,7 +1245,7 @@ bool run_model(RunnerContext& ctx, const void* model_pte) {
 
   ET_LOG(Info, "%d inferences finished", num_inferences);
   print_outputs(ctx);
-  bool model_ok = verify_result(ctx, model_pte);
+  bool model_ok = verify_result(ctx, model_data);
   ET_LOG(Info, "Model run: %d", model_ok);
 
   return model_ok;
@@ -1240,6 +1256,14 @@ bool run_model(RunnerContext& ctx, const void* model_pte) {
 int main(int argc, const char* argv[]) {
 #if defined(SEMIHOSTING)
   ET_LOG(Info, "Running executor with parameter:");
+#if defined(ET_COMPILED_PTE)
+  if (argc < 5) {
+    ET_LOG(Fatal, "Not right number of parameters!");
+    ET_LOG(Fatal, "app -o output_basename -i input.bin [-i input2.bin]");
+    ET_LOG(Fatal, "Exiting!");
+    _exit(1);
+  }
+#else
   if (argc < 7) {
     ET_LOG(Fatal, "Not right number of parameters!");
     ET_LOG(
@@ -1248,6 +1272,7 @@ int main(int argc, const char* argv[]) {
     ET_LOG(Fatal, "Exiting!");
     _exit(1);
   }
+#endif
   ET_LOG(Info, "   %s", argv[0]);
   for (int i = 1; i < argc; i++) {
     ET_LOG(Info, "   %s %s", argv[i], argv[++i]);
@@ -1259,14 +1284,18 @@ int main(int argc, const char* argv[]) {
 
   executorch::runtime::runtime_init();
   std::vector<std::pair<char*, size_t>> input_buffers;
+  const uint8_t* model_data = nullptr;
+  size_t model_size = 0;
 
 #if defined(ET_MODEL_PTE_ADDR)
-  // pte not in a known array but just on a memory/flash address
-  // As we dont know the size we pick something big enough
-  // Actual model is read from this area.
-  size_t pte_size = 0x10000000;
-#else
-  size_t pte_size = sizeof(model_pte);
+  // Read the PTE from a fixed memory/flash address configured via
+  // -DET_MODEL_PTE_ADDR=<address>. Since the runner does not know the exact
+  // size up front, use a large upper bound for the buffer span.
+  model_data = reinterpret_cast<const uint8_t*>(ET_MODEL_PTE_ADDR);
+  model_size = 0x10000000;
+#elif defined(ET_COMPILED_PTE)
+  model_data = model_pte;
+  model_size = sizeof(model_pte);
 #endif
 
   RunnerContext ctx;
@@ -1307,10 +1336,8 @@ int main(int argc, const char* argv[]) {
         _exit(1);
       }
 
-      // Store the model data with the same variable as if it was loaded
-      // from compiled in location.
-      model_pte = buffer;
-      pte_size = buffer_size;
+      model_data = reinterpret_cast<const uint8_t*>(buffer);
+      model_size = buffer_size;
     } else if (std::strcmp(argv[i], "-o") == 0) {
       // store the base filename to write output to.
       ctx.output_basename = argv[++i];
@@ -1320,17 +1347,23 @@ int main(int argc, const char* argv[]) {
 
   // Byte 4-7 is usually a nice magic number that could be good to print to make
   // sure it's OK ETxx for PTE and BPxx for bundled pte where xx is a number.
+  // cppcheck-suppress knownConditionTrueFalse
+  if (model_data == nullptr || model_size == 0) {
+    ET_LOG(Fatal, "Model data is not initialized");
+    return 1;
+  }
   ET_LOG(
       Info,
       "PTE @ %p [----%c%c%c%c]",
-      model_pte,
-      model_pte[4],
-      model_pte[5],
-      model_pte[6],
-      model_pte[7]);
-
-  runner_init(ctx, input_buffers, pte_size);
-  bool model_ok = run_model(ctx, model_pte);
+      model_data,
+      model_data[4],
+      model_data[5],
+      model_data[6],
+      model_data[7]);
+
+  runner_init(ctx, model_data, model_size, input_buffers);
+  bool model_ok = true;
+  model_ok = run_model(ctx, model_data);
   ET_LOG(Info, "Model run: %d", model_ok);
 
   log_mem_status(ctx);