[QNN EP] Added sustained high performance mode implementation

onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc

@@ -334,6 +334,7 @@ Ort::Status QnnBackendManager::LoadBackend() {
                                                                                           QNN_API_VERSION_PATCH},
                                                                                          &backend_interface_provider)));
   qnn_interface_ = backend_interface_provider->QNN_INTERFACE_VER_NAME;
+  htp_power_config_manager_.Init(qnn_interface_);
   backend_id_ = backend_interface_provider->backendId;
   backend_api_version_ = backend_interface_provider->apiVersion.backendApiVersion;
   SetQnnBackendType(backend_id_);
@@ -394,6 +395,7 @@ Ort::Status QnnBackendManager::LoadQnnSerializerBackend() {
        QNN_API_VERSION_PATCH},
       &serializer_interface_provider)));
   qnn_interface_ = serializer_interface_provider->QNN_INTERFACE_VER_NAME;  // NOTE: QnnSaver/Ir will provide the interfaces
+  htp_power_config_manager_.Init(qnn_interface_);
 
   Qnn_Version_t backend_interface_version = GetQnnInterfaceApiVersion(backend_interface_provider);
   Qnn_Version_t serializer_interface_version = GetQnnInterfaceApiVersion(serializer_interface_provider);
@@ -2001,9 +2003,17 @@ Ort::Status QnnBackendManager::SetupBackend(
   return status;
 }
 
-Ort::Status QnnBackendManager::CreateHtpPowerCfgId(uint32_t device_id,
-                                                   uint32_t core_id,
-                                                   uint32_t& htp_power_config_id) {
+Ort::Status QnnBackendManager::InitializePowerCfgId(uint32_t device_id, uint32_t core_id, uint32_t& htp_power_config_id) {
+  RETURN_IF_ERROR(CreateHtpPowerCfgId(device_id, core_id, htp_power_config_id));
+  htp_power_config_manager_.CreateTimerThread(htp_power_config_id);
+  return Ort::Status();
+}
+
+void QnnBackendManager::DeInitializePerfTimer() {
+  htp_power_config_manager_.ReleaseTimerThread();
+}
+
+Ort::Status QnnBackendManager::CreateHtpPowerCfgId(uint32_t device_id, uint32_t core_id, uint32_t& htp_power_config_id) {
   // This function is called in QNN EP's OnRunStart() even if QNN backend setup failed and the model is assigned
   // to a different EP. Therefore, we have to check that backend setup actually completed before trying to
   // create an HTP power config ID. Otherwise, this causes a segfault because the QNN backend lib is unloaded.
@@ -2023,26 +2033,6 @@ Ort::Status QnnBackendManager::CreateHtpPowerCfgId(uint32_t device_id,
   return Ort::Status();
 }
 
-Ort::Status QnnBackendManager::SetHtpPowerConfigs(uint32_t htp_power_config_client_id,
-                                                  HtpPerformanceMode htp_performance_mode,
-                                                  uint32_t rpc_polling_time,
-                                                  uint32_t rpc_control_latency) {
-  // This function is called in QNN EP's OnRunStart() even if QNN backend setup failed and the model is assigned
-  // to a different EP. Therefore, we have to check that backend setup actually completed before trying to
-  // set an HTP power config ID. Otherwise, this causes a segfault because the QNN backend lib is unloaded.
-  RETURN_IF_NOT(backend_setup_completed_, "Cannot set HTP power config ID if backend setup is not complete.");
-  RETURN_IF_ERROR(htp_power_config_manager_.AddRpcPollingTime(rpc_polling_time, *logger_ptr_));
-  RETURN_IF_ERROR(htp_power_config_manager_.AddRpcControlLatency(rpc_control_latency, *logger_ptr_));
-  RETURN_IF_ERROR(htp_power_config_manager_.AddHtpPerformanceMode(htp_performance_mode,
-                                                                  htp_power_config_client_id,
-                                                                  *logger_ptr_));
-  RETURN_IF_ERROR(htp_power_config_manager_.SetPowerConfig(htp_power_config_client_id,
-                                                           GetQnnInterface(),
-                                                           *logger_ptr_));
-
-  return Ort::Status();
-}
-
 Ort::Status QnnBackendManager::SetPerThreadHtpPowerConfigs(const std::thread::id& thread_id, bool pre_run) {
   PerThreadHtpPowerConfigs_t htp_power_configs;
   if (!GetPerThreadHtpPowerConfigMapping(thread_id, htp_power_configs)) {
@@ -2051,29 +2041,23 @@ Ort::Status QnnBackendManager::SetPerThreadHtpPowerConfigs(const std::thread::id
 
   auto htp_power_config_id = htp_power_configs.power_config_id;
   if (pre_run) {
+    // add in htp_power_configs the default power config id also so to run when we execute
     if (htp_power_configs.pre_run_perf_mode.has_value()) {
-      RETURN_IF_ERROR(htp_power_config_manager_.AddHtpPerformanceMode(*htp_power_configs.pre_run_perf_mode,
-                                                                      htp_power_config_id,
-                                                                      *logger_ptr_));
-    }
-
-    if (htp_power_configs.rpc_control_latency.has_value()) {
-      RETURN_IF_ERROR(htp_power_config_manager_.AddRpcControlLatency(*htp_power_configs.rpc_control_latency,
-                                                                     *logger_ptr_));
+      power::HtpPerfConfig_t config{htp_power_config_id, *htp_power_configs.pre_run_perf_mode, *htp_power_configs.rpc_polling_time, *htp_power_configs.rpc_control_latency};
+      RETURN_IF_ERROR(htp_power_config_manager_.SetState(power::GraphState::RUN_START, config, *logger_ptr_));
+    } else if (htp_power_configs.default_perf_mode.has_value()) {
+      power::HtpPerfConfig_t config{htp_power_config_id, *htp_power_configs.default_perf_mode, *htp_power_configs.rpc_polling_time, *htp_power_configs.rpc_control_latency};
+      RETURN_IF_ERROR(htp_power_config_manager_.SetState(power::GraphState::RUN_START, config, *logger_ptr_));
     }
-
-    if (htp_power_configs.rpc_polling_time.has_value()) {
-      RETURN_IF_ERROR(htp_power_config_manager_.AddRpcPollingTime(*htp_power_configs.rpc_polling_time,
-                                                                  *logger_ptr_));
+  } else {
+    if (htp_power_configs.post_run_perf_mode.has_value()) {
+      power::HtpPerfConfig_t config{htp_power_config_id, *htp_power_configs.post_run_perf_mode, *htp_power_configs.rpc_polling_time, *htp_power_configs.rpc_control_latency};
+      RETURN_IF_ERROR(htp_power_config_manager_.SetState(power::GraphState::RUN_DONE, config, *logger_ptr_));
+    } else if (htp_power_configs.default_perf_mode.has_value()) {
+      power::HtpPerfConfig_t config{htp_power_config_id, *htp_power_configs.default_perf_mode, *htp_power_configs.rpc_polling_time, *htp_power_configs.rpc_control_latency};
+      RETURN_IF_ERROR(htp_power_config_manager_.SetState(power::GraphState::RUN_DONE, config, *logger_ptr_));
     }
-  } else if (htp_power_configs.post_run_perf_mode.has_value()) {
-    RETURN_IF_ERROR(htp_power_config_manager_.AddHtpPerformanceMode(*htp_power_configs.post_run_perf_mode,
-                                                                    htp_power_config_id,
-                                                                    *logger_ptr_));
   }
-
-  RETURN_IF_ERROR(htp_power_config_manager_.SetPowerConfig(htp_power_config_id, GetQnnInterface(), *logger_ptr_));
-
   return Ort::Status();
 }
 
@@ -2108,7 +2092,7 @@ void QnnBackendManager::RemovePerThreadHtpPowerConfigMapping(const std::thread::
   per_thread_power_configs_.erase(thread_id);
 }
 
-Ort::Status QnnBackendManager::DestroyHTPPowerConfigID(uint32_t htp_power_config_id) {
+Ort::Status QnnBackendManager::DestroyHtpPowerConfigId(uint32_t htp_power_config_id) {
   QnnDevice_Infrastructure_t qnn_device_infra = nullptr;
   auto status = qnn_interface_.deviceGetInfrastructure(&qnn_device_infra);
   RETURN_IF(QNN_SUCCESS != status, "backendGetPerfInfrastructure failed.");

onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h

@@ -185,12 +185,11 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
       bool enable_htp_extended_udma_mode = false,
       bool enable_htp_prepare_only = false);
 
-  Ort::Status CreateHtpPowerCfgId(uint32_t deviceId, uint32_t coreId, uint32_t& htp_power_config_id);
+  Ort::Status InitializePowerCfgId(uint32_t deviceId, uint32_t coreId, uint32_t& htp_power_config_id);
+
+  void DeInitializePerfTimer();
 
-  Ort::Status SetHtpPowerConfigs(uint32_t htp_power_config_client_id,
-                                 HtpPerformanceMode htp_performance_mode,
-                                 uint32_t rpc_polling_time,
-                                 uint32_t rpc_control_latency);
+  Ort::Status DestroyHtpPowerConfigId(uint32_t htp_power_config_id);
 
   Ort::Status SetPerThreadHtpPowerConfigs(const std::thread::id& thread_id, bool pre_run);
 
@@ -264,8 +263,6 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
                                                      : backend_path.parent_path().string();
   }
 
-  Ort::Status DestroyHTPPowerConfigID(uint32_t htp_power_config_id);
-
   Ort::Status GetMaxSpillFillBufferSize(unsigned char* buffer,
                                         uint64_t buffer_length,
                                         uint64_t& max_spill_fill_buffer_size);
@@ -326,7 +323,13 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
   } FileMappingCallbackInfo_t;
 #endif
 
-  void ResetLogger(const Ort::Logger& logger) { logger_ptr_ = &logger; }
+  void ResetLogger(const Ort::Logger& logger) {
+    logger_ptr_ = &logger;
+  }
+
+  power::HtpPowerConfigManager& GetHtpPowerConfigManager() {
+    return htp_power_config_manager_;
+  }
 
  private:
   Ort::Status LoadBackend();
@@ -439,6 +442,8 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
 
   void* LibFunction(void* handle, const char* symbol, std::string& error_msg);
 
+  Ort::Status CreateHtpPowerCfgId(uint32_t deviceId, uint32_t coreId, uint32_t& htp_power_config_id);
+
   template <class T>
   inline T ResolveSymbol(void* lib_handle, const char* sym, const Ort::Logger& logger) {
     std::string error_msg = "";

onnxruntime/core/providers/qnn/builder/qnn_def.h

@@ -84,11 +84,18 @@ enum class HtpPerformanceMode : uint8_t {
   kHtpExtremePowerSaver,
 };
 
+// pre_run_perf_mode and post_run_perf_mode takes precedence over default_perf_mode. If pre_run_perf_mode is set,
+// it will be used for performance setting in OnRunStart().
+// If post_run_perf_mode is set, it will be used for performance setting in OnRunDone().
+// If default_perf_mode is set and pre_run_perf_mode or post_run_perf_mode is not set,
+// default_perf_mode will be used for performance setting in both OnRunStart() and OnRunDone().
+// rpc_control_latency and rpc_polling_time will be set beforehand in OnRunStart() as it depends on the performance mode set in OnRunStart().
 typedef struct PerThreadHtpPowerConfigs {
   std::optional<HtpPerformanceMode> pre_run_perf_mode;
   std::optional<HtpPerformanceMode> post_run_perf_mode;
   std::optional<uint32_t> rpc_control_latency;
   std::optional<uint32_t> rpc_polling_time;
+  std::optional<HtpPerformanceMode> default_perf_mode;
 
   uint32_t power_config_id = 0;
 } PerThreadHtpPowerConfigs_t;
@@ -130,17 +137,23 @@ bool IsQpuBackend(QnnBackendType backend_type);
 
 std::string QnnBackendTypeToString(QnnBackendType backend_type);
 
-// constexpr config values
+// latency values are in microseconds
 constexpr const int kSleepMinLatency = 40;
 constexpr const int kSleepLowLatency = 100;
 constexpr const int kSleepMediumLatency = 1000;
 constexpr const int kSleepHighLatency = 2000;
+constexpr const int kSleepHigherLatency = 65535;
+
+// constexpr config values
 constexpr const int kDcvsDisable = 0;
 constexpr const int kDcvsEnable = 1;
 constexpr const uint32_t kDisableRpcPolling = 0;
 constexpr const uint32_t kDisableRpcControlLatency = 0;
 constexpr const uint32_t kMaxRpcPolling = 9999;
 
+// Sustained high performance mode timer timeout duration in microseconds
+constexpr const uint64_t kDefaultTimerTimeoutUs = 300000;
+
 struct OnnxTensorInfo {
   ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(OnnxTensorInfo);
   OnnxTensorInfo(size_t index, int32_t data_type, std::vector<int64_t>&& shape) : index_(index), data_type_(data_type), shape_(std::move(shape)) {}