diff --git a/src/controller/src/rocprofvis_controller_analysis.cpp b/src/controller/src/rocprofvis_controller_analysis.cpp index 65504c9cc..e8e5082d2 100644 --- a/src/controller/src/rocprofvis_controller_analysis.cpp +++ b/src/controller/src/rocprofvis_controller_analysis.cpp @@ -505,7 +505,7 @@ rocprofvis_result_t Analysis::EventsTable::UnpackArguments(Arguments& args, Tabl result = args.GetUInt64(kRPVControllerTableArgsSortOrder, 0, &sort_order); sys_out->m_sort_column = sort_column_index; sys_out->m_sort_order = (rocprofvis_controller_sort_order_t)sort_order; - sys_out->m_where = ""; + sys_out->m_processor = nullptr; sys_out->m_filter = "__op = " + std::to_string(m_op); sys_out->m_group = (m_op == kRocProfVisDmOperationLaunchSample) ? "name, COUNT(*) AS Invocations, SUM(duration) AS DurationTotal" : "name, COUNT(*) AS Invocations, SUM(duration) AS DurationTotal, AVG(duration) AS DurationAvg, MIN(duration) AS DurationMin, MAX(duration) AS DurationMax"; diff --git a/src/controller/src/system/rocprofvis_controller_summary.cpp b/src/controller/src/system/rocprofvis_controller_summary.cpp index 0685e9bd6..40caff578 100644 --- a/src/controller/src/system/rocprofvis_controller_summary.cpp +++ b/src/controller/src/system/rocprofvis_controller_summary.cpp @@ -509,21 +509,22 @@ rocprofvis_result_t Summary::FetchTopKernels(rocprofvis_dm_trace_t dm_handle, No { rocprofvis_dm_result_t dm_result = kRocProfVisDmResultUnknownError; uint32_t op[1] = { TABLE_QUERY_PACK_OP_TYPE(kRocProfVisDmOperationDispatch) }; - std::string where_str; + rocprofvis_dm_processor_identifiers_t processor_id; if(node) { uint64_t node_id = 0; result = node->GetUInt64(kRPVControllerNodeId, 0, &node_id); if(result == kRocProfVisResultSuccess) { - where_str = "nodeId = " + std::to_string(node_id); + processor_id.node_id = &node_id; if(processor) { uint64_t agent_id = 0; result = processor->GetUInt64(kRPVControllerProcessorId, 0, &agent_id); + agent_id &= TOPOLOGY_ID_MASK; if(result == kRocProfVisResultSuccess) { - where_str += " AND agentId = " + std::to_string(agent_id & TOPOLOGY_ID_MASK); + processor_id.agent_id = &agent_id; } } } @@ -534,7 +535,7 @@ rocprofvis_result_t Summary::FetchTopKernels(rocprofvis_dm_trace_t dm_handle, No dm_result = rocprofvis_db_build_table_query(db, kRPVDMTableUseCaseEventTrackTable, static_cast(m_start_ts), static_cast(m_end_ts), 1, (rocprofvis_db_track_selection_t)op, - where_str.empty() ? nullptr : where_str.c_str(), nullptr, + &processor_id, nullptr, "name, COUNT(*) AS num_invocations, AVG(duration) AS avg_duration, MIN(duration) AS min_duration, MAX(duration) AS max_duration, SUM(duration) AS total_duration", "name", sort_column.c_str(), kRPVDMSortOrderDesc, 0, 0, false, &query); diff --git a/src/controller/src/system/rocprofvis_controller_table_system.cpp b/src/controller/src/system/rocprofvis_controller_table_system.cpp index b5a04cc72..549b9c12e 100644 --- a/src/controller/src/system/rocprofvis_controller_table_system.cpp +++ b/src/controller/src/system/rocprofvis_controller_table_system.cpp @@ -468,7 +468,6 @@ SystemTable::UnpackArguments(Arguments& args, TableArguments*& out) const uint64_t num_op_types = 0; double end_ts = 0; double start_ts = 0; - std::string where; std::string filter; std::string group; std::string group_cols; @@ -570,16 +569,6 @@ SystemTable::UnpackArguments(Arguments& args, TableArguments*& out) const } } - if(result == kRocProfVisResultSuccess) - { - uint32_t length = 0; - result = args.GetString(kRPVControllerTableArgsWhere, 0, nullptr, &length); - if(result == kRocProfVisResultSuccess) - { - where.resize(length); - result = args.GetString(kRPVControllerTableArgsWhere, 0, where.data(), &length); - } - } if(result == kRocProfVisResultSuccess) { uint32_t length = 0; @@ -612,7 +601,6 @@ SystemTable::UnpackArguments(Arguments& args, TableArguments*& out) const } if(result == kRocProfVisResultSuccess) { - sys_out->m_where = std::move(where); sys_out->m_filter = std::move(filter); sys_out->m_group = std::move(group); sys_out->m_group_cols = std::move(group_cols); @@ -634,7 +622,6 @@ SystemTable::GetCurrentArguments(TableArguments*& out) const } SystemTableArguments* sys_out = (SystemTableArguments*)out; Table::GetCurrentArguments(out); - sys_out->m_where = m_where; sys_out->m_filter = m_filter; sys_out->m_group = m_group; sys_out->m_group_cols = m_group_cols; @@ -649,7 +636,6 @@ SystemTable::SetCurrentArguments(TableArguments& in) { SystemTableArguments& sys_in = (SystemTableArguments&)in; Table::SetCurrentArguments(sys_in); - m_where = sys_in.m_where; m_filter = sys_in.m_filter; m_group = sys_in.m_group; m_group_cols = sys_in.m_group_cols; @@ -664,7 +650,7 @@ SystemTable::ArgumentsChanged(SystemTableArguments& in) const { bool result = true; if(m_tracks.size() == in.m_tracks.size() && m_start_ts == in.m_start_ts && - m_end_ts == in.m_end_ts && m_where == in.m_where && m_filter == in.m_filter && + m_end_ts == in.m_end_ts && m_filter == in.m_filter && m_group == in.m_group && m_group_cols == in.m_group_cols && m_use_case == in.m_use_case) { @@ -728,7 +714,7 @@ SystemTable::BuildQuery(rocprofvis_dm_database_t db, TableArguments& args, uint6 result = rocprofvis_db_build_table_query(db, arguments.m_use_case, static_cast(arguments.m_start_ts), static_cast(arguments.m_end_ts), static_cast(arguments.m_tracks.size()), arguments.m_tracks.data(), - arguments.m_where.c_str(), arguments.m_filter.c_str(), + nullptr, arguments.m_filter.c_str(), arguments.m_group.c_str(), arguments.m_group_cols.c_str(), sort_column, (rocprofvis_dm_sort_order_t)arguments.m_sort_order, count_only ? 0 : count, count_only ? 0 : index, count_only, out); diff --git a/src/controller/src/system/rocprofvis_controller_table_system.h b/src/controller/src/system/rocprofvis_controller_table_system.h index 4e1fb3d4c..7f37744b9 100644 --- a/src/controller/src/system/rocprofvis_controller_table_system.h +++ b/src/controller/src/system/rocprofvis_controller_table_system.h @@ -38,7 +38,7 @@ class SystemTable : public Table protected: struct SystemTableArguments : TableArguments { - std::string m_where; + rocprofvis_dm_processor_identifiers_ptr m_processor; std::string m_filter; std::string m_group; std::string m_group_cols; diff --git a/src/controller/src/system/rocprofvis_controller_table_system_search.cpp b/src/controller/src/system/rocprofvis_controller_table_system_search.cpp index 7de8b069b..d3655d485 100644 --- a/src/controller/src/system/rocprofvis_controller_table_system_search.cpp +++ b/src/controller/src/system/rocprofvis_controller_table_system_search.cpp @@ -151,7 +151,7 @@ EventSearchTable::BuildQuery(rocprofvis_dm_database_t db, TableArguments& args, result = rocprofvis_db_build_event_search_query(db, static_cast(arguments.m_start_ts), static_cast(arguments.m_end_ts), static_cast(arguments.m_tracks.size()), arguments.m_tracks.data(), - arguments.m_where.c_str(), + nullptr, static_cast(string_table_filters_ptr.size()), string_table_filters_ptr.data(), arguments.m_include_substrings, arguments.m_include_category, arguments.m_partial_matching, sort_column, (rocprofvis_dm_sort_order_t)arguments.m_sort_order, diff --git a/src/model/CMakeLists.txt b/src/model/CMakeLists.txt index 2b411d7f0..f2055944d 100644 --- a/src/model/CMakeLists.txt +++ b/src/model/CMakeLists.txt @@ -4,8 +4,6 @@ project(datamodel VERSION 0.1.0 LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 17) -add_subdirectory(../../thirdparty/sqlite3 thirdparty/sqlite3) - include_directories(inc ${CMAKE_CURRENT_SOURCE_DIR}/src/common ${CMAKE_CURRENT_SOURCE_DIR}/src/database ${CMAKE_CURRENT_SOURCE_DIR}/src/datamodel ${CMAKE_CURRENT_SOURCE_DIR}/../controller/inc) set(CPACK_PROJECT_NAME ${PROJECT_NAME}) @@ -39,12 +37,26 @@ set(WRAPPER_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/src/database/rocprofvis_db_trace_processor_dll.cpp) list(REMOVE_ITEM DATABASE_SRC ${WRAPPER_SOURCE}) +if(NOT PROFILER_HUB_BUILD_ROC_OPTIQ_IFACE_LIB) +message(STATUS "Profiler-hub disabled") +add_subdirectory(../../thirdparty/sqlite3 thirdparty/sqlite3) +set(PROFILER_HUB_IFACE_SRC + ${CMAKE_CURRENT_SOURCE_DIR}/src/database/profiler_hub_lib_interface.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/database/rocprofvis_db_profiler_hub.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/database/rocprofvis_db_profiler_hub_client.cpp) +list(REMOVE_ITEM DATABASE_SRC ${PROFILER_HUB_IFACE_SRC}) +else() +message(STATUS "Profiler-hub enabled") +add_definitions(-DUSE_PROFILER_HUB) +endif() + add_library(${PROJECT_NAME} ${DATAMODEL_SRC} ${DATABASE_SRC} ${COMMON_SRC} ) target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../core/inc) target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/jsoncpp) target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/perfetto/trace_processor/include) target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/perfetto/trace_processor/gen/build_config) + # Only needed on Windows when Perfetto is available if(WIN32 AND PERFETTO_AVAILABLE) set_source_files_properties( @@ -55,7 +67,14 @@ if(WIN32 AND PERFETTO_AVAILABLE) endif() target_link_libraries(${PROJECT_NAME} roc-optiq-core) +if(PROFILER_HUB_BUILD_ROC_OPTIQ_IFACE_LIB) +target_link_libraries(${PROJECT_NAME} profiler-hub) +target_link_libraries(${PROJECT_NAME} optiq_interface) +target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/profiler-hub-client/optiq) +get_target_property(SQLITE3_INCLUDE_DIR profiler-hub-sqlite3-static INCLUDE_DIRECTORIES) +else() target_link_libraries(${PROJECT_NAME} sqlite3) +endif() target_link_libraries(${PROJECT_NAME} spdlog) target_link_libraries(${PROJECT_NAME} yaml-cpp) target_link_libraries(${PROJECT_NAME} json) @@ -110,6 +129,23 @@ if(BUILD_TESTING) add_test(NAME ${SYSTEM_TEST_NAME}-DB COMMAND ${SYSTEM_TEST_NAME} --input_file "${CMAKE_SOURCE_DIR}/sample/rocpd-transpose.db") set_tests_properties(${SYSTEM_TEST_NAME}-DB PROPERTIES RUN_SERIAL ON) set_tests_properties(${SYSTEM_TEST_NAME}-DB PROPERTIES WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + + if(PROFILER_HUB_BUILD_ROC_OPTIQ_IFACE_LIB) + add_executable(${PROFILER_HUB_TEST_NAME} src/tests/rocprofvis_dm_profiler_hub_tests.cpp) + target_include_directories(${PROFILER_HUB_TEST_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../core/inc) + target_include_directories(${PROFILER_HUB_TEST_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/jsoncpp) + target_include_directories(${PROFILER_HUB_TEST_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/profiler-hub-client/optiq) + target_link_libraries(${PROFILER_HUB_TEST_NAME} PRIVATE ${PROJECT_NAME}) + target_link_libraries(${PROFILER_HUB_TEST_NAME} PRIVATE Catch2::Catch2WithMain) + target_link_libraries(${PROFILER_HUB_TEST_NAME} PRIVATE roc-optiq-core) + target_link_libraries(${PROFILER_HUB_TEST_NAME} PRIVATE spdlog) + target_link_libraries(${PROFILER_HUB_TEST_NAME} PRIVATE optiq_interface) + target_link_libraries(${PROFILER_HUB_TEST_NAME} PRIVATE json) + add_test(NAME ${PROFILER_HUB_TEST_NAME}-DB COMMAND ${SYSTEM_TEST_NAME} --input_file "${CMAKE_SOURCE_DIR}/sample/rocpd-transpose.db") + set_tests_properties(${PROFILER_HUB_TEST_NAME}-DB PROPERTIES RUN_SERIAL ON) + set_tests_properties(${PROFILER_HUB_TEST_NAME}-DB PROPERTIES WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + endif() + add_executable(${COMPUTE_TEST_NAME} src/tests/rocprofvis_dm_compute_tests.cpp) target_include_directories(${COMPUTE_TEST_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../core/inc) target_include_directories(${COMPUTE_TEST_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/jsoncpp) diff --git a/src/model/inc/rocprofvis_interface.h b/src/model/inc/rocprofvis_interface.h index 8d7eb1655..4968b4ad4 100644 --- a/src/model/inc/rocprofvis_interface.h +++ b/src/model/inc/rocprofvis_interface.h @@ -192,7 +192,7 @@ rocprofvis_dm_result_t rocprofvis_db_build_table_query( rocprofvis_dm_timestamp_t end, rocprofvis_db_num_of_tracks_t num, rocprofvis_db_track_selection_t tracks, - rocprofvis_dm_charptr_t where, + rocprofvis_dm_processor_identifiers_ptr processor, rocprofvis_dm_charptr_t filter, rocprofvis_dm_charptr_t group, rocprofvis_dm_charptr_t group_cols, @@ -212,7 +212,7 @@ rocprofvis_dm_result_t rocprofvis_db_build_event_search_query( rocprofvis_dm_timestamp_t end, rocprofvis_db_num_of_tracks_t num, rocprofvis_db_track_selection_t ops, - rocprofvis_dm_charptr_t where, + rocprofvis_dm_processor_identifiers_ptr processor, rocprofvis_dm_num_string_table_filters_t num_string_table_filters, rocprofvis_dm_string_table_filters_t string_table_filters, bool include_substring, diff --git a/src/model/inc/rocprofvis_interface_types.h b/src/model/inc/rocprofvis_interface_types.h index c181a8ae3..dbcbe2c0e 100644 --- a/src/model/inc/rocprofvis_interface_types.h +++ b/src/model/inc/rocprofvis_interface_types.h @@ -136,19 +136,21 @@ typedef enum rocprofvis_dm_event_operation_t { // Database type typedef enum rocprofvis_db_type_t { // input file auto-detection - kAutodetect = 0, + kAutodetect, + // supported by profiler hub + kProfilerHub , // old schema Rocpd database - kRocpdSqlite = 1, + kRocpdSqlite , // new schema Rocprof database - kRocprofSqlite = 2, + kRocprofSqlite, // new schema Rocprof multinode database - kRocprofMultinodeSqlite = 3, + kRocprofMultinodeSqlite, // compute database - kComputeSqlite = 4, + kComputeSqlite, // chrom trace or perfetto - kChromeTrace = 5, - kPerfettoTrace = 6, - kGoogleSqlite = 7 + kChromeTrace, + kPerfettoTrace, + kGoogleSqlite } rocprofvis_db_type_t; @@ -430,6 +432,12 @@ typedef enum rocprofvis_dm_sort_order_t { kRPVDMSortOrderDesc, } rocprofvis_dm_sort_order_t; +typedef struct rocprofvis_dm_processor_identifiers_t +{ + uint64_t* node_id; + uint64_t* agent_id; +} rocprofvis_dm_processor_identifiers_t, * rocprofvis_dm_processor_identifiers_ptr; + // Tags for hrocprofvis_dm_hashed_timestamp typedef enum rocprofvis_dm_hashed_timestamp_tag_t { diff --git a/src/model/inc/rocprofvis_shared_types.h b/src/model/inc/rocprofvis_shared_types.h index 3bf9ea1e8..ceb157e45 100644 --- a/src/model/inc/rocprofvis_shared_types.h +++ b/src/model/inc/rocprofvis_shared_types.h @@ -168,7 +168,7 @@ typedef enum rocprofvis_controller_process_properties_t : uint32_t typedef enum rocprofvis_controller_topology_node_properties_t : uint32_t { __kRPVControllerTopologyNodePropertiesFirst = 0xF6000000, - kRPVControllerTopologyNodeTrackId = __kRPVControllerProcessPropertiesFirst, + kRPVControllerTopologyNodeTrackId = __kRPVControllerNodePropertiesFirst, kRPVControllerTopologyNodeType, kRPVControllerTopologyNodeName, kRPVControllerTopologyNodeTrack, diff --git a/src/model/src/common/rocprofvis_c_interface.cpp b/src/model/src/common/rocprofvis_c_interface.cpp index f28d2010a..71f394f98 100644 --- a/src/model/src/common/rocprofvis_c_interface.cpp +++ b/src/model/src/common/rocprofvis_c_interface.cpp @@ -3,6 +3,7 @@ #include "rocprofvis_c_interface.h" #include "rocprofvis_core_profile.h" +#include "rocprofvis_db_profiler_hub.h" #include "rocprofvis_db_rocpd.h" #include "rocprofvis_db_rocprof.h" #include "rocprofvis_dm_trace.h" @@ -59,6 +60,26 @@ rocprofvis_dm_database_t rocprofvis_db_open_database( PROFILE; std::vector multinode_files; if (db_type == rocprofvis_db_type_t::kAutodetect) { +#ifdef USE_PROFILER_HUB + profiler_hub_db_type_t ph_db_type = profiler_hub_db_identify_type(filename); + if (ph_db_type == kDbSupported) + { + try { + RocProfVis::DataModel::Database* db = new RocProfVis::DataModel::ProfilerHub(filename); + if (kRocProfVisDmResultSuccess == db->Open()) { + return db; + } else { + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN("Error! Failed to open database via Profiler Hub!", + nullptr); + } + } + catch(std::exception ex) + { + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN( + RocProfVis::DataModel::ERROR_MEMORY_ALLOCATION_FAILURE, nullptr); + } + } +#endif #ifdef ROCPROFVIS_PERFETTO_ENABLED db_type = RocProfVis::DataModel::GoogleTraceProcessor::Detect(filename); if (db_type == rocprofvis_db_type_t::kAutodetect) @@ -195,7 +216,7 @@ rocprofvis_dm_database_t rocprofvis_db_open_database_multi( files.push_back(filenames[i]); } try { - RocProfVis::DataModel::Database* db = + RocProfVis::DataModel::SystemDatabase* db = new RocProfVis::DataModel::RocprofDatabase(files.front().c_str(), files); if (kRocProfVisDmResultSuccess == db->Open()) { return db; @@ -369,7 +390,7 @@ rocprofvis_dm_result_t rocprofvis_db_read_trace_slice_async( ROCPROFVIS_ASSERT_MSG_RETURN(database, RocProfVis::DataModel::ERROR_DATABASE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - RocProfVis::DataModel::Database* db = (RocProfVis::DataModel::Database*) database; + RocProfVis::DataModel::SystemDatabase* db = (RocProfVis::DataModel::SystemDatabase*) database; return db->ReadTraceSliceAsync(start,end,tag,num,tracks,object); } @@ -387,7 +408,7 @@ rocprofvis_db_read_trace_pmc_slice_async( ROCPROFVIS_ASSERT_MSG_RETURN(database, RocProfVis::DataModel::ERROR_DATABASE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - RocProfVis::DataModel::Database* db = (RocProfVis::DataModel::Database*) database; + RocProfVis::DataModel::SystemDatabase* db = (RocProfVis::DataModel::SystemDatabase*) database; return db->ReadTracePMCSliceAsync(start,end,tag,track,left_neighbor,right_neighbor,object); } @@ -395,7 +416,7 @@ rocprofvis_dm_result_t rocprofvis_db_build_table_query( rocprofvis_dm_database_t database, rocprofvis_dm_table_use_case_enum_t use_case, rocprofvis_dm_timestamp_t start, rocprofvis_dm_timestamp_t end, rocprofvis_db_num_of_tracks_t num, rocprofvis_db_track_selection_t tracks, - rocprofvis_dm_charptr_t where, rocprofvis_dm_charptr_t filter, + rocprofvis_dm_processor_identifiers_ptr processor, rocprofvis_dm_charptr_t filter, rocprofvis_dm_charptr_t group, rocprofvis_dm_charptr_t group_cols, rocprofvis_dm_charptr_t sort_column, rocprofvis_dm_sort_order_t sort_order, uint64_t max_count, uint64_t offset, bool count_only, @@ -407,12 +428,12 @@ rocprofvis_dm_result_t rocprofvis_db_build_table_query( kRocProfVisDmResultInvalidParameter); ROCPROFVIS_ASSERT_MSG_RETURN(out_query, "Error! Query cannot be null.", kRocProfVisDmResultInvalidParameter); - RocProfVis::DataModel::Database* db = (RocProfVis::DataModel::Database*) database; + RocProfVis::DataModel::SystemDatabase* db = (RocProfVis::DataModel::SystemDatabase*) database; std::string query; rocprofvis_dm_result_t result = db->BuildTableQuery(use_case, start, end, num, tracks, - where, filter, + processor, filter, group, group_cols, sort_column, sort_order, max_count, offset, count_only, query); @@ -432,7 +453,7 @@ rocprofvis_dm_result_t rocprofvis_db_build_event_search_query( rocprofvis_dm_database_t database, rocprofvis_dm_timestamp_t start, rocprofvis_dm_timestamp_t end, rocprofvis_db_num_of_tracks_t num, rocprofvis_db_track_selection_t ops, - rocprofvis_dm_charptr_t where, + rocprofvis_dm_processor_identifiers_ptr processor, rocprofvis_dm_num_string_table_filters_t num_string_table_filters, rocprofvis_dm_string_table_filters_t string_table_filters, bool include_substring, bool include_category, bool partial_matching, rocprofvis_dm_charptr_t sort_column, rocprofvis_dm_sort_order_t sort_order, @@ -445,11 +466,11 @@ rocprofvis_dm_result_t rocprofvis_db_build_event_search_query( kRocProfVisDmResultInvalidParameter); ROCPROFVIS_ASSERT_MSG_RETURN(out_query, "Error! Query cannot be null.", kRocProfVisDmResultInvalidParameter); - RocProfVis::DataModel::Database* db = (RocProfVis::DataModel::Database*) database; + RocProfVis::DataModel::SystemDatabase* db = (RocProfVis::DataModel::SystemDatabase*) database; std::string query; rocprofvis_dm_result_t result = db->BuildEventSearchQuery(start, end, num, ops, - where, + processor, num_string_table_filters, string_table_filters, include_substring, include_category, partial_matching, sort_column, sort_order, @@ -477,7 +498,7 @@ rocprofvis_dm_result_t rocprofvis_db_build_compute_query( kRocProfVisDmResultInvalidParameter); ROCPROFVIS_ASSERT_MSG_RETURN(out_query, "Error! Query cannot be null.", kRocProfVisDmResultInvalidParameter); - RocProfVis::DataModel::Database* db = (RocProfVis::DataModel::Database*) database; + RocProfVis::DataModel::ComputeDatabase* db = (RocProfVis::DataModel::ComputeDatabase*) database; std::string query; rocprofvis_dm_result_t result = db->BuildComputeQuery(use_case, num, params, query); if (result == kRocProfVisDmResultSuccess) @@ -506,7 +527,7 @@ rocprofvis_dm_result_t rocprofvis_db_export_table_csv_async( kRocProfVisDmResultInvalidParameter); ROCPROFVIS_ASSERT_MSG_RETURN(file_path, "Error! Output path cannot be null.", kRocProfVisDmResultInvalidParameter); - RocProfVis::DataModel::Database* db = (RocProfVis::DataModel::Database*) database; + RocProfVis::DataModel::SystemDatabase* db = (RocProfVis::DataModel::SystemDatabase*) database; return db->ExportTableCSVAsync(query, file_path, object); } @@ -524,7 +545,7 @@ rocprofvis_dm_result_t rocprofvis_db_trim_save_async(rocprofvis_dm_database_t da kRocProfVisDmResultInvalidParameter); ROCPROFVIS_ASSERT_MSG_RETURN(new_db_path, "Error! Database path cannot be null.", kRocProfVisDmResultInvalidParameter); - RocProfVis::DataModel::Database* db = (RocProfVis::DataModel::Database*) database; + RocProfVis::DataModel::SystemDatabase* db = (RocProfVis::DataModel::SystemDatabase*) database; return db->SaveTrimmedDataAsync(start, end, new_db_path, object); } @@ -554,7 +575,7 @@ rocprofvis_dm_result_t rocprofvis_db_read_event_property_async( ROCPROFVIS_ASSERT_MSG_RETURN(database, RocProfVis::DataModel::ERROR_DATABASE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - RocProfVis::DataModel::Database* db = (RocProfVis::DataModel::Database*) database; + RocProfVis::DataModel::SystemDatabase* db = (RocProfVis::DataModel::SystemDatabase*) database; return db->ReadEventPropertyAsync(type, event_id, object); } @@ -583,7 +604,7 @@ rocprofvis_dm_result_t rocprofvis_db_execute_query_async( ROCPROFVIS_ASSERT_MSG_RETURN(database, RocProfVis::DataModel::ERROR_DATABASE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - RocProfVis::DataModel::Database* db = (RocProfVis::DataModel::Database*) database; + RocProfVis::DataModel::SystemDatabase* db = (RocProfVis::DataModel::SystemDatabase*) database; return db->ExecuteQueryAsync(query, description, object, id); } @@ -612,7 +633,7 @@ rocprofvis_dm_result_t rocprofvis_db_execute_compute_query_async( ROCPROFVIS_ASSERT_MSG_RETURN(database, RocProfVis::DataModel::ERROR_DATABASE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - RocProfVis::DataModel::Database* db = (RocProfVis::DataModel::Database*) database; + RocProfVis::DataModel::ComputeDatabase* db = (RocProfVis::DataModel::ComputeDatabase*) database; return db->ExecuteComputeQueryAsync(use_case, query, object, id); } diff --git a/src/model/src/common/rocprofvis_common_types.h b/src/model/src/common/rocprofvis_common_types.h index 5d5f55f4c..a93506e1f 100644 --- a/src/model/src/common/rocprofvis_common_types.h +++ b/src/model/src/common/rocprofvis_common_types.h @@ -92,7 +92,8 @@ typedef enum rocprofvis_db_system_source_type_t { kRPVSystemSourceRocpd, kRPVSystemSourceRocprof, - kRPVSystemSourcePerfetto + kRPVSystemSourcePerfetto, + kRPVSystemSourceProfillerHub }rocprofvis_db_system_source_type_t; typedef struct rocprofvis_dm_track_identifiers_t diff --git a/src/model/src/database/profiler_hub_lib_interface.cpp b/src/model/src/database/profiler_hub_lib_interface.cpp new file mode 100644 index 000000000..c5c5fc3d9 --- /dev/null +++ b/src/model/src/database/profiler_hub_lib_interface.cpp @@ -0,0 +1,331 @@ +#include "profiler_hub_lib_interface.h" +#include "spdlog/spdlog.h" +#define ANSI_COLOR_ERROR "\x1b[31m" + +const char* error_fmt = ANSI_COLOR_ERROR"Profiler hub failed : {}"; + +profiler_hub_future_handle_t profiler_hub_future_alloc(profiler_hub::progress_callback_t progress_callback) { + try { + return profiler_hub::interface::FutureAlloc(progress_callback); + } + catch (const std::exception& e) + { + spdlog::error(error_fmt, e.what()); + return nullptr; + } +} + +profiler_hub_result_t profiler_hub_future_free(profiler_hub_future_handle_t handle) { + try { + return profiler_hub::interface::FutureFree(handle); + } + catch (const std::exception& e) + { + spdlog::error(error_fmt, e.what()); + return kProfilerHubStatusUnknownError; + } +} + +profiler_hub_result_t profiler_hub_future_wait(profiler_hub_future_handle_t handle, uint64_t timeout_ms) { + try { + return profiler_hub::interface::FutureWait(handle, timeout_ms); + } + catch (const std::exception& e) + { + spdlog::error(error_fmt, e.what()); + return kProfilerHubStatusUnknownError; + } +} + +profiler_hub_result_t profiler_hub_future_cancel(profiler_hub_future_handle_t handle) { + try { + return profiler_hub::interface::FutureCancel(handle); + } + catch (const std::exception& e) + { + spdlog::error(error_fmt, e.what()); + return kProfilerHubStatusUnknownError; + } +} + +profiler_hub_db_type_t profiler_hub_db_identify_type( + profiler_hub_string_t trace_file_path +) { + profiler_hub_db_type_t type = kDbNotSupported; + try { + type = profiler_hub::interface::DetectTrace(trace_file_path); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + type = kDbNotSupported; + } + return type; +} + + +profiler_hub_trace_handle_t profiler_hub_open_trace( + profiler_hub_string_t trace_file_path +) { + profiler_hub_trace_handle_t trace = nullptr; + try { + trace = profiler_hub::interface::OpenTrace(trace_file_path); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + trace = nullptr; + } + return trace; +} + +profiler_hub_result_t profiler_hub_set_trace_properties(profiler_hub_trace_handle_t trace, + client_trace_handle_t client_trace, + profiler_hub_string_t config_path, + size_t histogram_bucket_count) +{ + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::SetTraceProperties(trace, client_trace, config_path, histogram_bucket_count); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} + + +profiler_hub_result_t profiler_hub_read_metadata( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace +) { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::ReadTraceMetadata(future_handle, trace); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} + +profiler_hub_result_t profiler_hub_close_trace( + profiler_hub_trace_handle_t trace +) { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::CloseTrace(trace); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} + + +profiler_hub_result_t profiler_hub_get_time_slice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end +) { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::GetTimeSlice(future_handle, trace, track_id, slice_container, timestamp_start, timestamp_end); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} + +profiler_hub_result_t profiler_hub_get_pmc_time_slice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end, + bool left_neighbor, + bool right_neighbor +) { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::GetPmcTimeSlice(future_handle, trace, track_id, slice_container, timestamp_start, timestamp_end, left_neighbor, right_neighbor); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} + +profiler_hub_result_t profiler_hub_get_table_time_slice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_table_handle_t table_handle, + profiler_hub_track_id_t track_id, + uint64_t timestamp_start, + uint64_t timestamp_end +) { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::GetTableTimeSlice( + future_handle, + trace, + table_handle, + track_id, + timestamp_start, + timestamp_end); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} + +profiler_hub_result_t profiler_hub_get_search_time_slice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_table_handle_t table_handle, + profiler_hub_event_operation_t operation, + uint64_t timestamp_start, + uint64_t timestamp_end, + size_t num_search_strings, + profiler_hub_search_strings_t string_filters +) { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::GetSearchTimeSlice( + future_handle, + trace, + instance, + table_handle, + operation, + timestamp_start, + timestamp_end, + num_search_strings, + string_filters); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} + +profiler_hub_result_t profiler_hub_get_event_data_flow( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_flowtrace_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id +) { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::GetEventDataFlow( + future_handle, + trace, + instance, + container, + operation, + event_id); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} + +profiler_hub_result_t profiler_hub_get_event_extended_data( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id +) { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::GetEventExtendedData( + future_handle, + trace, + instance, + container, + operation, + event_id); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} + +profiler_hub_result_t profiler_hub_get_event_stack_trace( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_call_stack_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id +) { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::GetEventStackTrace( + future_handle, + trace, + instance, + container, + operation, + event_id); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} + +profiler_hub_result_t profiler_hub_trim_save_trace( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + uint64_t timestamp_start, + uint64_t timestamp_end, + profiler_hub_string_t new_path) { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + try { + result = profiler_hub::interface::TrimTraceDatabase( + future_handle, + trace, + timestamp_start, + timestamp_end, + new_path); + } + catch (std::exception& e) + { + spdlog::error(error_fmt, e.what()); + result = kProfilerHubStatusUnknownError; + } + return result; +} diff --git a/src/model/src/database/profiler_hub_lib_interface.h b/src/model/src/database/profiler_hub_lib_interface.h new file mode 100644 index 000000000..2da9f175f --- /dev/null +++ b/src/model/src/database/profiler_hub_lib_interface.h @@ -0,0 +1,114 @@ +#pragma once + +#ifdef USE_PROFILER_HUB + +#include "profiler_hub_interface.h" + +extern "C" +{ + profiler_hub_future_handle_t profiler_hub_future_alloc(profiler_hub::progress_callback_t progress_callback); + profiler_hub_result_t profiler_hub_future_free(profiler_hub_future_handle_t handle); + profiler_hub_result_t profiler_hub_future_wait(profiler_hub_future_handle_t handle, uint64_t timeout_ms); + profiler_hub_result_t profiler_hub_future_cancel(profiler_hub_future_handle_t handle); + + profiler_hub_db_type_t profiler_hub_db_identify_type( + profiler_hub_string_t trace_file_path + ); + + profiler_hub_trace_handle_t profiler_hub_open_trace( + profiler_hub_string_t trace_file_path + ); + + profiler_hub_result_t profiler_hub_set_trace_properties(profiler_hub_trace_handle_t trace, + client_trace_handle_t client_trace, + profiler_hub_string_t config_path, + size_t histogram_bucket_count); + + profiler_hub_result_t profiler_hub_read_metadata( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace + ); + + profiler_hub_result_t profiler_hub_close_trace( + profiler_hub_trace_handle_t trace + ); + + profiler_hub_result_t profiler_hub_get_time_slice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end + ); + + profiler_hub_result_t profiler_hub_get_pmc_time_slice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end, + bool left_neighbor, + bool right_neighbor + ); + + + profiler_hub_result_t profiler_hub_get_table_time_slice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_table_handle_t table_handle, + profiler_hub_track_id_t track_id, + uint64_t timestamp_start, + uint64_t timestamp_end + ); + + profiler_hub_result_t profiler_hub_get_search_time_slice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_table_handle_t table_handle, + profiler_hub_event_operation_t operation, + uint64_t timestamp_start, + uint64_t timestamp_end, + size_t num_search_strings, + profiler_hub_search_strings_t string_filters + ); + + profiler_hub_result_t profiler_hub_get_event_data_flow( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_flowtrace_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id + ); + + profiler_hub_result_t profiler_hub_get_event_extended_data( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id + ); + + profiler_hub_result_t profiler_hub_get_event_stack_trace( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_call_stack_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id + ); + + profiler_hub_result_t profiler_hub_trim_save_trace( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + uint64_t timestamp_start, + uint64_t timestamp_end, + profiler_hub_string_t new_path); + +} + +#endif \ No newline at end of file diff --git a/src/model/src/database/rocprofvis_db.cpp b/src/model/src/database/rocprofvis_db.cpp index a4984b652..b0f92078a 100644 --- a/src/model/src/database/rocprofvis_db.cpp +++ b/src/model/src/database/rocprofvis_db.cpp @@ -4,9 +4,7 @@ #include "rocprofvis_db.h" #include "rocprofvis_db_profile.h" #include -#include #include -#include #include namespace RocProfVis @@ -39,17 +37,6 @@ bool Database::IsNumber(const std::string& s) { return iss >> std::noskipws >> d && iss.eof(); } -rocprofvis_dm_result_t Database::AddTrackProperties( - rocprofvis_dm_track_params_t& props) { - try { - m_track_properties.push_back(std::make_unique(props)); - } - catch (const std::exception&) - { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ERROR_MEMORY_ALLOCATION_FAILURE, kRocProfVisDmResultAllocFailure); - } - return kRocProfVisDmResultSuccess; -} void Database::ShowProgress( double step, @@ -102,255 +89,6 @@ rocprofvis_dm_result_t Database::ReadTraceMetadataAsync( return kRocProfVisDmResultSuccess; } -rocprofvis_dm_result_t Database::ReadTraceSliceAsync( - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_num_of_tracks_t num, - rocprofvis_db_track_selection_t tracks, - rocprofvis_db_future_t object){ - Future* future = (Future*) object; - ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, kRocProfVisDmResultResourceBusy); - rocprofvis_dm_result_t result = BindObject()->FuncCheckSliceExists(BindObject()->trace_object, start, end, tag, num, tracks); - if(result != kRocProfVisDmResultNotLoaded) - { - spdlog::debug("Slice ({},{}) exists!", start, end); - return future->SetPromise(result); - } - try { - future->SetWorker(std::move(std::thread(Database::ReadTraceSliceStatic, this, start, end, tag, num, tracks, future))); - } - catch (const std::exception& ex) - { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); - } - return kRocProfVisDmResultSuccess; -} - -rocprofvis_dm_result_t -Database::ReadTracePMCSliceAsync( - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_track_selection_t track, - bool left_neighbor, - bool right_neighbor, - rocprofvis_db_future_t object){ - Future* future = (Future*) object; - ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, kRocProfVisDmResultResourceBusy); - rocprofvis_dm_result_t result = BindObject()->FuncCheckSliceExists(BindObject()->trace_object, start, end, tag, 1, track); - if(result != kRocProfVisDmResultNotLoaded) - { - spdlog::debug("Slice ({},{}) exists!", start, end); - return future->SetPromise(result); - } - try { - future->SetWorker(std::move(std::thread(Database::ReadTracePMCSliceStatic, this, start, end, tag, track, left_neighbor, right_neighbor, future))); - } - catch (const std::exception& ex) - { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); - } - return kRocProfVisDmResultSuccess; -} - -rocprofvis_dm_result_t Database::ReadEventPropertyAsync( - rocprofvis_dm_event_property_type_t type, - rocprofvis_dm_event_id_t event_id, - rocprofvis_db_future_t object){ - Future* future = (Future*) object; - ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, kRocProfVisDmResultResourceBusy); - rocprofvis_dm_result_t result = BindObject()->FuncCheckEventPropertyExists(BindObject()->trace_object, type, event_id); - if(result != kRocProfVisDmResultNotLoaded) - { - return future->SetPromise(kRocProfVisDmResultResourceBusy); - } - try { - future->SetWorker(std::move(std::thread(ReadEventPropertyStatic, this, type, event_id, future))); - } - catch (std::exception ex) - { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); - } - return kRocProfVisDmResultSuccess; -} - -rocprofvis_dm_result_t Database::BuildEventSearchQuery( - rocprofvis_dm_timestamp_t start, rocprofvis_dm_timestamp_t end, - rocprofvis_db_num_of_tracks_t num, rocprofvis_db_track_selection_t ops, - rocprofvis_dm_charptr_t where, - rocprofvis_dm_num_string_table_filters_t num_string_table_filters, rocprofvis_dm_string_table_filters_t string_table_filters, - bool include_substring, bool include_category, bool partial_matching, - rocprofvis_dm_charptr_t sort_column, rocprofvis_dm_sort_order_t sort_order, - uint64_t max_count, uint64_t offset, bool count_only, rocprofvis_dm_string_t& query) -{ - (void) start; - (void) end; - (void) num; - (void) ops; - (void) where; - (void) num_string_table_filters; - (void) string_table_filters; - (void) include_substring; - (void) include_category; - (void) partial_matching; - (void) sort_column; - (void) sort_order; - (void) max_count; - (void) offset; - (void) count_only; - (void) query; - return kRocProfVisDmResultNotSupported; -} - -rocprofvis_dm_result_t Database::ExportTableCSVAsync(rocprofvis_dm_string_t query, - rocprofvis_dm_string_t file_path, - rocprofvis_db_future_t object) -{ - Future* future = (Future*) object; - ROCPROFVIS_ASSERT_MSG_RETURN(!file_path.empty(), "Output path cannot be empty.", - kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, - kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, - kRocProfVisDmResultResourceBusy); - try - { - future->SetWorker(std::move(std::thread(&ExportTableCSVStatic, this, query, file_path, future))); - } catch(std::exception ex) - { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); - } - return kRocProfVisDmResultSuccess; -} - -rocprofvis_dm_result_t Database::ExportTableCSVStatic(Database* db, - rocprofvis_dm_string_t query, - rocprofvis_dm_string_t file_path, - Future* future) -{ - ROCPROFVIS_ASSERT_MSG_RETURN(!file_path.empty(), "New DB path cannot be empty.", - kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, - kRocProfVisDmResultInvalidParameter); - return db->ExportTableCSV(query.c_str(), file_path.c_str(), future); -} - -rocprofvis_dm_result_t Database::ExportTableCSV(rocprofvis_dm_charptr_t query, - rocprofvis_dm_charptr_t file_path, - Future* future) -{ - (void) query; - (void) file_path; - (void) future; - return kRocProfVisDmResultNotSupported; -} - -rocprofvis_dm_result_t -Database::SaveTrimmedDataAsync(rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_string_t new_db_path, - rocprofvis_db_future_t object) -{ - Future* future = (Future*) object; - ROCPROFVIS_ASSERT_MSG_RETURN(!new_db_path.empty(), "New DB path cannot be empty.", - kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, - kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, - kRocProfVisDmResultResourceBusy); - try - { - future->SetWorker(std::move(std::thread(&SaveTrimmedDataStatic, this, start, end, new_db_path, future))); - } catch(std::exception ex) - { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); - } - return kRocProfVisDmResultSuccess; -} - -rocprofvis_dm_result_t Database::SaveTrimmedDataStatic(Database* db, rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, rocprofvis_dm_string_t new_db_path, Future* future) -{ - ROCPROFVIS_ASSERT_MSG_RETURN(!new_db_path.empty(), "New DB path cannot be empty.", - kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, - kRocProfVisDmResultInvalidParameter); - - - //check if a db file exists and if it does delete it (we will overwrite it) - std::ifstream file(new_db_path); - if(file.good()) - { - file.close(); - int remove_result = std::remove(new_db_path.c_str()); - if(remove_result != 0) - { - spdlog::error("Failed to overwrite existing file: {}, code: {}", new_db_path, - remove_result); - - db->ShowProgress(0, "Failed to trim track! Could not overwrite existing file.", kRPVDbError, future); - future->SetPromise(kRocProfVisDmResultDbAccessFailed); - return kRocProfVisDmResultDbAccessFailed; - } - } - - return db->SaveTrimmedData(start, end, new_db_path.c_str(), future); -} - - -rocprofvis_dm_result_t Database::ExecuteQueryAsync( - rocprofvis_dm_charptr_t query, - rocprofvis_dm_charptr_t description, - rocprofvis_db_future_t object, - rocprofvis_dm_table_id_t* id) -{ - Future* future = (Future*) object; - ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, kRocProfVisDmResultResourceBusy); - *id = std::hash{}(query); - rocprofvis_dm_result_t result = BindObject()->FuncCheckTableExists(BindObject()->trace_object, *id); - if(result != kRocProfVisDmResultNotLoaded) - { - return future->SetPromise(result); - } - try { - future->SetWorker(std::move(std::thread(ExecuteQueryStatic, this, query, description, future))); - } - catch (std::exception ex) - { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); - } - return kRocProfVisDmResultSuccess; -} - -rocprofvis_dm_result_t Database::ExecuteComputeQueryAsync( - rocprofvis_db_compute_use_case_enum_t use_case, - rocprofvis_dm_charptr_t query, - rocprofvis_db_future_t object, - rocprofvis_dm_table_id_t* id) -{ - Future* future = (Future*) object; - ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, kRocProfVisDmResultResourceBusy); - *id = std::hash{}(query); - rocprofvis_dm_result_t result = BindObject()->FuncCheckTableExists(BindObject()->trace_object, *id); - if(result != kRocProfVisDmResultNotLoaded) - { - return future->SetPromise(result); - } - try { - future->SetWorker(std::move(std::thread(ExecuteComputeQueryStatic, this, use_case, query, future))); - } - catch (std::exception ex) - { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); - } - return kRocProfVisDmResultSuccess; -} rocprofvis_dm_result_t Database::CleanupStatic(Database* db, Future* future, bool rebuild) { return db->Cleanup(future, rebuild); @@ -362,96 +100,6 @@ rocprofvis_dm_result_t Database::ReadTraceMetadataStatic( return db->ReadTraceMetadata(object); } -rocprofvis_dm_result_t Database::ReadTraceSliceStatic( - Database* db, - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_num_of_tracks_t num, - rocprofvis_db_track_selection_t tracks, - Future* object){ - return db->ReadTraceSlice(start, end, tag, num, tracks, object); -} - -rocprofvis_dm_result_t Database::ReadTracePMCSliceStatic( - Database* db, - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_track_selection_t track, - bool left_neighbor, - bool right_neighbor, - Future* object){ - return db->ReadTracePMCSlice(start, end, tag, track, left_neighbor, right_neighbor, object); -} - -rocprofvis_dm_result_t Database::ReadTracePMCSlice( - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_track_selection_t track, - bool left_neighbor, - bool right_neighbor, - Future* object){ - (void) start; - (void) end; - (void) tag; - (void) track; - (void) left_neighbor; - (void) right_neighbor; - object->SetPromise(kRocProfVisDmResultNotSupported); - return kRocProfVisDmResultNotSupported; -} - -rocprofvis_dm_result_t Database::ReadEventPropertyStatic( - Database* db, - rocprofvis_dm_event_property_type_t type, - rocprofvis_dm_event_id_t event_id, - Future* object){ - switch (type) { - case kRPVDMEventFlowTrace: - return db->ReadFlowTraceInfo(event_id,object); - case kRPVDMEventStackTrace: - return db->ReadStackTraceInfo(event_id,object); - case kRPVDMEventExtData: - return db->ReadExtEventInfo(event_id,object); - } - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ERROR_UNSUPPORTED_PROPERTY, kRocProfVisDmResultNotSupported); -} - - - -rocprofvis_dm_result_t Database::ExecuteQueryStatic( - Database* db, - rocprofvis_dm_charptr_t query, - rocprofvis_dm_charptr_t description, - Future* object){ - return db->ExecuteQuery(query,description,object); -} - -rocprofvis_dm_result_t Database::ExecuteComputeQueryStatic( - Database* db, - rocprofvis_db_compute_use_case_enum_t use_case, - rocprofvis_dm_charptr_t query, - Future* object){ - return db->ExecuteComputeQuery(use_case, query,object); -} - - -const char* Database::SubProcessNameSuffixFor(rocprofvis_dm_track_category_t category){ - switch(category){ - case kRocProfVisDmPmcTrack: - case kRocProfVisDmKernelDispatchTrack: - case kRocProfVisDmMemoryAllocationTrack: - case kRocProfVisDmMemoryCopyTrack: - return "Queue "; - case kRocProfVisDmRegionTrack: - case kRocProfVisDmRegionMainTrack: - case kRocProfVisDmRegionSampleTrack: - return "Thread "; - } - return ""; -} rocprofvis_dm_result_t Database::FindCachedTableValue( const rocprofvis_dm_database_t object, @@ -504,131 +152,15 @@ const size_t Database::GetInfoTableRowNumCells(rocprofvis_dm_table_row_t object) return row->values.size(); } -rocprofvis_dm_size_t DatabaseCache::GetMemoryFootprint() -{ - size_t size = 0; - return size; -} - -rocprofvis_dm_size_t Database::GetMemoryFootprint() -{ +rocprofvis_dm_size_t Database::GetMemoryFootprint(void) { rocprofvis_dm_size_t size = 0; for (auto guid : m_db_instances) { size+= m_cached_tables[guid.first.GuidIndex()].GetMemoryFootprint(); } - size+=NumTracks()*(sizeof(rocprofvis_dm_track_params_t)+sizeof(std::unique_ptr)); - size+=strlen(Path()); return size; } -void Database::CreateTracksOrderRanking() { - - - using Track = rocprofvis_dm_track_params_t; - - std::map, std::vector> partitions; - - for (auto& track_prop : m_track_properties) { - Track* t = track_prop.get(); - partitions[t->load_id].push_back(t); - } - - uint32_t order_base = 0; - - for (auto& [load_set, partition_tracks] : partitions) { - - - std::map> groups; - - for (auto* t : partition_tracks) { - DbInstance* db_instance = (DbInstance*)t->track_indentifiers.db_instance; - uint32_t file_index = db_instance ? db_instance->FileIndex() : 0; - groups[file_index].push_back(t); - } - - for (auto& [db, vec] : groups) { - std::sort(vec.begin(), vec.end(),[]( - const Track* a, const Track* b) { - return a->track_indentifiers.track_id < - b->track_indentifiers.track_id; - }); - } - - size_t max_len = 0; - for (const auto& [_, vec] : groups) { - max_len = std::max(max_len, vec.size()); - } - - uint32_t local_order = 0; - - for (size_t i = 0; i < max_len; ++i) { - for (auto& [db, vec] : groups) { - if (i < vec.size()) { - vec[i]->order_id = order_base + local_order; - ++local_order; - } - } - } - - // Advance the base past every track assigned in this partition so - // order_ids stay globally unique regardless of how many tracks a - // partition holds (a fixed stride per partition would collide once a - // partition exceeds that stride). - order_base += local_order; - } - -} - -void DatabaseVersion::SetVersion(const char* version) { - m_db_version = ConvertVersionStringToInt(version); -} - -std::vector -DatabaseVersion::ConvertVersionStringToInt(const char* version) -{ - std::vector version_array; - std::stringstream ss(version); - std::string token; - while(std::getline(ss, token, '.')) - { - version_array.push_back(std::stoi(token)); - } - return version_array; -} - -bool DatabaseVersion::IsVersionEqual(const char* version) -{ - std::vector db_version = ConvertVersionStringToInt(version); - - for (int i = 0; i < db_version.size(); i++) - { - uint32_t token = (m_db_version.size() > i) ? m_db_version[i] : 0; - if(db_version[i] != token) - { - return false; - } - } - return true; -} - -bool DatabaseVersion::IsVersionGreaterOrEqual(const char* version) -{ - std::vector db_version = ConvertVersionStringToInt(version); - - for(int i = 0; i < db_version.size(); i++) - { - uint32_t token = (m_db_version.size() > i) ? m_db_version[i] : 0; - if(token > db_version[i]) - { - return true; - } else if (token < db_version[i]) - { - return false; - } - } - return true; -} } // namespace DataModel } // namespace RocProfVis \ No newline at end of file diff --git a/src/model/src/database/rocprofvis_db.h b/src/model/src/database/rocprofvis_db.h index 74b3bb38a..c82960830 100644 --- a/src/model/src/database/rocprofvis_db.h +++ b/src/model/src/database/rocprofvis_db.h @@ -7,6 +7,10 @@ #include "rocprofvis_db_cache.h" #include "rocprofvis_db_track.h" #include "rocprofvis_db_version.h" +#include "rocprofvis_db_packed_storage.h" +#ifdef USE_PROFILER_HUB +#include "profiler_hub_client_interface.h" +#endif #include #include #include @@ -20,18 +24,10 @@ namespace RocProfVis namespace DataModel { -typedef std::vector>::iterator rocprofvis_dm_track_params_it; - -// type of map array for generating time slice query for multiple tracks -typedef std::unordered_map> slice_query_map_t; -// type of map array for storing slice handlers for multi-track request -typedef std::unordered_map slice_array_t; -// type of map array for storing string id filters for op table queries -typedef std::unordered_map> table_string_id_filter_map_t; - typedef std::pair GuidInfo; typedef std::vector guid_list_t; + class TemporaryDbInstance : public DbInstance { public: @@ -44,49 +40,26 @@ class SingleNodeDbInstance : public DbInstance SingleNodeDbInstance() : DbInstance(0, 0) {}; }; -typedef enum class rocprofvis_db_string_type:uint32_t -{ - kRPVStringTypeNameOrCategory, - kRPVStringTypeKernelSymbol -} rocprofvis_db_string_type_t; - -typedef struct rocprofvis_db_string_id_t -{ - uint64_t m_string_id; - uint32_t m_guid_id; - rocprofvis_db_string_type_t m_string_type; - - bool operator==(const rocprofvis_db_string_id_t& other) const { - return m_string_id == other.m_string_id && m_guid_id == other.m_guid_id && m_string_type == other.m_string_type; - } -} rocprofvis_db_string_id_t; +// type of sqlite3_exec callback function +typedef int (*RpvCallback)(void*, int, void*, char**); + +// structure to pass parameters to query callbacks +typedef struct{ + // pointer tp Database object + Database* db; + // pointer to Future object, to check if thread has been interrupted + Future* future; + // pointer to container object handle, to add processed rows data to the container + rocprofvis_dm_handle_t handle; + // callback method pointer + RpvCallback callback; + // pointer to query string, convenient for multiuse callback debugging + std::vector query; + rocprofvis_dm_track_id_t track_id; + rocprofvis_dm_event_operation_t operation; + DbInstance* db_instance; +} rocprofvis_db_query_callback_parameters; -class Database; - -// Helper class to lock processes in order of database instances -class OrderedMutex { -public: - void init(uint32_t num_instances) { for (uint32_t i = 0; i < num_instances; i++) { m_instances.insert(i); } } - - void lock(uint32_t id) { - std::unique_lock lock(m_lock); - m_cv.wait(lock, [&] { return id == *m_instances.begin(); }); - } - - void unlock(uint32_t id) { - { - std::lock_guard lock(m_lock); - m_instances.erase(id); - } - m_cv.notify_all(); - } - - -private: - std::set m_instances; - std::mutex m_lock; - std::condition_variable m_cv; -}; class Database { @@ -96,8 +69,7 @@ class Database Database( rocprofvis_db_filename_t path): m_path(path), - m_binding_info(nullptr), - m_track_lookup(this) { + m_binding_info(nullptr) { }; // Database destructor, must be defined as virtual to free resources of derived classes virtual ~Database(){}; @@ -108,6 +80,7 @@ class Database // Method to close database, must be overriden by derived classes // @return status of operation virtual rocprofvis_dm_result_t Close() = 0; + // Get amount of memory used by database resource // @return memory size virtual rocprofvis_dm_size_t GetMemoryFootprint(void); @@ -127,145 +100,10 @@ class Database // @return status of operation rocprofvis_dm_result_t ReadTraceMetadataAsync( rocprofvis_db_future_t object); - // Asynchronously read a time slice (records from specified number of tracks for specified time frame) from database - // @param start - start timestamp of time slice - // @param end - end timestamp of time slice - // @param num - number of tracks - // @param tracks - uint32_t array with track IDs - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - rocprofvis_dm_result_t ReadTraceSliceAsync( - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_num_of_tracks_t num, - rocprofvis_db_track_selection_t tracks, - rocprofvis_db_future_t object); - - // Asynchronously read a PMC time slice (records from specified track for specified time frame) from database - // @param start - start timestamp of time slice - // @param end - end timestamp of time slice - // @param track - track ID - // @param left_neighbor - include the left neighbor of the time range - // @param right_neighbor - include the right neighbor of the time range - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - rocprofvis_dm_result_t ReadTracePMCSliceAsync( - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_track_selection_t track, - bool left_neighbor, - bool right_neighbor, - rocprofvis_db_future_t object); - - // Asynchronously read different types of event properties (flowtrace, stacktrace, extdata) for event ID - // @param type - event property type (flowtrace, stacktrace, extdata) - // @param event_id - 60-bit event id and 4-bit operation type - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - rocprofvis_dm_result_t ReadEventPropertyAsync( - rocprofvis_dm_event_property_type_t type, - rocprofvis_dm_event_id_t event_id, - rocprofvis_db_future_t object); - // Asynchronously run any table query and store results into Table object - // @param query - database query - // @param description - database description - // @param object - future object providing asynchronous execution mechanism - // @param id new id is assigned to the table and returned using this reference pointer - // @return status of operation - rocprofvis_dm_result_t ExecuteQueryAsync( - rocprofvis_dm_charptr_t query, - rocprofvis_dm_charptr_t description, - rocprofvis_db_future_t object, - rocprofvis_dm_table_id_t* id); - // Asynchronously run compute table query and store results into Table object - // @param query - database query - // @param description - database description - // @param object - future object providing asynchronous execution mechanism - // @param id new id is assigned to the table and returned using this reference pointer - // @return status of operation - rocprofvis_dm_result_t ExecuteComputeQueryAsync( - rocprofvis_db_compute_use_case_enum_t use_case, - rocprofvis_dm_charptr_t query, - rocprofvis_db_future_t object, - rocprofvis_dm_table_id_t* id); - // method to build a query for compute use case - // @param use_case - use case enumeration - // @param num - number of parameters - // @param params -parameters array - // @param query - reference to query string - // @return status of operation - virtual rocprofvis_dm_result_t BuildComputeQuery( - rocprofvis_db_compute_use_case_enum_t use_case, - rocprofvis_db_num_of_params_t num, - rocprofvis_db_compute_params_t params, - rocprofvis_dm_string_t& query) = 0; - - virtual rocprofvis_dm_result_t BuildTableQuery( - rocprofvis_dm_table_use_case_enum_t use_case, - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_db_num_of_tracks_t num, - rocprofvis_db_track_selection_t tracks, - rocprofvis_dm_charptr_t where, - rocprofvis_dm_charptr_t filter, - rocprofvis_dm_charptr_t group, - rocprofvis_dm_charptr_t group_cols, - rocprofvis_dm_charptr_t sort_column, - rocprofvis_dm_sort_order_t sort_order, - uint64_t max_count, - uint64_t offset, - bool count_only, - rocprofvis_dm_string_t& query) = 0; - - virtual rocprofvis_dm_result_t BuildEventSearchQuery( - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_db_num_of_tracks_t num, - rocprofvis_db_track_selection_t ops, - rocprofvis_dm_charptr_t where, - rocprofvis_dm_num_string_table_filters_t num_string_table_filters, - rocprofvis_dm_string_table_filters_t string_table_filters, - bool include_substring, - bool include_category, - bool partial_matching, - rocprofvis_dm_charptr_t sort_column, - rocprofvis_dm_sort_order_t sort_order, - uint64_t max_count, - uint64_t offset, - bool count_only, - rocprofvis_dm_string_t& query); - - - // Asynchronously writes the results of a table query to .CSV - // @param query - database query - // @param file_path - .CSV output path - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - rocprofvis_dm_result_t ExportTableCSVAsync(rocprofvis_dm_string_t query, - rocprofvis_dm_string_t file_path, - rocprofvis_db_future_t object); - - virtual rocprofvis_dm_result_t SaveTrimmedData(rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_charptr_t new_db_path, - Future* future) = 0; - - rocprofvis_dm_result_t SaveTrimmedDataAsync(rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_string_t new_db_path, - rocprofvis_db_future_t object); - - static rocprofvis_dm_result_t SaveTrimmedDataStatic(Database* db, - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_string_t new_db_path, - Future* object); + virtual void InterruptQuery(void* connection) { (void) connection; }; - private: /************************static methods to be used as a parameter to std::thread**********************/ @@ -275,74 +113,7 @@ class Database static rocprofvis_dm_result_t ReadTraceMetadataStatic( Database* db, Future* object); - //static method to read time slice. Required to launch a unique thread for asynchronous time slice read - // @param db - pointer to database object - // @param start - start timestamp of time slice - // @param end - end timestamp of time slice - // @param num - number of tracks - // @param tracks - uint32_t array with track IDs - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - static rocprofvis_dm_result_t ReadTraceSliceStatic( - Database* db, - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_num_of_tracks_t num, - rocprofvis_db_track_selection_t tracks, - Future* object); - - //static method to read PMC time slice. Required to launch a unique thread for asynchronous time slice read - // @param db - pointer to database object - // @param start - start timestamp of time slice - // @param end - end timestamp of time slice - // @param track - track ID - // @param left_neighbor - include the left neighbor of the time range - // @param right_neighbor - include the right neighbor of the time range - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - static rocprofvis_dm_result_t ReadTracePMCSliceStatic( - Database* db, - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_track_selection_t track, - bool left_neighbor, - bool right_neighbor, - Future* object); - //static method to read Event properties. Required to launch a unique thread for asynchronous event properties read - // @param db - pointer to database object - // @param type - event property type (flowtrace, stacktrace, extdata) - // @param event_id - 60-bit event id and 4-bit operation type - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - static rocprofvis_dm_result_t ReadEventPropertyStatic( - Database* db, - rocprofvis_dm_event_property_type_t type, - rocprofvis_dm_event_id_t event_id, - Future* object); - //static method to launch any query. Required to launch a unique thread for asynchronous database query - // @param db - pointer to database object - // @param query - database query - // @param description - database description - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - static rocprofvis_dm_result_t ExecuteQueryStatic( - Database* db, - rocprofvis_dm_charptr_t query, - rocprofvis_dm_charptr_t description, - Future* object); - //static method to launch compute query. - // @param db - pointer to database object - // @param query - database query - // @param description - database description - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - static rocprofvis_dm_result_t ExecuteComputeQueryStatic( - Database* db, - rocprofvis_db_compute_use_case_enum_t use_case, - rocprofvis_dm_charptr_t query, - Future* object); + // static method to find a value in cached tables by specifying reserved table name, instance id and column name // @param object - database handler // @param table_name - a name of cached table assigned at the time of caching @@ -352,25 +123,15 @@ class Database // @param value - reference pointer to database cell value // @return status of operation static rocprofvis_dm_result_t FindCachedTableValue( const rocprofvis_dm_database_t object, - rocprofvis_dm_charptr_t table_name, - const rocprofvis_dm_id_t instance_id, - rocprofvis_dm_charptr_t column_name, - rocprofvis_dm_node_id_t node, - rocprofvis_dm_charptr_t* value); - - // static method to export the results of a table query to .CSV - // @param db - pointer to database object - // @param query - database query - // @param file_path - .CSV output path - // @param future - future object providing asynchronous execution mechanism - // @return status of operation - static rocprofvis_dm_result_t ExportTableCSVStatic( Database* db, - rocprofvis_dm_string_t query, - rocprofvis_dm_string_t file_path, - Future* future); + rocprofvis_dm_charptr_t table_name, + const rocprofvis_dm_id_t instance_id, + rocprofvis_dm_charptr_t column_name, + rocprofvis_dm_node_id_t node, + rocprofvis_dm_charptr_t* value); static rocprofvis_dm_result_t CleanupStatic(Database* db, Future* future, bool rebuild); + /************************pure virtual worker methods to be implemented in derived classes**********************/ // worker method to read trace metadata @@ -378,77 +139,6 @@ class Database // @return status of operation virtual rocprofvis_dm_result_t ReadTraceMetadata( Future* object) = 0; - // worker method to read time slice - // @param start - start timestamp of time slice - // @param end - end timestamp of time slice - // @param num - number of tracks - // @param tracks - uint32_t array with track IDs - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - virtual rocprofvis_dm_result_t ReadTraceSlice( - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_num_of_tracks_t num, - rocprofvis_db_track_selection_t tracks, - Future* object) = 0; - - virtual rocprofvis_dm_result_t ReadTracePMCSlice( - rocprofvis_dm_timestamp_t start, - rocprofvis_dm_timestamp_t end, - rocprofvis_dm_hashed_timestamp_tag_t tag, - rocprofvis_db_track_selection_t track, - bool left_neighbor, - bool right_neighbor, - Future* object); - - // worker method to read flow trace info, called from ReadEventPropertyStatic - // @param event_id - 60-bit event id and 4-bit operation type - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - virtual rocprofvis_dm_result_t ReadFlowTraceInfo( - rocprofvis_dm_event_id_t event_id, - Future* object) = 0; - // worker method to read stack trace info, called from ReadEventPropertyStatic - // @param event_id - 60-bit event id and 4-bit operation type - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - virtual rocprofvis_dm_result_t ReadStackTraceInfo( - rocprofvis_dm_event_id_t event_id, - Future* object) = 0; - // worker method to read extended info, called from ReadEventPropertyStatic - // @param event_id - 60-bit event id and 4-bit operation type - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - virtual rocprofvis_dm_result_t ReadExtEventInfo( - rocprofvis_dm_event_id_t event_id, - Future* object) = 0; - // worker method to execute any database query - // @param query - database query - // @param description - database description - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - virtual rocprofvis_dm_result_t ExecuteQuery( - rocprofvis_dm_charptr_t query, - rocprofvis_dm_charptr_t description, - Future* object) = 0; - // worker method to execute compute database query - // @param query - database query - // @param description - database description - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - virtual rocprofvis_dm_result_t ExecuteComputeQuery( - rocprofvis_db_compute_use_case_enum_t use_case, - rocprofvis_dm_charptr_t query, - Future* future) = 0; - - // method to export the results of a table query to .CSV - // @param query - database query - // @param file_path - .CSV output path - // @return status of operation - virtual rocprofvis_dm_result_t ExportTableCSV( rocprofvis_dm_charptr_t query, - rocprofvis_dm_charptr_t file_path, - Future* future); virtual rocprofvis_dm_result_t Cleanup(Future* future, bool rebuild) { (void) future; (void) rebuild; return kRocProfVisDmResultSuccess; }; @@ -456,16 +146,13 @@ class Database // pointer to a binding information structure physically located in Trace object and passed to Database object during binding // binding structure contains methods to transfer data between database and trace objects rocprofvis_dm_db_bind_struct *m_binding_info; + // map array of cached tables, mostly with non-essential Track information + std::unordered_map m_cached_tables; // database file path std::string m_path; // app config path std::string m_config_path; - // vector array of track parameters. Used as a reference for data model Track objects and for Database component to generate proper database queries - std::vector> m_track_properties; - // map array of cached tables, mostly with non-essential Track information - std::unordered_map m_cached_tables; guid_list_t m_db_instances; - TrackLookup m_track_lookup; protected: @@ -475,33 +162,15 @@ class Database std::string GuidAt(int index) { return index < m_db_instances.size() ? m_db_instances[index].second : std::string(); } std::string GuidSymAt(int index) { std::string s = GuidAt(index); std::replace(s.begin(), s.end(), '_', '-'); return s; } DbInstance* DbInstancePtrAt(int index) { return index < m_db_instances.size() ? &m_db_instances[index].first : nullptr; } + // returns pointer to cached tables map array + DatabaseCache* CachedTables(uint32_t node_id) {return &m_cached_tables[node_id];} // returns pointer to database file path rocprofvis_db_filename_t Path() {return m_path.c_str();} - // returns pointer to last registered Track properties structure - rocprofvis_dm_track_params_t* TrackPropertiesLast() { return m_track_properties.back().get(); } - // returns track properties begin iterator - rocprofvis_dm_track_params_it TrackPropertiesBegin() { return m_track_properties.begin(); } - // returns track properties end iterator - rocprofvis_dm_track_params_it TrackPropertiesEnd() { return m_track_properties.end(); } // returns pointer to trace properties, which contains shared trace information rocprofvis_dm_trace_params_t* TraceProperties() { return m_binding_info->trace_properties; } - // returns pointer to cached tables map array - DatabaseCache* CachedTables(uint32_t node_id) {return &m_cached_tables[node_id];} - - TrackLookup* TrackTracker() { return& m_track_lookup; } - // return current number of tracks - rocprofvis_dm_size_t NumTracks() { return m_track_properties.size(); } - // returns pointer to track properties structure. Takes index of track as a parameter - rocprofvis_dm_track_params_t* TrackPropertiesAt(rocprofvis_dm_index_t index) { return m_track_properties[index].get(); } - // validated track index - bool IsTrackIndexValid(rocprofvis_dm_index_t index) { return index < m_track_properties.size(); } // ---------------------------------------------Helpers--------------------------------------- - // register new track - // @param props - track properties structure - // @return status of operation - rocprofvis_dm_result_t AddTrackProperties( - rocprofvis_dm_track_params_t& props); + // calls Future object callback method, if provided. The callback method is optionally provided by caller in order to display or save current database progress. // @param step - approximate percentage of single database operation @@ -513,28 +182,7 @@ class Database rocprofvis_dm_charptr_t action, rocprofvis_db_status_t status, Future* future); - // remap string IDs in new event record structure - // @param record - event data record - // @return status of operation - virtual rocprofvis_dm_result_t RemapStringIds( - rocprofvis_db_record_data_t & record) { (void) record; return kRocProfVisDmResultSuccess;}; - virtual rocprofvis_dm_result_t RemapStringIds( - rocprofvis_db_flow_data_t & record) { (void) record; return kRocProfVisDmResultSuccess;}; - virtual rocprofvis_dm_result_t StringIndexToId( - rocprofvis_dm_index_t index, std::vector& id) { (void) index; (void) id; return kRocProfVisDmResultSuccess;}; - - // return suffix to sub-process name for provided track category ('TID', 'Queue') - // @param category - track category - // @return track sub-process name suffix ('TID', 'Queue') - static const char* SubProcessNameSuffixFor(rocprofvis_dm_track_category_t category); - // create tracks ranking so they can be sorted accordingly in UI - void CreateTracksOrderRanking(); - - //--------------------------------------Static helpers----------------------------------------------------------------- - static bool SanitizeFilePath(const std::string& filename, std::filesystem::path& out_path); - static bool IsNumber(const std::string& s); - //--------------------------------------Direct interface to info tables----------------------------------------------------------------- static rocprofvis_dm_table_t GetInfoTableHandle(const rocprofvis_dm_database_t object, rocprofvis_dm_node_id_t node, rocprofvis_dm_charptr_t table_name); static size_t GetInfoTableNumColumns(rocprofvis_dm_table_t object); @@ -543,12 +191,18 @@ class Database static rocprofvis_dm_table_row_t GetInfoTableRowHandle(rocprofvis_dm_table_t object, size_t row_index); static const char* GetInfoTableRowCellValue(rocprofvis_dm_table_row_t object, size_t column_index); static const size_t GetInfoTableRowNumCells(rocprofvis_dm_table_row_t object); + //--------------------------------------Static helpers----------------------------------------------------------------- + static bool SanitizeFilePath(const std::string& filename, std::filesystem::path& out_path); + static bool IsNumber(const std::string& s); public: // declare DatabaseCache as friend class, for having access to protected members friend class DatabaseCache; friend class TableProcessor; friend class TrackLookup; + friend class PackedTable; + friend class SqliteDatabase; + friend class ProfilerHubClientMethods; }; } // namespace DataModel diff --git a/src/model/src/database/rocprofvis_db_cache.cpp b/src/model/src/database/rocprofvis_db_cache.cpp index 5759e077b..8dd88d1cb 100644 --- a/src/model/src/database/rocprofvis_db_cache.cpp +++ b/src/model/src/database/rocprofvis_db_cache.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: MIT #include "rocprofvis_db_cache.h" -#include "rocprofvis_db.h" +#include "rocprofvis_db_systems.h" #include "rocprofvis_db_query_builder.h" namespace RocProfVis @@ -151,7 +151,13 @@ namespace DataModel return &tables[table_name]; } - rocprofvis_dm_result_t DatabaseCache::PopulateTrackExtendedDataTemplate(Database * db, uint32_t db_instance_id, std::string table_name, uint64_t process_id ){ + rocprofvis_dm_size_t DatabaseCache::GetMemoryFootprint() + { + size_t size = 0; + return size; + } + + rocprofvis_dm_result_t DatabaseCache::PopulateTrackExtendedDataTemplate(SystemDatabase * db, uint32_t db_instance_id, std::string table_name, uint64_t process_id ){ rocprofvis_dm_track_params_t* track_properties = db->TrackPropertiesLast(); TableCache& table = tables[table_name]; uint32_t num_columns = table.NumColumns(); diff --git a/src/model/src/database/rocprofvis_db_cache.h b/src/model/src/database/rocprofvis_db_cache.h index af01df6c4..26dad060f 100644 --- a/src/model/src/database/rocprofvis_db_cache.h +++ b/src/model/src/database/rocprofvis_db_cache.h @@ -19,6 +19,7 @@ typedef std::map table_map_t; typedef std::map ref_map_t; class Database; +class SystemDatabase; // Helper class to manage cached information tables (node, agent, queue, process, thread information) @@ -70,7 +71,7 @@ class DatabaseCache void* GetTableHandle(const char* table_name); // Populate track extended data objects and topology tree with table content - rocprofvis_dm_result_t PopulateTrackExtendedDataTemplate(Database * db, uint32_t node_id, std::string table_name, uint64_t instance_id ); + rocprofvis_dm_result_t PopulateTrackExtendedDataTemplate(SystemDatabase * db, uint32_t node_id, std::string table_name, uint64_t instance_id ); // Populate track extended data objects and topology tree with table content rocprofvis_dm_result_t PopulateTrackTopologyData(Database * db, rocprofvis_dm_track_identifiers_t * track_indentifiers, uint32_t db_instance_id, std::string table_name, uint64_t process_id ); // Get amount of memory used by the cached values map diff --git a/src/model/src/database/rocprofvis_db_compute.cpp b/src/model/src/database/rocprofvis_db_compute.cpp index 024a80a81..ba731b44b 100644 --- a/src/model/src/database/rocprofvis_db_compute.cpp +++ b/src/model/src/database/rocprofvis_db_compute.cpp @@ -919,27 +919,6 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work return kRocProfVisDmResultSuccess; } - rocprofvis_dm_result_t ComputeDatabase::ExecuteQuery( - rocprofvis_dm_charptr_t query, - rocprofvis_dm_charptr_t description, - Future* future){ - - ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); - while (true) - { - ROCPROFVIS_ASSERT_MSG_BREAK(BindObject()->trace_properties, ERROR_TRACE_PROPERTIES_CANNOT_BE_NULL); - ROCPROFVIS_ASSERT_MSG_BREAK(BindObject()->trace_properties->metadata_loaded, ERROR_METADATA_IS_NOT_LOADED); - rocprofvis_dm_table_t table = BindObject()->FuncAddTable(BindObject()->trace_object, query, description); - TemporaryDbInstance tmp_db_instance(0); - ROCPROFVIS_ASSERT_MSG_RETURN(table, ERROR_TABLE_CANNOT_BE_NULL, kRocProfVisDmResultUnknownError); - if (kRocProfVisDmResultSuccess != ExecuteSQLQuery(future, &tmp_db_instance, query, table, &CallbackRunQuery)) break; - ShowProgress(100, "Query successfully executed!",kRPVDbSuccess, future); - return future->SetPromise(kRocProfVisDmResultSuccess); - } - ShowProgress(0, "Query could not be executed!", kRPVDbError, future ); - return future->SetPromise(future->Interrupted() ? kRocProfVisDmResultDbAbort : kRocProfVisDmResultDbAccessFailed); - } - rocprofvis_dm_result_t ComputeDatabase::ReadTraceMetadata(Future* future) { ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); @@ -1040,6 +1019,34 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work } + rocprofvis_dm_result_t ComputeDatabase::ExecuteComputeQueryAsync( + rocprofvis_db_compute_use_case_enum_t use_case, + rocprofvis_dm_charptr_t query, + rocprofvis_db_future_t object, + rocprofvis_dm_table_id_t* id) + { + Future* future = (Future*) object; + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, kRocProfVisDmResultResourceBusy); + *id = std::hash{}(query); + rocprofvis_dm_result_t result = BindObject()->FuncCheckTableExists(BindObject()->trace_object, *id); + if(result != kRocProfVisDmResultNotLoaded) + { + return future->SetPromise(result); + } + try { + future->SetWorker(std::move(std::thread([this, use_case, query, future] { + return ExecuteComputeQuery(use_case, query,future); + }))); + } + catch (std::exception ex) + { + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); + } + return kRocProfVisDmResultSuccess; + } + + rocprofvis_dm_result_t ComputeDatabase::ExecuteComputeQuery( rocprofvis_db_compute_use_case_enum_t use_case, rocprofvis_dm_charptr_t query, @@ -1064,7 +1071,7 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work } TemporaryDbInstance tmp_db_instance(0); ROCPROFVIS_ASSERT_MSG_RETURN(table, ERROR_TABLE_CANNOT_BE_NULL, kRocProfVisDmResultUnknownError); - RpvSqliteExecuteQueryCallback callback = nullptr; + RpvSqliteCallback callback = nullptr; switch (use_case) { case kRPVComputeFetchListOfWorkloads: @@ -1199,7 +1206,7 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work ROCPROFVIS_ASSERT_MSG_RETURN(argc==4, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackParseMetadata; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ComputeDatabase* db = (ComputeDatabase*)callback_params->db; db->m_db_version = "0.0.0"; for (int i = 0; i < argc; i++) @@ -1217,7 +1224,7 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work int ComputeDatabase::CallbackGetComputeKernelWorkloadLookupTable(void* data, int argc, sqlite3_stmt* stmt, char** azColName) { (void) argc; ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ComputeDatabase* db = (ComputeDatabase*)callback_params->db; void* func = (void*)&CallbackGetComputeKernelWorkloadLookupTable; if (callback_params->future->Interrupted()) return 1; @@ -1230,7 +1237,7 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work int ComputeDatabase::CallbackGetComputeGeneric(void *data, int argc, sqlite3_stmt* stmt, char **azColName){ ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ComputeDatabase* db = (ComputeDatabase*)callback_params->db; void* func = (void*)&CallbackGetComputeGeneric; if (callback_params->future->Interrupted()) return 1; @@ -1275,7 +1282,7 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work int ComputeDatabase::CallbackGetComputeRooflineCeiling(void *data, int argc, sqlite3_stmt* stmt, char **azColName){ (void) argc; ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ComputeDatabase* db = (ComputeDatabase*)callback_params->db; void* func = (void*)&CallbackGetComputeRooflineCeiling; if (callback_params->future->Interrupted()) return 1; @@ -1314,8 +1321,8 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work char** azColName) { ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) data; ComputeDatabase* db = (ComputeDatabase*) callback_params->db; void* func = (void*) &CallbackGetComputeKernelMetricsMatrix; if(callback_params->future->Interrupted()) return 1; @@ -1385,7 +1392,7 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work int ComputeDatabase::CallbackGetComputeWorkloadTopKernels(void* data, int argc, sqlite3_stmt* stmt, char** azColName) { (void) argc; ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ComputeDatabase* db = (ComputeDatabase*)callback_params->db; void* func = (void*)&CallbackGetComputeWorkloadTopKernels; if (callback_params->future->Interrupted()) return 1; @@ -1407,7 +1414,7 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work int ComputeDatabase::CallbackStoreMetricsLookupTable(void* data, int argc, sqlite3_stmt* stmt, char** azColName) { (void) argc; ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ComputeDatabase* db = (ComputeDatabase*)callback_params->db; void* func = (void*)&CallbackStoreMetricsLookupTable; if (callback_params->future->Interrupted()) return 1; @@ -1424,7 +1431,7 @@ void ComputeQueryFactory::ParseMetricParam(std::string metric_str, uint32_t work int ComputeDatabase::CallbackGetComputeMetricsData(void* data, int argc, sqlite3_stmt* stmt, char** azColName) { (void) argc; ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ComputeDatabase* db = (ComputeDatabase*)callback_params->db; void* func = (void*)&CallbackGetComputeMetricsData; if (callback_params->future->Interrupted()) return 1; diff --git a/src/model/src/database/rocprofvis_db_compute.h b/src/model/src/database/rocprofvis_db_compute.h index 353f8dbcf..6d0e1d0f4 100644 --- a/src/model/src/database/rocprofvis_db_compute.h +++ b/src/model/src/database/rocprofvis_db_compute.h @@ -80,11 +80,12 @@ namespace DataModel std::vector metrics; // pivoted metric columns }; - class ComputeDatabase : public SqliteDatabase + class ComputeDatabase : public Database, public SqliteDatabase { public: ComputeDatabase(rocprofvis_db_filename_t path) : - SqliteDatabase(path), + Database(path), + SqliteDatabase(this), m_query_factory(this), m_last_matrix_workload_id(INVALID_INDEX) { @@ -92,83 +93,38 @@ namespace DataModel }; // class destructor, not really required, unless declared as virtual - ~ComputeDatabase()override {}; + ~ComputeDatabase() override {}; + + // Method to open sqlite database + // @return status of operation + rocprofvis_dm_result_t Open() override { return OpenAsSqlite(); }; + // Method to close sqlite database + // @return status of operation + rocprofvis_dm_result_t Close() override { return CloseAsSqlite(); }; + // worker method to read trace metadata // @param object - future object providing asynchronous execution mechanism // @return status of operation rocprofvis_dm_result_t ReadTraceMetadata( - Future* object) override; + Future* object); rocprofvis_dm_result_t BuildComputeQuery( rocprofvis_db_compute_use_case_enum_t use_case, rocprofvis_db_num_of_params_t num, rocprofvis_db_compute_params_t params, - rocprofvis_dm_string_t& query) override; + rocprofvis_dm_string_t& query); - rocprofvis_dm_result_t ExecuteComputeQuery( + rocprofvis_dm_result_t ExecuteComputeQueryAsync( rocprofvis_db_compute_use_case_enum_t use_case, rocprofvis_dm_charptr_t query, - Future* future) override; - - protected: + rocprofvis_db_future_t object, + rocprofvis_dm_table_id_t* id); - // worker method to read flow trace info - // @param event_id - 60-bit event id and 4-bit operation type - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - rocprofvis_dm_result_t ReadFlowTraceInfo( - rocprofvis_dm_event_id_t /*event_id*/, - Future* /*object*/) override { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN("Compute database does not support flow trace", kRocProfVisDmResultNotSupported); - }; - // worker method to read stack trace info - // @param event_id - 60-bit event id and 4-bit operation type - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - rocprofvis_dm_result_t ReadStackTraceInfo( - rocprofvis_dm_event_id_t /*event_id*/, - Future* /*object*/) override { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN("Compute database does not support stack trace", kRocProfVisDmResultNotSupported); - }; - // worker method to read extended info - // @param event_id - 60-bit event id and 4-bit operation type - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - rocprofvis_dm_result_t ReadExtEventInfo( - rocprofvis_dm_event_id_t /*event_id*/, - Future* /*object*/) override { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN("Compute database does not support extended data", kRocProfVisDmResultNotSupported); - }; - // worker method to read time slice - // @param start - start timestamp of time slice - // @param end - end timestamp of time slice - // @param num - number of tracks - // @param tracks - uint32_t array with track IDs - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - rocprofvis_dm_result_t ReadTraceSlice( - rocprofvis_dm_timestamp_t /*start*/, - rocprofvis_dm_timestamp_t /*end*/, - rocprofvis_dm_hashed_timestamp_tag_t /*tag*/, - rocprofvis_db_num_of_tracks_t /*num*/, - rocprofvis_db_track_selection_t /*tracks*/, - Future* /*object*/) override { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN("Compute database does not support extended data", kRocProfVisDmResultNotSupported); - }; - // worker method to execute database query - // @param query - database query - // @param description - database description - // @param object - future object providing asynchronous execution mechanism - // @return status of operation - rocprofvis_dm_result_t ExecuteQuery( + private: + rocprofvis_dm_result_t ExecuteComputeQuery( + rocprofvis_db_compute_use_case_enum_t use_case, rocprofvis_dm_charptr_t query, - rocprofvis_dm_charptr_t description, - Future* future) override; + Future* future); - rocprofvis_dm_result_t SaveTrimmedData(rocprofvis_dm_timestamp_t /*start*/, - rocprofvis_dm_timestamp_t /*end*/, - rocprofvis_dm_charptr_t /*new_db_path*/, - Future* /*future*/) override { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN("Compute database does not support trimming", kRocProfVisDmResultNotSupported); - }; + protected: const rocprofvis_null_data_exceptions_int* GetNullDataExceptionsInt() override { @@ -183,25 +139,6 @@ namespace DataModel return &s_null_data_exceptions_skip; } - rocprofvis_dm_result_t BuildTableQuery( - rocprofvis_dm_table_use_case_enum_t /*use_case*/, - rocprofvis_dm_timestamp_t /*start*/, - rocprofvis_dm_timestamp_t /*end*/, - rocprofvis_db_num_of_tracks_t /*num*/, - rocprofvis_db_track_selection_t /*tracks*/, - rocprofvis_dm_charptr_t /*where*/, - rocprofvis_dm_charptr_t /*filter*/, - rocprofvis_dm_charptr_t /*group*/, - rocprofvis_dm_charptr_t /*group_cols*/, - rocprofvis_dm_charptr_t /*sort_column*/, - rocprofvis_dm_sort_order_t /*sort_order*/, - uint64_t /*max_count*/, - uint64_t /*offset*/, - bool /*count_only*/, - rocprofvis_dm_string_t& /*query*/) override { - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN("Compute database does not build table query", kRocProfVisDmResultNotSupported); - } - private: ComputeQueryFactory m_query_factory; diff --git a/src/model/src/database/rocprofvis_db_future.cpp b/src/model/src/database/rocprofvis_db_future.cpp index e17444778..518bf6534 100644 --- a/src/model/src/database/rocprofvis_db_future.cpp +++ b/src/model/src/database/rocprofvis_db_future.cpp @@ -33,6 +33,35 @@ Future::~Future(){ } } +#ifdef USE_PROFILER_HUB +profiler_hub_future_handle_t Future::AddProfilerHubFuture() +{ + profiler_hub_future_handle_t handle = profiler_hub_future_alloc((profiler_hub::progress_callback_t)m_progress_callback); + std::unique_lock lock(m_mutex); + m_ph_futures.push_back(handle); + return handle; +} + +void +Future::DeleteProfilerHubFuture(profiler_hub_future_handle_t ph_future) { + + std::unique_lock lock(m_mutex); + auto it = std::find_if(m_ph_futures.begin(), m_ph_futures.end(), [&](profiler_hub_future_handle_t & f) { return f == ph_future; }); + if (it != m_ph_futures.end()) + { + m_ph_futures.erase(it); + profiler_hub_future_free(ph_future); + } +} + +profiler_hub_result_t +Future::WaitAndDeleteProfilerHubFuture(profiler_hub_future_handle_t ph_future) { + profiler_hub_result_t result = profiler_hub_future_wait(ph_future, UINT64_MAX); + DeleteProfilerHubFuture(ph_future); + return result; +} +#endif + Future* Future::AddSubFuture() { Future* sub_feature = (Future*)rocprofvis_db_future_alloc(nullptr); std::unique_lock lock(m_mutex); @@ -69,13 +98,17 @@ void Future::SetInterrupted() { std::unique_lock lock(m_mutex); - if (m_sub_futures.size() > 0) + + for (auto future : m_sub_futures) + { + future->SetInterrupted(); + } +#ifdef USE_PROFILER_HUB + for (auto future : m_ph_futures) { - for (auto future : m_sub_futures) - { - future->SetInterrupted(); - } + rocprofvis_db_future_cancel(future); } +#endif if (m_db != nullptr && m_connection != nullptr) { m_db->InterruptQuery(m_connection); diff --git a/src/model/src/database/rocprofvis_db_future.h b/src/model/src/database/rocprofvis_db_future.h index 4f387794a..82b7f312f 100644 --- a/src/model/src/database/rocprofvis_db_future.h +++ b/src/model/src/database/rocprofvis_db_future.h @@ -4,6 +4,7 @@ #pragma once #include "rocprofvis_common_types.h" +#include "profiler_hub_lib_interface.h" #include #include #include @@ -112,6 +113,12 @@ class Future void DeleteSubFuture(Future* sub_future); rocprofvis_dm_result_t WaitAndDeleteSubFuture(Future* sub_future); +#ifdef USE_PROFILER_HUB + profiler_hub_future_handle_t AddProfilerHubFuture(); + void DeleteProfilerHubFuture(profiler_hub_future_handle_t ph_future); + profiler_hub_result_t WaitAndDeleteProfilerHubFuture(profiler_hub_future_handle_t ph_future); +#endif + private: static std::atomic s_counter; // unique identifier for this future @@ -137,6 +144,9 @@ class Future std::string m_async_query; std::vector m_sub_futures; std::array(kRPVFutureRuntimeStorageSize)> m_runtime_storage; +#ifdef USE_PROFILER_HUB + std::vector m_ph_futures; +#endif }; } // namespace DataModel diff --git a/src/model/src/database/rocprofvis_db_packed_storage.cpp b/src/model/src/database/rocprofvis_db_packed_storage.cpp index 25ff41116..f21be293d 100644 --- a/src/model/src/database/rocprofvis_db_packed_storage.cpp +++ b/src/model/src/database/rocprofvis_db_packed_storage.cpp @@ -80,13 +80,13 @@ namespace DataModel throw std::out_of_range("Column out of range"); } - DbInstance* PackedTable::GetDbInstanceForRow(QueryManager * db, int row_index) + DbInstance* PackedTable::GetDbInstanceForRow(SystemDatabase * db, int row_index) { ROCPROFVIS_ASSERT_MSG_RETURN(row_index < m_rows.size(), ERROR_INDEX_OUT_OF_RANGE, nullptr); return GetDbInstanceForRow(db, m_rows[row_index].get()); } - DbInstance* PackedTable::GetDbInstanceForRow(QueryManager * db, PackedRow* row) + DbInstance* PackedTable::GetDbInstanceForRow(SystemDatabase * db, PackedRow* row) { auto columns = GetMergedColumns(); uint8_t op = row->Get(0); @@ -146,9 +146,8 @@ namespace DataModel return m_rows[row]->Get(0); } - Numeric PackedTable::GetMergeTableValue(uint8_t op, size_t row, size_t col, QueryManager* requestor) const + Numeric PackedTable::GetMergeTableValue(uint8_t op, size_t row, size_t col) const { - (void) requestor; if (row >= m_rows.size() || col >= m_merged_columns.size()) throw std::out_of_range("Row/Column out of range"); @@ -172,7 +171,7 @@ namespace DataModel } - const char* PackedTable::ConvertSqlStringReference(QueryManager* db, uint32_t column_index, uint64_t value, uint32_t node_id, bool & numeric_string) { + const char* PackedTable::ConvertSqlStringReference(SystemDatabase* db, uint32_t column_index, uint64_t value, uint32_t node_id, bool & numeric_string) { numeric_string = false; uint64_t string_index = 0; if (column_index == Builder::SCHEMA_INDEX_NODE_ID) @@ -181,8 +180,7 @@ namespace DataModel return db->CachedTables(node_id)->GetTableCellByIndex("Node", static_cast(value), "id"); } else if (column_index == Builder::SCHEMA_INDEX_CATEGORY || column_index == Builder::SCHEMA_INDEX_CATEGORY_RPD || - column_index == Builder::SCHEMA_INDEX_EVENT_NAME || column_index == Builder::SCHEMA_INDEX_EVENT_NAME_RPD || - column_index == Builder::SCHEMA_INDEX_EVENT_ARGS_RPD) + column_index == Builder::SCHEMA_INDEX_EVENT_NAME || column_index == Builder::SCHEMA_INDEX_EVENT_NAME_RPD) { if (kRocProfVisDmResultSuccess == db->RemapStringId(value, rocprofvis_db_string_type_t::kRPVStringTypeNameOrCategory, node_id, string_index)) { @@ -357,8 +355,7 @@ namespace DataModel m_aggregation.m_string_data.Clear(); } - void PackedTable::SortAggregationByColumn(QueryManager* db, std::string sort_column, bool sort_order) { - (void) db; + void PackedTable::SortAggregationByColumn(std::string sort_column, bool sort_order) { if (m_aggregation.agg_params[0].public_name == sort_column) { @@ -385,7 +382,7 @@ namespace DataModel } - void PackedTable::AggregateRow(QueryManager* db, int row_index, int map_index) + void PackedTable::AggregateRow(SystemDatabase* db, int row_index, int map_index) { // Aggregate a single packed row into the target aggregation map. // High-level flow: @@ -542,7 +539,7 @@ namespace DataModel } } - void PackedTable::SortByColumn(QueryManager* db, std::string column, bool ascending) + void PackedTable::SortByColumn(SystemDatabase* db, std::string column, bool ascending) { auto it = std::find_if(m_merged_columns.begin(), m_merged_columns.end(), [column](MergedColumnDef& cdef) { return cdef.m_name == column; }); if (it != m_merged_columns.end()) diff --git a/src/model/src/database/rocprofvis_db_packed_storage.h b/src/model/src/database/rocprofvis_db_packed_storage.h index 6a817bed1..b8b4804d2 100644 --- a/src/model/src/database/rocprofvis_db_packed_storage.h +++ b/src/model/src/database/rocprofvis_db_packed_storage.h @@ -23,7 +23,7 @@ namespace RocProfVis { namespace DataModel { - typedef struct rocprofvis_db_sqlite_track_identifier_index_t + typedef struct rocprofvis_db_track_identifier_index_t { uint32_t nid_index = INVALID_INDEX; uint32_t pid_index = INVALID_INDEX; @@ -32,7 +32,7 @@ namespace DataModel uint32_t stream_index = INVALID_INDEX; bool is_pmc_identifier = false; bool is_rocpd_pmc = false; - } rocprofvis_db_sqlite_track_identifier_index_t; + } rocprofvis_db_track_identifier_index_t; enum class ColumnType : uint8_t { @@ -154,7 +154,7 @@ namespace DataModel void PlaceValue(size_t col, double value); void PlaceValue(size_t col, uint64_t value); - Numeric GetMergeTableValue(uint8_t op, size_t row, size_t col, QueryManager* requestor) const; + Numeric GetMergeTableValue(uint8_t op, size_t row, size_t col) const; uint8_t GetOperationValue(size_t row) const; size_t ColumnCount() const { return m_columns.size(); } @@ -167,8 +167,8 @@ namespace DataModel const std::vector& GetColumns() const { return m_columns; } const std::vector& GetAggregationSpec() const { return m_aggregation.agg_params; } const std::vector& GetMergedColumns() const { return m_merged_columns; } - DbInstance* GetDbInstanceForRow(QueryManager * db, int row_index); - DbInstance* GetDbInstanceForRow(QueryManager* db, PackedRow* row); + DbInstance* GetDbInstanceForRow(SystemDatabase * db, int row_index); + DbInstance* GetDbInstanceForRow(SystemDatabase* db, PackedRow* row); uint32_t SortedIndex(uint32_t index) { return m_sort_order[index]; }; void Merge(std::vector>& tables); @@ -179,15 +179,15 @@ namespace DataModel void RemoveDuplicates(); void CreateSortOrderArray(); - void SortByColumn(QueryManager * db, std::string column, bool ascending); + void SortByColumn(SystemDatabase * db, std::string column, bool ascending); bool SetupAggregation(std::string agg_spec, int num_threads); void FinalizeAggregation(); void ClearAggregation(); - void AggregateRow(QueryManager * db, int row_index, int map_index); - void SortAggregationByColumn(QueryManager* db, std::string sort_column, bool sort_order); + void AggregateRow(SystemDatabase * db, int row_index, int map_index); + void SortAggregationByColumn(std::string sort_column, bool sort_order); void RemoveRowsForSetOfTracks(std::set & selected_tracks, std::set & unselected_tracks, bool remove_all); - static const char* ConvertSqlStringReference(QueryManager* db, uint32_t column_index, uint64_t index, uint32_t node_id, bool & numeric_string); + static const char* ConvertSqlStringReference(SystemDatabase* db, uint32_t column_index, uint64_t index, uint32_t node_id, bool & numeric_string); void ResetTrackIdetifiers() { track_ids_indices.nid_index = track_ids_indices.process_index = track_ids_indices.sub_process_index = track_ids_indices.stream_index = track_ids_indices.pid_index = INVALID_INDEX; @@ -203,7 +203,7 @@ namespace DataModel size_t m_currentRow = static_cast(INVALID_INDEX_64); Aggregation m_aggregation; public: - rocprofvis_db_sqlite_track_identifier_index_t track_ids_indices; + rocprofvis_db_track_identifier_index_t track_ids_indices; }; diff --git a/src/model/src/database/rocprofvis_db_profile.cpp b/src/model/src/database/rocprofvis_db_profile.cpp index 0f81c675e..8928e1637 100644 --- a/src/model/src/database/rocprofvis_db_profile.cpp +++ b/src/model/src/database/rocprofvis_db_profile.cpp @@ -23,8 +23,8 @@ ProfileDatabase::CallbackAddStackTrace(void* data, int argc, sqlite3_stmt* stmt, ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); ROCPROFVIS_ASSERT_MSG_RETURN(argc == 7, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); void* func = (void*)&CallbackAddStackTrace; - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) data; ProfileDatabase* db = (ProfileDatabase*) callback_params->db; rocprofvis_db_stack_data_t record = {"","","",0,0}; static const char * empty_blob = "{}"; @@ -110,7 +110,7 @@ ProfileDatabase::CallbackAddStackTrace(void* data, int argc, sqlite3_stmt* stmt, int ProfileDatabase::CallbackCacheTable(void *data, int argc, sqlite3_stmt* stmt, char **azColName){ ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackCacheTable; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ProfileDatabase* db = (ProfileDatabase*)callback_params->db; DatabaseCache * ref_tables = (DatabaseCache *)callback_params->handle; std::lock_guard lock(db->m_lock); @@ -147,7 +147,7 @@ int ProfileDatabase::CallBackAddTrack(void *data, int argc, sqlite3_stmt* stmt, ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallBackAddTrack; rocprofvis_dm_track_params_t track_params = {0}; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); ProfileDatabase* db = (ProfileDatabase*)callback_params->db; if(callback_params->future->Interrupted()) return SQLITE_ABORT; @@ -189,7 +189,7 @@ int ProfileDatabase::CallBackLoadTrack(void *data, int argc, sqlite3_stmt* stmt, ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallBackLoadTrack; rocprofvis_dm_track_params_t track_params = {0}; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); uint32_t db_instance = callback_params->db_instance->GuidIndex(); ProfileDatabase* db = (ProfileDatabase*)callback_params->db; @@ -246,8 +246,8 @@ ProfileDatabase::CallbackGetTrackRecordsCount(void* data, int argc, sqlite3_stmt ROCPROFVIS_ASSERT_MSG_RETURN(argc == 3, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void *func = (void*)&CallbackGetTrackRecordsCount; - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) data; ProfileDatabase* db = (ProfileDatabase*) callback_params->db; if(callback_params->future->Interrupted()) return SQLITE_ABORT; uint32_t index = db->Sqlite3ColumnInt(func, stmt, azColName, 2); @@ -265,8 +265,8 @@ ProfileDatabase::CallbackTrimTableQuery(void* data, int argc, sqlite3_stmt* stmt (void) argc; ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void *func = (void*)&CallbackTrimTableQuery; - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) data; rocprofvis_db_sqlite_trim_parameters* params = (rocprofvis_db_sqlite_trim_parameters*) callback_params->handle; ProfileDatabase* db = (ProfileDatabase*) callback_params->db; @@ -286,8 +286,8 @@ int ProfileDatabase::CallbackGetTrackProperties(void* data, int argc, sqlite3_st ROCPROFVIS_ASSERT_MSG_RETURN(argc == 5, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void *func = (void*)&CallbackGetTrackProperties; - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); uint32_t db_instance = callback_params->db_instance->GuidIndex(); ProfileDatabase* db = (ProfileDatabase*) callback_params->db; @@ -312,7 +312,7 @@ int ProfileDatabase::CallbackAddAnyRecord(void* data, int argc, sqlite3_stmt* st ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void *func = (void*)&CallbackAddAnyRecord; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); uint32_t db_instance = callback_params->db_instance->GuidIndex(); ProfileDatabase* db = (ProfileDatabase*)callback_params->db; @@ -352,7 +352,7 @@ int ProfileDatabase::CallbackAddFlowTrace(void *data, int argc, sqlite3_stmt* st ROCPROFVIS_ASSERT_MSG_RETURN(argc == rocprofvis_db_sqlite_dataflow_query_format::NUM_PARAMS, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); void* func = (void*)&CallbackAddFlowTrace; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); uint32_t db_instance = callback_params->db_instance->GuidIndex(); ProfileDatabase* db = (ProfileDatabase*)callback_params->db; @@ -385,7 +385,7 @@ int ProfileDatabase::CallbackAddFlowTrace(void *data, int argc, sqlite3_stmt* st int ProfileDatabase::CallbackAddExtInfo(void* data, int argc, sqlite3_stmt* stmt, char** azColName) { ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackAddExtInfo; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); ProfileDatabase* db = (ProfileDatabase*)callback_params->db; rocprofvis_db_ext_data_t record; @@ -423,7 +423,7 @@ int ProfileDatabase::CallbackAddArgumentsInfo(void* data, int argc, sqlite3_stmt (void) argc; ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackAddArgumentsInfo; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); ProfileDatabase* db = (ProfileDatabase*)callback_params->db; rocprofvis_db_argument_data_t record; @@ -443,7 +443,7 @@ int ProfileDatabase::CallbackAddEssentialInfo(void* data, int argc, sqlite3_stmt ROCPROFVIS_ASSERT_MSG_RETURN(argc == rocprofvis_db_sqlite_essential_data_query_format::NUM_PARAMS, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); void* func = (void*)&CallbackAddEssentialInfo; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); ProfileDatabase* db = (ProfileDatabase*)callback_params->db; rocprofvis_db_ext_data_t record; @@ -487,7 +487,7 @@ int ProfileDatabase::CallbackAddEssentialInfo(void* data, int argc, sqlite3_stmt record.category = "Track"; record.name = "levelForTrack"; record.type = kRPVDataTypeInt; - column_data = std::to_string(db->Sqlite3ColumnInt64(func, stmt, azColName, argc - 2)); + column_data = std::to_string(db->Sqlite3ColumnInt64(func, stmt, azColName, argc - 3)); record.data = column_data.c_str(); record.category_enum = kRocProfVisEventEssentialDataLevel; record.db_instance = callback_params->db_instance->GuidIndex(); @@ -545,7 +545,7 @@ bool ProfileDatabase::FindTrack(rocprofvis_dm_track_category_t category, uint64_ void ProfileDatabase::GetTrackIdentifierIndices( int column_index, char** azColName, - rocprofvis_db_sqlite_track_identifier_index_t& track_ids_indices) + rocprofvis_db_track_identifier_index_t& track_ids_indices) { std::string column_name = azColName[column_index]; @@ -820,8 +820,8 @@ int ProfileDatabase::CalculateEventLevels(void* data, int argc, sqlite3_stmt* st ROCPROFVIS_ASSERT_MSG_RETURN(argc == rocprofvis_db_sqlite_level_query_format::NUM_PARAMS+1 , ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CalculateEventLevels; - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) data; ProfileDatabase* db = (ProfileDatabase*) callback_params->db; if(callback_params->future->Interrupted()) { @@ -1047,7 +1047,7 @@ int ProfileDatabase::CallBackLoadHistogram(void* data, int argc, sqlite3_stmt* s ROCPROFVIS_ASSERT_MSG_RETURN(argc == 5, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallBackLoadHistogram; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ProfileDatabase* db = (ProfileDatabase*)callback_params->db; if (callback_params->future->Interrupted()) return SQLITE_ABORT; uint32_t track_id = db->Sqlite3ColumnInt(func, stmt, azColName, 1); diff --git a/src/model/src/database/rocprofvis_db_profile.h b/src/model/src/database/rocprofvis_db_profile.h index 8afa19381..b1ca2b0d1 100644 --- a/src/model/src/database/rocprofvis_db_profile.h +++ b/src/model/src/database/rocprofvis_db_profile.h @@ -154,7 +154,7 @@ class ProfileDatabase : public QueryManager uint64_t GetHistogramQueryAndSchemaHash(); // get indeces of colums representing track identifiers - void GetTrackIdentifierIndices(int column_index, char** azColName, rocprofvis_db_sqlite_track_identifier_index_t& track_ids_indices) override; + void GetTrackIdentifierIndices(int column_index, char** azColName, rocprofvis_db_track_identifier_index_t& track_ids_indices) override; // process track discovery data and populate track parameters virtual int ProcessTrack(rocprofvis_dm_track_params_t& track_params, std::vector & newqueries) = 0; diff --git a/src/model/src/database/rocprofvis_db_profiler_hub.cpp b/src/model/src/database/rocprofvis_db_profiler_hub.cpp new file mode 100644 index 000000000..147e4a2d8 --- /dev/null +++ b/src/model/src/database/rocprofvis_db_profiler_hub.cpp @@ -0,0 +1,921 @@ +// Copyright Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include "rocprofvis_db_profiler_hub.h" +#include "rocprofvis_db_table_processor.h" +#include "rocprofvis_db_query_builder.h" +#include +//#include "json.h" + +namespace RocProfVis +{ +namespace DataModel +{ + + std::string ProfilerHub::TableColumnText(void* func, void* handle, char** azColName, int index) { + ProfilerHubTableRow* row = (ProfilerHubTableRow*)handle; + auto cell = row->CellAt(index); + if (std::holds_alternative(cell->value)) + return std::get(cell->value); + else if (std::holds_alternative(cell->value)) + return std::to_string(std::get(cell->value)); + else if (std::holds_alternative(cell->value)) + return std::to_string(std::get(cell->value)); + return ""; + } + int ProfilerHub::TableColumnInt(void* func, void* handle, char** azColName, int index) { + ProfilerHubTableRow* row = (ProfilerHubTableRow*)handle; + auto cell = row->CellAt(index); + if (std::holds_alternative(cell->value)) + return std::get(cell->value); + return 0; + } + int64_t ProfilerHub::TableColumnInt64(void* func, void* handle, char** azColName, int index) { + ProfilerHubTableRow* row = (ProfilerHubTableRow*)handle; + auto cell = row->CellAt(index); + if (std::holds_alternative(cell->value)) + return std::get(cell->value); + return 0; + } + double ProfilerHub::TableColumnDouble(void* func, void* handle, char** azColName, int index) { + ProfilerHubTableRow* row = (ProfilerHubTableRow*)handle; + auto cell = row->CellAt(index); + if (std::holds_alternative(cell->value)) + return std::get(cell->value); + return 0; + } + + rocprofvis_dm_result_t ProfilerHub::ConvertProfilerHubResult(profiler_hub_result_t ph_result) + { + switch (ph_result) + { + case kProfilerHubStatusSuccess: + return kRocProfVisDmResultSuccess; + case kProfilerHubStatusUnknownError: + return kRocProfVisDmResultUnknownError; + case kProfilerHubStatusNotSupported: + return kRocProfVisDmResultNotSupported; + case kProfilerHubStatusInvalidArgument: + return kRocProfVisDmResultInvalidParameter; + case kProfilerHubStatusTimeout: + return kRocProfVisDmResultTimeout; + case kProfilerHubStatusNotLoaded: + return kRocProfVisDmResultNotLoaded; + } + return kRocProfVisDmResultUnknownError; + } + + rocprofvis_dm_event_operation_t ProfilerHub::ConvertProfilerHubEventType(profiler_hub::reader_types::event_type_t event_type) + { + switch (event_type) + { + case profiler_hub::reader_types::event_type_t::region: + return kRocProfVisDmOperationLaunch; + case profiler_hub::reader_types::event_type_t::kernel_dispatch: + return kRocProfVisDmOperationDispatch; + case profiler_hub::reader_types::event_type_t::memory_copy: + return kRocProfVisDmOperationMemoryCopy; + case profiler_hub::reader_types::event_type_t::memory_allocate: + return kRocProfVisDmOperationMemoryAllocate; + case profiler_hub::reader_types::event_type_t::sample: + return kRocProfVisDmOperationLaunchSample; + case profiler_hub::reader_types::event_type_t::pmc_event: + return kRocProfVisDmOperationNoOp; + } + return kRocProfVisDmOperationNoOp; + } + + + rocprofvis_dm_result_t ProfilerHub::Open() { + + m_ph_trace = profiler_hub_open_trace(Path()); + return m_ph_trace ? kRocProfVisDmResultSuccess : kRocProfVisDmResultUnknownError; + } + + rocprofvis_dm_result_t ProfilerHub::Close() { + + return ConvertProfilerHubResult(profiler_hub_close_trace(m_ph_trace)); + } + + void ProfilerHub::IdentificatorNamesUpdate(rocprofvis_dm_track_params_t& track_params) + { + ROCPROFVIS_ASSERT_MSG_RETURN(track_params.track_indentifiers.db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, ); + DbInstance* db_instance = (DbInstance*)track_params.track_indentifiers.db_instance; + if(track_params.track_indentifiers.category == kRocProfVisDmRegionMainTrack || + track_params.track_indentifiers.category == kRocProfVisDmRegionSampleTrack) + { + if (track_params.track_indentifiers.name[TRACK_ID_PID].empty()) + { + track_params.track_indentifiers.name[TRACK_ID_PID] = CachedTables(db_instance->GuidIndex())->GetTableCell("Process", track_params.track_indentifiers.id[TRACK_ID_PID], "command"); + track_params.track_indentifiers.name[TRACK_ID_PID] += "("; + track_params.track_indentifiers.name[TRACK_ID_PID] += std::to_string(track_params.track_indentifiers.id[TRACK_ID_PID]); + track_params.track_indentifiers.name[TRACK_ID_PID] += ")"; + } + + if (track_params.track_indentifiers.name[TRACK_ID_TID].empty()) + { + track_params.track_indentifiers.name[TRACK_ID_TID] = CachedTables(db_instance->GuidIndex())->GetTableCell("Thread", track_params.track_indentifiers.id[TRACK_ID_TID], "name"); + track_params.track_indentifiers.name[TRACK_ID_TID] += "("; + track_params.track_indentifiers.name[TRACK_ID_TID] += std::to_string(track_params.track_indentifiers.id[TRACK_ID_TID]); + track_params.track_indentifiers.name[TRACK_ID_TID] += ")"; + } + } + else if(track_params.track_indentifiers.category == kRocProfVisDmKernelDispatchTrack || + track_params.track_indentifiers.category == kRocProfVisDmMemoryAllocationTrack || + track_params.track_indentifiers.category == kRocProfVisDmMemoryCopyTrack || + track_params.track_indentifiers.category == kRocProfVisDmPmcTrack) + { + if (track_params.track_indentifiers.name[TRACK_ID_AGENT].empty()) + { + track_params.track_indentifiers.name[TRACK_ID_AGENT] = CachedTables(db_instance->GuidIndex())->GetTableCell("Agent", track_params.track_indentifiers.id[TRACK_ID_AGENT], "product_name"); + track_params.track_indentifiers.name[TRACK_ID_AGENT] += "("; + track_params.track_indentifiers.name[TRACK_ID_AGENT] += CachedTables(db_instance->GuidIndex())->GetTableCell("Agent", track_params.track_indentifiers.id[TRACK_ID_AGENT], "type_index"); + track_params.track_indentifiers.name[TRACK_ID_AGENT] += ")"; + } + if(track_params.track_indentifiers.category == kRocProfVisDmKernelDispatchTrack || + track_params.track_indentifiers.category == kRocProfVisDmMemoryAllocationTrack || + track_params.track_indentifiers.category == kRocProfVisDmMemoryCopyTrack) + { + if (track_params.track_indentifiers.name[TRACK_ID_QUEUE].empty()) + { + track_params.track_indentifiers.name[TRACK_ID_QUEUE] = CachedTables(db_instance->GuidIndex())->GetTableCell("Queue", track_params.track_indentifiers.id[TRACK_ID_QUEUE], "name"); + } + } + else { + if (track_params.track_indentifiers.name[TRACK_ID_COUNTER].empty()) + { + track_params.track_indentifiers.name[TRACK_ID_COUNTER] = CachedTables(db_instance->GuidIndex())->GetTableCell("PMC", track_params.track_indentifiers.id[TRACK_ID_QUEUE], "symbol"); + } + } + + } + else if(track_params.track_indentifiers.category == kRocProfVisDmStreamTrack) + { + if (track_params.track_indentifiers.name[TRACK_ID_STREAM].empty()) + { + track_params.track_indentifiers.name[TRACK_ID_STREAM] = CachedTables(db_instance->GuidIndex())->GetTableCell("Stream", track_params.track_indentifiers.id[TRACK_ID_STREAM], "name"); + track_params.track_indentifiers.name[TRACK_ID_PID] = track_params.track_indentifiers.name[TRACK_ID_STREAM]; + } + } + } + + rocprofvis_dm_result_t ProfilerHub::ProcessTrack(rocprofvis_dm_track_params_t& track_params) + { + ROCPROFVIS_ASSERT_MSG_RETURN(track_params.track_indentifiers.db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + DbInstance* db_instance = (DbInstance*)track_params.track_indentifiers.db_instance; + track_params.track_indentifiers.source_type = kRPVSystemSourceRocprof; + rocprofvis_dm_track_params_it it = TrackTracker()->FindTrackParamsIterator(track_params.track_indentifiers, db_instance->GuidIndex()); + if(it == TrackPropertiesEnd()) + { + TrackTracker()->AddTrack(track_params.track_indentifiers, db_instance->GuidIndex(), track_params.track_indentifiers.track_id); + + IdentificatorNamesUpdate(track_params); + + if (kRocProfVisDmResultSuccess != AddTrackProperties(track_params)) return kRocProfVisDmResultUnknownError; + + if (BindObject()->FuncAddTrack(BindObject()->trace_object, TrackPropertiesLast()) != kRocProfVisDmResultSuccess) return kRocProfVisDmResultUnknownError; + + if (BindObject()->FuncAddTopologyNode(BindObject()->trace_object, &track_params.track_indentifiers) != kRocProfVisDmResultSuccess) return kRocProfVisDmResultUnknownError; + + if (CachedTables(db_instance->GuidIndex())->PopulateTrackExtendedDataTemplate( + this, + db_instance->GuidIndex(), + "Node", + track_params.track_indentifiers.id[TRACK_ID_NODE]) != kRocProfVisDmResultSuccess) return kRocProfVisDmResultUnknownError; + if(track_params.track_indentifiers.category == kRocProfVisDmRegionMainTrack || + track_params.track_indentifiers.category == kRocProfVisDmRegionSampleTrack) + { + if (CachedTables(db_instance->GuidIndex())->PopulateTrackExtendedDataTemplate( + this, db_instance->GuidIndex(), + "Process", + track_params.track_indentifiers.id[TRACK_ID_PID]) != kRocProfVisDmResultSuccess) return kRocProfVisDmResultUnknownError; + if (CachedTables(db_instance->GuidIndex())->PopulateTrackExtendedDataTemplate( + this, db_instance->GuidIndex(), + "Thread", + track_params.track_indentifiers.id[TRACK_ID_TID]) != kRocProfVisDmResultSuccess) return kRocProfVisDmResultUnknownError; + } + else if(track_params.track_indentifiers.category == kRocProfVisDmStreamTrack) + { + if (CachedTables(db_instance->GuidIndex())->PopulateTrackExtendedDataTemplate( + this, + db_instance->GuidIndex(), + "Process", + track_params.track_indentifiers.id[TRACK_ID_PID]) != kRocProfVisDmResultSuccess) return kRocProfVisDmResultUnknownError; + if (CachedTables(db_instance->GuidIndex())->PopulateTrackExtendedDataTemplate( + this, + db_instance->GuidIndex(), + "Stream", + track_params.track_indentifiers.id[TRACK_ID_STREAM]) != kRocProfVisDmResultSuccess) return kRocProfVisDmResultUnknownError; + } + else if(track_params.track_indentifiers.category == kRocProfVisDmKernelDispatchTrack || + track_params.track_indentifiers.category == kRocProfVisDmMemoryAllocationTrack || + track_params.track_indentifiers.category == kRocProfVisDmMemoryCopyTrack || + track_params.track_indentifiers.category == kRocProfVisDmPmcTrack) + { + if (CachedTables(db_instance->GuidIndex())->PopulateTrackExtendedDataTemplate( + this, + db_instance->GuidIndex(), + "Agent", + track_params.track_indentifiers.id[TRACK_ID_AGENT]) != kRocProfVisDmResultSuccess) return kRocProfVisDmResultUnknownError; + if(track_params.track_indentifiers.category == kRocProfVisDmKernelDispatchTrack || + track_params.track_indentifiers.category == kRocProfVisDmMemoryAllocationTrack || + track_params.track_indentifiers.category == kRocProfVisDmMemoryCopyTrack) + { + if (CachedTables(db_instance->GuidIndex())->PopulateTrackExtendedDataTemplate( + this, + db_instance->GuidIndex(), + "Queue", + track_params.track_indentifiers.id[TRACK_ID_QUEUE]) != kRocProfVisDmResultSuccess) return kRocProfVisDmResultUnknownError; + } + else + { + if(track_params.track_indentifiers.id[TRACK_ID_QUEUE] > 0) + if (CachedTables(db_instance->GuidIndex())->PopulateTrackExtendedDataTemplate( + this, + db_instance->GuidIndex(), + "PMC", + track_params.track_indentifiers.id[TRACK_ID_COUNTER]) != kRocProfVisDmResultSuccess) return kRocProfVisDmResultUnknownError; + } + } + return kRocProfVisDmResultSuccess; + } + + return kRocProfVisDmResultNotLoaded; + } + + + rocprofvis_dm_result_t ProfilerHub::ReadTraceMetadata(Future* future) + { + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + while (true) + { + ROCPROFVIS_ASSERT_MSG_BREAK(BindObject()->trace_properties, ERROR_TRACE_PROPERTIES_CANNOT_BE_NULL); + std::string value; + rocprofvis_dm_result_t result = kRocProfVisDmResultSuccess; + profiler_hub_future_handle_t ph_future = future->AddProfilerHubFuture(); + ShowProgress(1, "Call profiler hub to read trace metadata", kRPVDbBusy, future); + result = ConvertProfilerHubResult(profiler_hub_read_metadata(ph_future, m_ph_trace)); + if (result != kRocProfVisDmResultSuccess) + break; + + result = ConvertProfilerHubResult(future->WaitAndDeleteProfilerHubFuture(ph_future)); + if (result != kRocProfVisDmResultSuccess) + break; + + TraceProperties()->metadata_loaded=true; + BindObject()->FuncMetadataLoaded(BindObject()->trace_object); + ShowProgress(100-future->Progress(), "Trace metadata successfully loaded", kRPVDbSuccess, future ); + return future->SetPromise(kRocProfVisDmResultSuccess); + + } + ShowProgress(0, "Trace metadata not loaded!", kRPVDbError, future ); + return future->SetPromise(future->Interrupted() ? kRocProfVisDmResultDbAbort : kRocProfVisDmResultDbAccessFailed); + } + + rocprofvis_dm_result_t ProfilerHub::ExecuteQuery( + rocprofvis_dm_charptr_t query, + rocprofvis_dm_charptr_t description, + Future* future) { + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + while (true) + { + ROCPROFVIS_ASSERT_MSG_BREAK(BindObject()->trace_properties, ERROR_TRACE_PROPERTIES_CANNOT_BE_NULL); + ROCPROFVIS_ASSERT_MSG_BREAK(BindObject()->trace_properties->metadata_loaded, ERROR_METADATA_IS_NOT_LOADED); + rocprofvis_dm_table_t table = BindObject()->FuncAddTable(BindObject()->trace_object, query, description); + ROCPROFVIS_ASSERT_MSG_RETURN(table, ERROR_TABLE_CANNOT_BE_NULL, kRocProfVisDmResultUnknownError); + std::unordered_map> queries; + std::vector commands; + std::set tracks; + if (TableProcessor::IsCompoundQuery(query, queries, tracks, commands)) + { + auto it = std::find_if(commands.begin(), commands.end(), [](rocprofvis_db_compound_query_command& cmd) { return cmd.name == "TYPE"; }); + rocprofvis_db_compound_table_type data_type = kRPVTableDataTypeEvent; + if (it != commands.end()) + { + data_type = (rocprofvis_db_compound_table_type)std::atol(it->parameter.c_str()); + } + bool query_updated = !m_table_processor[data_type].IsCurrentQuery(queries); + m_table_processor[data_type].SaveCurrentQuery(queries); + if (kRocProfVisDmResultSuccess != m_table_processor[data_type].ExecuteCompoundQuery(future, queries, tracks, commands, table, query_updated)) break; + } + else + { + ShowProgress(100, "Direct database query is not supported!",kRPVDbSuccess, future); + return future->SetPromise(kRocProfVisDmResultNotSupported); + } + + ShowProgress(100, "Query successfully executed!",kRPVDbSuccess, future); + return future->SetPromise(kRocProfVisDmResultSuccess); + } + ShowProgress(0, "Query could not be executed!", kRPVDbError, future ); + return future->SetPromise(future->Interrupted() ? kRocProfVisDmResultDbAbort : kRocProfVisDmResultDbAccessFailed); + return kRocProfVisDmResultDbAccessFailed; + } + + rocprofvis_dm_result_t ProfilerHub::ExportTableCSV( + rocprofvis_dm_charptr_t query, + rocprofvis_dm_charptr_t file_path, + Future* future) { + return kRocProfVisDmResultNotSupported; + } + + rocprofvis_dm_result_t ProfilerHub::BuildTableQuery( + rocprofvis_dm_table_use_case_enum_t use_case, + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t tracks, + rocprofvis_dm_processor_identifiers_ptr processor, + rocprofvis_dm_charptr_t filter, + rocprofvis_dm_charptr_t group, + rocprofvis_dm_charptr_t group_cols, + rocprofvis_dm_charptr_t sort_column, + rocprofvis_dm_sort_order_t sort_order, + uint64_t max_count, + uint64_t offset, + bool count_only, + rocprofvis_dm_string_t& query) { + + ParamSerializer ser; + ser.set("start",start); + ser.set("end",end); + if (processor) + { + if (processor->node_id) + { + ser.set("node",*processor->node_id); + if (processor->agent_id) + { + ser.set("agent",*processor->agent_id); + } + } + } + std::vector v_tracks; + for (int i = 0; i < num; i++) + { + v_tracks.push_back(tracks[i]); + } + ser.setArray("tracks", std::move(v_tracks)); + query = ser.toString(); + + bool sample_query = false; + if(TABLE_QUERY_UNPACK_OP_TYPE(tracks[0]) == 0) + { + sample_query = + TrackPropertiesAt(tracks[0])->track_indentifiers.category == + kRocProfVisDmPmcTrack; + } + else + { + sample_query = + (rocprofvis_dm_event_operation_t) TABLE_QUERY_UNPACK_OP_TYPE( + tracks[0]) == kRocProfVisDmOperationNoOp; + } + + return TableProcessor::BuildTableSemanticSubQuery( + use_case, + filter, + group, + group_cols, + sort_column, + sort_order, + max_count, + offset, + count_only, + sample_query, + query); + + } + + rocprofvis_dm_result_t ProfilerHub::BuildEventSearchQuery( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t ops, + rocprofvis_dm_processor_identifiers_ptr processor, + rocprofvis_dm_num_string_table_filters_t num_string_table_filters, + rocprofvis_dm_string_table_filters_t string_table_filters, + bool include_substring, + bool include_category, + bool partial_matching, + rocprofvis_dm_charptr_t sort_column, + rocprofvis_dm_sort_order_t sort_order, + uint64_t max_count, + uint64_t offset, + bool count_only, + rocprofvis_dm_string_t& query) { + + ParamSerializer ser; + ser.set("start",start); + ser.set("end",end); + if (processor) + { + if (processor->node_id) + { + ser.set("node",*processor->node_id); + if (processor->agent_id) + { + ser.set("agent",*processor->agent_id); + } + } + } + std::vector v_ops; + for (int i = 0; i < num; i++) + { + v_ops.push_back(ops[i]); + } + ser.setArray("tracks", std::move(v_ops)); + + std::vector v_filters; + for (int i = 0; i < num_string_table_filters; i++) + { + v_filters.push_back(string_table_filters[i]); + } + ser.setArray("tracks", std::move(v_ops)); + ser.setArray("filters", std::move(v_filters)); + ser.set("include_substring",include_substring); + ser.set("include_category",include_category); + ser.set("partial_matching",partial_matching); + + query = ser.toString(); + + return TableProcessor::BuildTableSemanticSubQuery( + kRPVDMTableUseCaseEventSearch, + nullptr, + nullptr, + nullptr, + sort_column, + sort_order, + max_count, + offset, + count_only, + false, + query); + } + + rocprofvis_dm_result_t ProfilerHub::SaveTrimmedData( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_charptr_t new_db_path, + Future* future) { + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + while(true) + { + profiler_hub_future_handle_t ph_future = future->AddProfilerHubFuture(); + profiler_hub_result_t result = profiler_hub_trim_save_trace(ph_future, m_ph_trace, start, end, new_db_path); + if (result != kProfilerHubStatusSuccess) + break; + result = future->WaitAndDeleteProfilerHubFuture(ph_future); + if (result != kProfilerHubStatusSuccess) + break; + ShowProgress(100 - future->Progress(), "Trace successfully trimmed!", kRPVDbSuccess, future); + return future->SetPromise(kRocProfVisDmResultSuccess); + } + + ShowProgress(0, "Failed trimming trace!", kRPVDbError, future ); + return future->SetPromise(future->Interrupted() ? kRocProfVisDmResultDbAbort : kRocProfVisDmResultDbAccessFailed); + } + + rocprofvis_dm_result_t ProfilerHub::ReadFlowTraceInfo( + rocprofvis_dm_event_id_t event_id, + Future* future) { + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + while (true) + { + profiler_hub_future_handle_t ph_future = future->AddProfilerHubFuture(); + rocprofvis_dm_flowtrace_t container = BindObject()->FuncAddFlowTrace(BindObject()->trace_object, event_id); + profiler_hub_result_t result = profiler_hub_get_event_data_flow( + ph_future, + m_ph_trace, + event_id.bitfield.event_node, + container, + (profiler_hub_event_operation_t)event_id.bitfield.event_op, + event_id.bitfield.event_id); + if (result != kProfilerHubStatusSuccess) + break; + result = future->WaitAndDeleteProfilerHubFuture(ph_future); + if (result != kProfilerHubStatusSuccess) + break; + ShowProgress(100 - future->Progress(), "Successfully read flow trace!", kRPVDbSuccess, future); + return future->SetPromise(kRocProfVisDmResultSuccess); + } + ShowProgress(0, "Failed reading flow trace!", kRPVDbError, future ); + return future->SetPromise(future->Interrupted() ? kRocProfVisDmResultDbAbort : kRocProfVisDmResultDbAccessFailed); + } + + rocprofvis_dm_result_t ProfilerHub::ReadStackTraceInfo( + rocprofvis_dm_event_id_t event_id, + Future* future) { + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + while (true) + { + profiler_hub_future_handle_t ph_future = future->AddProfilerHubFuture(); + rocprofvis_dm_stacktrace_t container = BindObject()->FuncAddFlowTrace(BindObject()->trace_object, event_id); + profiler_hub_result_t result = profiler_hub_get_event_stack_trace( + ph_future, + m_ph_trace, + event_id.bitfield.event_node, + container, + (profiler_hub_event_operation_t)event_id.bitfield.event_op, + event_id.bitfield.event_id); + if (result != kProfilerHubStatusSuccess) + break; + result = future->WaitAndDeleteProfilerHubFuture(ph_future); + if (result != kProfilerHubStatusSuccess) + break; + ShowProgress(100 - future->Progress(), "Successfully read flow trace!", kRPVDbSuccess, future); + return future->SetPromise(kRocProfVisDmResultSuccess); + } + ShowProgress(0, "Failed reading flow trace!", kRPVDbError, future ); + return future->SetPromise(future->Interrupted() ? kRocProfVisDmResultDbAbort : kRocProfVisDmResultDbAccessFailed); + } + + rocprofvis_dm_result_t ProfilerHub::ReadExtEventInfo( + rocprofvis_dm_event_id_t event_id, + Future* future) { + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + while (true) + { + profiler_hub_future_handle_t ph_future = future->AddProfilerHubFuture(); + rocprofvis_dm_extdata_t container = BindObject()->FuncAddFlowTrace(BindObject()->trace_object, event_id); + profiler_hub_result_t result = profiler_hub_get_event_extended_data( + ph_future, + m_ph_trace, + event_id.bitfield.event_node, + container, + (profiler_hub_event_operation_t)event_id.bitfield.event_op, + event_id.bitfield.event_id); + if (result != kProfilerHubStatusSuccess) + break; + result = future->WaitAndDeleteProfilerHubFuture(ph_future); + if (result != kProfilerHubStatusSuccess) + break; + ShowProgress(100 - future->Progress(), "Successfully read flow trace!", kRPVDbSuccess, future); + return future->SetPromise(kRocProfVisDmResultSuccess); + } + ShowProgress(0, "Failed reading flow trace!", kRPVDbError, future ); + return future->SetPromise(future->Interrupted() ? kRocProfVisDmResultDbAbort : kRocProfVisDmResultDbAccessFailed); + } + + rocprofvis_dm_result_t ProfilerHub::ReadTraceSlice( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_hashed_timestamp_tag_t tag, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t tracks, + Future* future) { + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG(BindObject()->trace_properties, ERROR_TRACE_PROPERTIES_CANNOT_BE_NULL); + ROCPROFVIS_ASSERT_MSG(BindObject()->trace_properties->metadata_loaded, ERROR_METADATA_IS_NOT_LOADED); + ROCPROFVIS_ASSERT_MSG(num == 1, ERROR_UNSUPPORTED_FEATURE); + + rocprofvis_dm_track_params_t* props = TrackPropertiesAt(*tracks); + if(props->track_indentifiers.category == kRocProfVisDmPmcTrack) + { + return ReadTracePMCSlice(start, end, tag, tracks, true, true, future); + } + else + { + while (true) + { + std::string slice_query; + slice_array_t slices; + + auto it = std::find_if(TrackPropertiesBegin(), TrackPropertiesEnd(), + [tracks](std::unique_ptr& params) { + return params.get()->track_indentifiers.track_id == *tracks; + }); + if (it == TrackPropertiesEnd()) break; + + slices[*tracks]=BindObject()->FuncAddSlice(BindObject()->trace_object, *tracks, start, end, tag); + profiler_hub_future_handle_t ph_future = future->AddProfilerHubFuture(); + profiler_hub_result_t result = + profiler_hub_get_time_slice( + ph_future, + m_ph_trace, + it->get()->track_indentifiers.track_id, + slices[*tracks], + start, + end); + if (result == kProfilerHubStatusSuccess) + { + result = future->WaitAndDeleteProfilerHubFuture(ph_future); + } + if (result == kProfilerHubStatusSuccess) + { + BindObject()->FuncCompleteSlice(slices[*tracks]); + } + else + { + BindObject()->FuncRemoveSlice(BindObject()->trace_object, *tracks, slices[*tracks]); + break; + } + ShowProgress(100 - future->Progress(), "Time slice successfully loaded!", kRPVDbSuccess, future); + return future->SetPromise(kRocProfVisDmResultSuccess); + } + } + + ShowProgress(0, "Not all tracks are loaded!", kRPVDbError, future ); + return future->SetPromise(future->Interrupted() ? kRocProfVisDmResultDbAbort : kRocProfVisDmResultDbAccessFailed); + } + + rocprofvis_dm_result_t ProfilerHub::ReadTracePMCSlice( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_hashed_timestamp_tag_t tag, + rocprofvis_db_track_selection_t tracks, + bool left_neighbor, + bool right_neighbor, + Future* future) { + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG(BindObject()->trace_properties, ERROR_TRACE_PROPERTIES_CANNOT_BE_NULL); + ROCPROFVIS_ASSERT_MSG(BindObject()->trace_properties->metadata_loaded, ERROR_METADATA_IS_NOT_LOADED); + + rocprofvis_dm_track_params_t* props = TrackPropertiesAt(*tracks); + while (true) + { + std::string slice_query; + slice_array_t slices; + + auto it = std::find_if(TrackPropertiesBegin(), TrackPropertiesEnd(), + [tracks](std::unique_ptr& params) { + return params.get()->track_indentifiers.track_id == *tracks; + }); + if (it == TrackPropertiesEnd()) break; + + slices[*tracks]=BindObject()->FuncAddSlice(BindObject()->trace_object, *tracks, start, end, tag); + profiler_hub_future_handle_t ph_future = future->AddProfilerHubFuture(); + profiler_hub_result_t result = + profiler_hub_get_pmc_time_slice( + ph_future, + m_ph_trace, + it->get()->track_indentifiers.track_id, + slices[*tracks], + start, + end, + left_neighbor, + right_neighbor); + future->DeleteProfilerHubFuture(ph_future); + if (result == kProfilerHubStatusSuccess) + { + BindObject()->FuncCompleteSlice(slices[*tracks]); + } + else + { + BindObject()->FuncRemoveSlice(BindObject()->trace_object, *tracks, slices[*tracks]); + break; + } + ShowProgress(100 - future->Progress(), "Time slice successfully loaded!", kRPVDbSuccess, future); + return future->SetPromise(kRocProfVisDmResultSuccess); + } + + ShowProgress(0, "Not all tracks are loaded!", kRPVDbError, future ); + return future->SetPromise(future->Interrupted() ? kRocProfVisDmResultDbAbort : kRocProfVisDmResultDbAccessFailed); + } + + rocprofvis_dm_result_t ProfilerHub::GetEventTablesAsync( + std::vector>& queries, + Future* future, + rocprofvis_dm_handle_t handle, + RpvCallback callback) { + for (auto query : queries) + { + ParamDeserializer des(query.second); + uint64_t start, end, node, agent; + uint64_t* node_prt = &node; + uint64_t* agent_ptr = &agent; + std::vector tracks; + std::vector filters; + bool include_substring, include_category, partial_matching; + if (!des.get("start", start)) + return kRocProfVisDmResultInvalidParameter; + if (!des.get("end", end)) + return kRocProfVisDmResultInvalidParameter; + if (!des.get("node", node)) + node_prt=nullptr; + if (!des.get("agent", agent)) + agent_ptr=nullptr; + bool tracks_selected = des.getArray("tracks", tracks); + bool search_query = des.getArray("filters", filters); + + if (!des.get("include_substring", include_substring)) + return kRocProfVisDmResultInvalidParameter; + if (!des.get("include_category", include_category)) + return kRocProfVisDmResultInvalidParameter; + if (!des.get("partial_matching", partial_matching)) + return kRocProfVisDmResultInvalidParameter; + + //todo : create mask and send it to pprofiler hub + + while (true) + { + profiler_hub_result_t result = kProfilerHubStatusUnknownError; + if (search_query) + { + if (!tracks_selected || tracks.size() == 0) + return kRocProfVisDmResultInvalidParameter; + if (filters.size() == 0) + return kRocProfVisDmResultInvalidParameter; + std::vector filters_cc; + for (auto filter : filters) + filters_cc.push_back(filter.c_str()); + std::vector table_handles; + uint32_t chunk_number = 0; + for (auto& operation : tracks) + { + profiler_hub_future_handle_t ph_future = future->AddProfilerHubFuture(); + table_handles.push_back({ handle, chunk_number++, future, ph_future, (rocprofvis_dm_event_operation_t)operation, query.first}); + result = profiler_hub_get_search_time_slice( + ph_future, + m_ph_trace, + query.first->GuidIndex(), + &table_handles.back(), + (profiler_hub_event_operation_t)operation, + start, + end, + filters_cc.size(), + filters_cc.data()); + if (result != kProfilerHubStatusSuccess) + break; + result = future->WaitAndDeleteProfilerHubFuture(ph_future); + } + for (auto table_handle : table_handles) + { + result = future->WaitAndDeleteProfilerHubFuture(table_handle.ph_future); + } + } + else + { + if (!tracks_selected || tracks.size() != 1) + return kRocProfVisDmResultInvalidParameter; + profiler_hub_future_handle_t ph_future = future->AddProfilerHubFuture(); + ProfilerHubTableHandler table_handle = { handle, 0, future, ph_future, (rocprofvis_dm_event_operation_t)TrackPropertiesAt(tracks[0])->op, query.first}; + result = profiler_hub_get_table_time_slice( + ph_future, + m_ph_trace, + &table_handle, + tracks[0], + start, + end); + if (result != kProfilerHubStatusSuccess) + break; + result = future->WaitAndDeleteProfilerHubFuture(ph_future); + } + if (result != kProfilerHubStatusSuccess) + break; + } + ShowProgress(100 - future->Progress(), "Table data successfully loaded!", kRPVDbSuccess, future); + return future->SetPromise(kRocProfVisDmResultSuccess); + } + ShowProgress(0, "Failed to load table data!", kRPVDbError, future ); + return future->SetPromise(future->Interrupted() ? kRocProfVisDmResultDbAbort : kRocProfVisDmResultDbAccessFailed); + } + + rocprofvis_dm_result_t ProfilerHub::RemapStringId( + uint64_t id, + rocprofvis_db_string_type_t type, + uint32_t node, + uint64_t& result) { + result = id; + return kRocProfVisDmResultNotSupported; + } + + void ProfilerHub::GetTrackIdentifierIndices( + int column_index, + char** azColName, + rocprofvis_db_track_identifier_index_t& + track_ids_indices) { + + std::string column_name = azColName[column_index]; + + if (column_name == Builder::NODE_ID_SERVICE_NAME) + { + track_ids_indices.nid_index = column_index; + } + else if (column_name == Builder::AGENT_ID_SERVICE_NAME) + { + track_ids_indices.process_index = column_index; + } + else if (column_name == Builder::QUEUE_ID_SERVICE_NAME) + { + track_ids_indices.sub_process_index = column_index; + } + else if (column_name == Builder::STREAM_ID_SERVICE_NAME) + { + track_ids_indices.stream_index = column_index; + } + else if (column_name == Builder::PROCESS_ID_SERVICE_NAME) + { + track_ids_indices.process_index = column_index; + } + else if (column_name == Builder::THREAD_ID_SERVICE_NAME) + { + track_ids_indices.sub_process_index = column_index; + } + else if (column_name == Builder::COUNTER_ID_SERVICE_NAME) + { + track_ids_indices.is_pmc_identifier = true; + track_ids_indices.sub_process_index = column_index; + } + else if (column_name == Builder::COUNTER_NAME_SERVICE_NAME) + { + track_ids_indices.is_pmc_identifier = true; + track_ids_indices.is_rocpd_pmc = true; + track_ids_indices.sub_process_index = column_index; + } + else if (column_name == Builder::PROCESS_ID_PUBLIC_NAME) + { + track_ids_indices.pid_index = column_index; + } + } + + bool ProfilerHub::FindTrack( + rocprofvis_dm_track_category_t category, + uint64_t id_process, + uint64_t id_subprocess, + uint32_t db_instance, + uint32_t& out_track + ) + { + return TrackTracker()->FindTrack(category, id_process, id_subprocess, db_instance, out_track); + } + + ProfilerHubCell::ProfilerHubCell(profiler_hub_string_t name, + profiler_hub_value_type_t type, + ProfilerHubCellValue value) + : name(name) + , type(type) + , value(std::move(value)) + { + } + + ProfilerHubTableRow::ProfilerHubTableRow( profiler_hub_table_handle_t table_handle, size_t num_columns):m_table_handle(table_handle) { + m_cells.resize(num_columns, ProfilerHubCell(nullptr, kPprofilerHubDataTypeUndefined, std::monostate{})); + } + + void ProfilerHubTableRow::SetCell(size_t index, + profiler_hub_string_t name, + profiler_hub_value_type_t type, + ProfilerHubCellValue value) { + if (index < m_cells.size()) { + m_cells[index] = ProfilerHubCell(name, type, std::move(value)); + } + } + + size_t ProfilerHubTableRow::NumCells() const { + return m_cells.size(); + } + + const ProfilerHubCell* ProfilerHubTableRow::CellAt(size_t index) const { + return index < m_cells.size() ? &m_cells[index] : nullptr; + } + + ProfilerHubCell* ProfilerHubTableRow::CellAt(size_t index) { + return index < m_cells.size() ? &m_cells[index] : nullptr; + } + + profiler_hub_table_handle_t ProfilerHubTableRow::TableHandle() { + return m_table_handle; + } + + ProfilerHubTableRow* ProfilerHubTableRowArray::AddRow(profiler_hub_table_handle_t table_handle, size_t num_columns) { + auto row = std::make_unique(table_handle, num_columns); + ProfilerHubTableRow* ptr = row.get(); + std::unique_lock lock(m_mutex); + m_rows.push_back(std::move(row)); + return ptr; + } + + bool ProfilerHubTableRowArray::RowExists(ProfilerHubTableRow* row) { + std::unique_lock lock(m_mutex); + auto it = std::find_if(m_rows.begin(), m_rows.end(), + [row](const std::unique_ptr& p) { + return p.get() == row; + }); + if (it == m_rows.end()) return false; + return true; + } + + bool ProfilerHubTableRowArray::RemoveRow(ProfilerHubTableRow* row) { + std::unique_lock lock(m_mutex); + auto it = std::find_if(m_rows.begin(), m_rows.end(), + [row](const std::unique_ptr& p) { + return p.get() == row; + }); + if (it == m_rows.end()) return false; + m_rows.erase(it); + return true; + } + + size_t ProfilerHubTableRowArray::NumRows() const { + std::shared_lock lock(m_mutex); + return m_rows.size(); + } + + void ProfilerHubTableRowArray::Clear() { + std::unique_lock lock(m_mutex); + m_rows.clear(); + } + +} // namespace DataModel +} // namespace RocProfVis diff --git a/src/model/src/database/rocprofvis_db_profiler_hub.h b/src/model/src/database/rocprofvis_db_profiler_hub.h new file mode 100644 index 000000000..69693c189 --- /dev/null +++ b/src/model/src/database/rocprofvis_db_profiler_hub.h @@ -0,0 +1,282 @@ +// Copyright Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +#ifdef USE_PROFILER_HUB + +#include "rocprofvis_db_profiler_hub_client.h" +#include "rocprofvis_db_systems.h" +#include "profiler_hub_lib_interface.h" +#include "rocprofvis_db_table_processor.h" + +namespace RocProfVis +{ +namespace DataModel +{ +typedef std::unordered_map string_index_map_t; + +using ProfilerHubCellValue = std::variant; + +struct ProfilerHubCell { + profiler_hub_string_t name; + profiler_hub_value_type_t type; + ProfilerHubCellValue value; + + ProfilerHubCell(profiler_hub_string_t name, + profiler_hub_value_type_t type, + ProfilerHubCellValue value = std::monostate{}); +}; + +class ProfilerHubTableRow { +public: + explicit ProfilerHubTableRow( profiler_hub_table_handle_t table_handle, size_t num_columns); + + void SetCell(size_t index, + profiler_hub_string_t name, + profiler_hub_value_type_t type, + ProfilerHubCellValue value); + + size_t NumCells() const; + const ProfilerHubCell* CellAt(size_t index) const; + ProfilerHubCell* CellAt(size_t index); + profiler_hub_table_handle_t TableHandle(); + +private: + std::vector m_cells; + profiler_hub_table_handle_t m_table_handle; +}; + +class ProfilerHubTableRowArray { +public: + ProfilerHubTableRow* AddRow(profiler_hub_table_handle_t table_handle, size_t num_columns); + + bool RemoveRow(ProfilerHubTableRow* row); + + bool RowExists(ProfilerHubTableRow* row); + + size_t NumRows() const; + + void Clear(); + +private: + std::vector> m_rows; + mutable std::shared_mutex m_mutex; +}; + +typedef struct ProfilerHubTableHandler +{ + profiler_hub_table_handle_t handle; + uint32_t chunk; + Future* dm_future; + profiler_hub_future_handle_t ph_future; + rocprofvis_dm_event_operation_t op; + DbInstance* instance; +} ProfilerHubTableHandler; + +class ProfilerHub : public SystemDatabase +{ +public: + // Database constructor + // @param path - full path to database file + ProfilerHub(rocprofvis_db_filename_t path) : + SystemDatabase(path), m_ph_trace(nullptr), + m_table_processor{ TableProcessor(this),TableProcessor(this),TableProcessor(this) } { + } + // ProfilerHub destructor, must be defined as virtual to free resources of derived classes + virtual ~ProfilerHub() { Close(); } + // Method to open sqlite database + // @return status of operation + rocprofvis_dm_result_t Open() override; + // Method to close sqlite database + // @return status of operation + rocprofvis_dm_result_t Close() override; + void InterruptQuery(void* connection) override {}; + +protected: + + rocprofvis_dm_result_t ExecuteQuery( + rocprofvis_dm_charptr_t query, + rocprofvis_dm_charptr_t description, + Future* future) override; + + rocprofvis_dm_result_t ExportTableCSV( + rocprofvis_dm_charptr_t query, + rocprofvis_dm_charptr_t file_path, + Future* future) override; + + rocprofvis_dm_result_t BuildTableQuery( + rocprofvis_dm_table_use_case_enum_t use_case, + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t tracks, + rocprofvis_dm_processor_identifiers_ptr processor, + rocprofvis_dm_charptr_t filter, + rocprofvis_dm_charptr_t group, + rocprofvis_dm_charptr_t group_cols, + rocprofvis_dm_charptr_t sort_column, + rocprofvis_dm_sort_order_t sort_order, + uint64_t max_count, + uint64_t offset, + bool count_only, + rocprofvis_dm_string_t& query) override; + + rocprofvis_dm_result_t BuildEventSearchQuery( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t ops, + rocprofvis_dm_processor_identifiers_ptr processor, + rocprofvis_dm_num_string_table_filters_t num_string_table_filters, + rocprofvis_dm_string_table_filters_t string_table_filters, + bool include_substring, + bool include_category, + bool partial_matching, + rocprofvis_dm_charptr_t sort_column, + rocprofvis_dm_sort_order_t sort_order, + uint64_t max_count, + uint64_t offset, + bool count_only, + rocprofvis_dm_string_t& query) override; + + rocprofvis_dm_result_t SaveTrimmedData(rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_charptr_t new_db_path, + Future* future) override; + + rocprofvis_dm_result_t ReadTraceMetadata( + Future* object) override; + + rocprofvis_dm_result_t ReadFlowTraceInfo( + rocprofvis_dm_event_id_t event_id, + Future* object) override; + + rocprofvis_dm_result_t ReadStackTraceInfo( + rocprofvis_dm_event_id_t event_id, + Future* object) override; + + rocprofvis_dm_result_t ReadExtEventInfo( + rocprofvis_dm_event_id_t event_id, + Future* object) override; + + rocprofvis_dm_result_t ReadTraceSlice( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_hashed_timestamp_tag_t tag, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t tracks, + Future* object) override; + + rocprofvis_dm_result_t ReadTracePMCSlice( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_hashed_timestamp_tag_t tag, + rocprofvis_db_track_selection_t track, + bool left_neighbor, + bool right_neighbor, + Future* object) override; + + + rocprofvis_dm_result_t GetEventTablesAsync( + std::vector>& queries, + Future* parent, + rocprofvis_dm_handle_t handle, + RpvCallback callback) override; + + rocprofvis_dm_result_t RemapStringId(uint64_t id, + rocprofvis_db_string_type_t type, + uint32_t node, + uint64_t& result) override; + + void GetTrackIdentifierIndices( + int column_index, + char** azColName, + rocprofvis_db_track_identifier_index_t& + track_ids_indices) override; + + bool FindTrack( + rocprofvis_dm_track_category_t category, + uint64_t id_process, + uint64_t id_subprocess, + uint32_t db_instance, + uint32_t& out_track) override; + + StringTable& DbFiles() { return m_db_files; } + + rocprofvis_dm_result_t ProcessTrack(rocprofvis_dm_track_params_t& track_params); + + //--------------------------------------Table accessors----------------------------------------------------------------- + std::string TableColumnText(void* func, void* handle, char** azColName, int index) override; + int TableColumnInt(void* func, void* handle, char** azColName, int index) override; + int64_t TableColumnInt64(void* func, void* handle, char** azColName, int index) override; + double TableColumnDouble(void* func, void* handle, char** azColName, int index) override; + +private: + + static rocprofvis_dm_result_t ConvertProfilerHubResult(profiler_hub_result_t ph_result); + static rocprofvis_dm_event_operation_t ConvertProfilerHubEventType(profiler_hub::reader_types::event_type_t event_type); + void IdentificatorNamesUpdate(rocprofvis_dm_track_params_t& track_params); + + profiler_hub_trace_handle_t m_ph_trace; + TableProcessor m_table_processor[kRPVTableDataTypesNum]; + StringTable m_db_files; + string_index_map_t m_string_index_map; // id to index + std::mutex m_mutex; + ProfilerHubTableRowArray m_table_rows_cache; + + inline static const rocprofvis_event_data_category_map_t + s_rocprof_categorized_data = { + { + kRocProfVisDmOperationNoOp, + { + { "id", kRocProfVisEventEssentialDataId }, + { "category", kRocProfVisEventEssentialDataCategory }, + { "name", kRocProfVisEventEssentialDataName }, + { "start", kRocProfVisEventEssentialDataStart }, + { "end", kRocProfVisEventEssentialDataEnd }, + { "duration", kRocProfVisEventEssentialDataDuration }, + { "nid", kRocProfVisEventEssentialDataNode }, + { "pid", kRocProfVisEventEssentialDataProcess }, + { "tid", kRocProfVisEventEssentialDataThread }, + { "queue_name", kRocProfVisEventEssentialDataQueue }, + { "stream_name", kRocProfVisEventEssentialDataStream }, + { "stack_id", kRocProfVisEventEssentialDataInternal }, + { "parent_stack_id", kRocProfVisEventEssentialDataInternal }, + { "corr_id", kRocProfVisEventEssentialDataInternal }, + { "stream_id", kRocProfVisEventEssentialDataInternal }, + { "queue_id", kRocProfVisEventEssentialDataInternal }, + }, + }, + { + kRocProfVisDmOperationDispatch, + { + { "agent_type", kRocProfVisEventEssentialDataAgentType }, + { "agent_type_index", kRocProfVisEventEssentialDataAgentIndex }, + }, + }, + { + kRocProfVisDmOperationMemoryAllocate, + { + { "agent_type", kRocProfVisEventEssentialDataAgentType }, + { "agent_type_index", kRocProfVisEventEssentialDataAgentIndex }, + { "type", kRocProfVisEventEssentialDataName }, + }, + }, + { + kRocProfVisDmOperationMemoryCopy, + { + { "dst_agent_type", kRocProfVisEventEssentialDataAgentType }, + { "dst_agent_type_index", kRocProfVisEventEssentialDataAgentIndex }, + } + } + }; + + friend class ProfilerHubClientMethods; +}; + + +} // namespace DataModel +} // namespace RocProfVis + +#endif \ No newline at end of file diff --git a/src/model/src/database/rocprofvis_db_profiler_hub_client.cpp b/src/model/src/database/rocprofvis_db_profiler_hub_client.cpp new file mode 100644 index 000000000..1acb95e41 --- /dev/null +++ b/src/model/src/database/rocprofvis_db_profiler_hub_client.cpp @@ -0,0 +1,666 @@ +#include "profiler_hub_client_interface.h" +#include "rocprofvis_db_profiler_hub.h" +#include "rocprofvis_db_profiler_hub_client.h" +#include "rocprofvis_db_query_builder.h" + +namespace RocProfVis::DataModel +{ + + + profiler_hub_result_t ProfilerHubClientMethods::AddInstance( + client_trace_handle_t trace, + profiler_hub_instance_id_t id, + profiler_hub_string_t file, + profiler_hub_string_t uuid + ) { + ProfilerHub* db = (ProfilerHub*)trace; + uint32_t file_id = db->DbFiles().ToInt(file); + db->DbInstances().push_back({ DbInstance(file_id, static_cast(db->DbInstances().size())), uuid }); + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddString( + client_trace_handle_t trace, + profiler_hub_string_t string, + uint32_t string_id + + ) { + ProfilerHub* db = (ProfilerHub*)trace; + uint32_t string_index = db->BindObject()->FuncAddString(db->BindObject()->trace_object, string); + db->m_string_index_map[string_id] = string_index; + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddTrack( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_track_id_t track_id, + profiler_hub_string_t track_name, + profiler_hub_track_category_t category, + profiler_hub_optional_int_t node_id, + profiler_hub_string_t node_name, + profiler_hub_optional_int_t process_id, + profiler_hub_string_t process_name, + profiler_hub_optional_int_t thread_id, + profiler_hub_string_t thread_name, + profiler_hub_optional_int_t stream_id, + profiler_hub_string_t stream_name, + profiler_hub_agent_type_t agent_type, + profiler_hub_optional_int_t agent_id, + profiler_hub_string_t agent_name, + profiler_hub_optional_int_t queue_id, + profiler_hub_string_t queue_name, + profiler_hub_optional_int_t counter_id, + profiler_hub_string_t counter_name, + uint32_t records_count, + uint64_t min_timestamp, + uint64_t max_timestamp, + double min_level_or_value, + double max_level_or_value + ) { + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + rocprofvis_dm_track_params_t track_params; + if (records_count == 0) + return kProfilerHubStatusInvalidArgument; + switch (category) + { + + case kPprofilerHubCategoryRegionInstrumented: + track_params.track_indentifiers.category = kRocProfVisDmRegionMainTrack; + break; + case kPprofilerHubCategoryRegionSampled: + track_params.track_indentifiers.category = kRocProfVisDmRegionSampleTrack; + break; + case kPprofilerHubCategoryKernelDispatch: + track_params.track_indentifiers.category = kRocProfVisDmKernelDispatchTrack; + break; + case kPprofilerHubCategoryMemoryAllocate: + track_params.track_indentifiers.category = kRocProfVisDmMemoryAllocationTrack; + break; + case kPprofilerHubCategoryMemoryCopy: + track_params.track_indentifiers.category = kRocProfVisDmMemoryCopyTrack; + break; + case kPprofilerHubCategoryPerformanceCounter: + track_params.track_indentifiers.category = kRocProfVisDmPmcTrack; + break; + case kPprofilerHubCategoryStream: + track_params.track_indentifiers.category = kRocProfVisDmStreamTrack; + break; + case kPprofilerHubCategoryUndefined: + default: + return kProfilerHubStatusUnknownError; + } + track_params.track_indentifiers.track_id = track_id; + track_params.track_indentifiers.process_id = process_id == nullptr ? 0 : *process_id; + track_params.track_indentifiers.source_type = kRPVSystemSourceProfillerHub; + track_params.track_indentifiers.db_instance = db->DbInstancePtrAt(instance); + track_params.track_indentifiers.id[TRACK_ID_NODE] = node_id == nullptr ? 0 : *node_id; + track_params.track_indentifiers.name[TRACK_ID_NODE] = node_name ? node_name : std::string("Node ")+std::to_string(*node_id); + track_params.track_indentifiers.tag[TRACK_ID_NODE] = Builder::NODE_ID_SERVICE_NAME; + track_params.track_indentifiers.is_numeric[TRACK_ID_NODE] = true; + if (category == kPprofilerHubCategoryRegionInstrumented || + category == kPprofilerHubCategoryRegionSampled || + category == kPprofilerHubCategoryStream) + { + if (!process_id) + return kProfilerHubStatusInvalidArgument; + track_params.track_indentifiers.id[TRACK_ID_PID] = *process_id; + track_params.track_indentifiers.name[TRACK_ID_PID] = process_name ? process_name : ""; + track_params.track_indentifiers.tag[TRACK_ID_PID] = Builder::PROCESS_ID_SERVICE_NAME; + track_params.track_indentifiers.is_numeric[TRACK_ID_PID] = true; + if (!thread_id) + return kProfilerHubStatusInvalidArgument; + } + if (category == kPprofilerHubCategoryRegionInstrumented || + category == kPprofilerHubCategoryRegionSampled) + { + track_params.track_indentifiers.id[TRACK_ID_TID] = *thread_id; + track_params.track_indentifiers.name[TRACK_ID_TID] = thread_name ? thread_name : ""; + track_params.track_indentifiers.tag[TRACK_ID_TID] = Builder::THREAD_ID_SERVICE_NAME; + track_params.track_indentifiers.is_numeric[TRACK_ID_TID] = true; + + } + if (category == kPprofilerHubCategoryStream) + { + track_params.track_indentifiers.id[TRACK_ID_STREAM] = *stream_id; + track_params.track_indentifiers.name[TRACK_ID_STREAM] = stream_name ? stream_name : ""; + track_params.track_indentifiers.tag[TRACK_ID_STREAM] = Builder::STREAM_ID_SERVICE_NAME; + track_params.track_indentifiers.is_numeric[TRACK_ID_STREAM] = true; + } + + if (category == kPprofilerHubCategoryKernelDispatch || + category == kPprofilerHubCategoryMemoryAllocate || + category == kPprofilerHubCategoryMemoryCopy || + category == kPprofilerHubCategoryPerformanceCounter) + { + if (!agent_id) + return kProfilerHubStatusInvalidArgument; + track_params.track_indentifiers.id[TRACK_ID_AGENT] = *agent_id; + track_params.track_indentifiers.name[TRACK_ID_AGENT] = agent_name ? agent_name : ""; + track_params.track_indentifiers.tag[TRACK_ID_AGENT] = Builder::AGENT_ID_SERVICE_NAME; + track_params.track_indentifiers.is_numeric[TRACK_ID_AGENT] = true; + + } + if (category == kPprofilerHubCategoryPerformanceCounter) + { + if (!counter_id) + return kProfilerHubStatusInvalidArgument; + track_params.track_indentifiers.id[TRACK_ID_COUNTER] = *counter_id; + track_params.track_indentifiers.name[TRACK_ID_COUNTER] = counter_name ? counter_name : std::string("Counter ") + std::to_string(*counter_id); + track_params.track_indentifiers.tag[TRACK_ID_COUNTER] = Builder::THREAD_ID_SERVICE_NAME; + track_params.track_indentifiers.is_numeric[TRACK_ID_COUNTER] = true; + } + track_params.record_count = records_count; + track_params.min_ts = min_timestamp; + track_params.max_ts = max_timestamp; + track_params.min_value = min_level_or_value; + track_params.max_value = max_level_or_value; + + return db->ProcessTrack(track_params) == kRocProfVisDmResultSuccess ? kProfilerHubStatusSuccess : kProfilerHubStatusUnknownError; + } + + + profiler_hub_result_t ProfilerHubClientMethods::AddTrackHistogramBucket( + client_trace_handle_t trace, + profiler_hub_track_id_t track_id, + uint32_t bucket_number, + uint32_t events_count, + double bucket_value + ) { + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + auto it = std::find_if(db->TrackPropertiesBegin(), db->TrackPropertiesEnd(), + [track_id](std::unique_ptr& params) { + return params.get()->track_indentifiers.track_id == track_id; + }); + if (it != db->TrackPropertiesEnd()) + { + it->get()->histogram[bucket_number] = std::make_pair( events_count, bucket_value ); + db->TraceProperties()->histogram[bucket_number] += events_count; + } + return kProfilerHubStatusNotSupported; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddInfoProperty( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_string_t category, + profiler_hub_string_t name, + profiler_hub_value_type_t type, + profiler_hub_row_id_t row_id, + profiler_hub_value_handle_t value + ) { + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + if (type == kPprofilerHubDataTypeString || type == kPprofilerHubDataTypeBlob) + { + db->CachedTables(instance)->AddTableCell(category, row_id, name, (rocprofvis_db_data_type_t)type, (const char*)value); + } + else + if (type == kPprofilerHubDataTypeInt) + { + db->CachedTables(instance)->AddTableCell(category, row_id, name, (rocprofvis_db_data_type_t)type, std::to_string(*(size_t*)value)); + } + else + if (type == kPprofilerHubDataTypeDouble) + { + db->CachedTables(instance)->AddTableCell(category, row_id, name, (rocprofvis_db_data_type_t)type, std::to_string(*(double*)value)); + } + else + { + return kProfilerHubStatusInvalidArgument; + } + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddEventRecord( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_timeslice_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id, + uint64_t timestamp, + uint64_t duration, + profiler_hub_string_id_t category_id, + profiler_hub_string_id_t symbol_id, + profiler_hub_event_level_t level + ) { + rocprofvis_db_record_data_t record; + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + record.event.id.bitfield.event_op = db->ConvertProfilerHubEventType(operation); + record.event.id.bitfield.event_node = instance; + + record.event.id.bitfield.event_id = event_id; + record.event.timestamp = timestamp; + record.event.duration = duration; + record.event.category = category_id; + record.event.symbol = symbol_id; + record.event.level = level; + + if(db->BindObject()->FuncAddRecord(container, record) != kRocProfVisDmResultSuccess) + return kProfilerHubStatusUnknownError; + + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddPmcRecord( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_timeslice_handle_t container, + uint64_t timestamp, + double value + ) { + rocprofvis_db_record_data_t record; + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + record.pmc.timestamp = timestamp; + record.pmc.value = value; + return kProfilerHubStatusNotSupported; + } + + profiler_hub_table_row_handle_t ProfilerHubClientMethods::AddTableRowContainer( + client_trace_handle_t trace, + profiler_hub_table_handle_t table_handle, + size_t num_columns + ) { + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr || table_handle == nullptr) + return nullptr; + ProfilerHubTableRow* row = db->m_table_rows_cache.AddRow(table_handle, num_columns); + return row; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddTableCell( + client_trace_handle_t trace, + profiler_hub_table_row_handle_t container, + uint32_t column_index, + profiler_hub_string_t column_name, + profiler_hub_value_type_t column_type, + profiler_hub_value_handle_t value + ) + { + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + ProfilerHubTableRow* row = (ProfilerHubTableRow*)container; + if (row == nullptr || !db->m_table_rows_cache.RowExists(row) || row->TableHandle() == nullptr) + return kProfilerHubStatusInvalidArgument; + if (column_type == kPprofilerHubDataTypeInt) + row->SetCell(column_index, column_name, column_type, *(size_t*)value); + else if (column_type == kPprofilerHubDataTypeDouble) + row->SetCell(column_index, column_name, column_type, *(double*)value); + else if (column_type == kPprofilerHubDataTypeString || column_type == kPprofilerHubDataTypeBlob) + row->SetCell(column_index, column_name, column_type, (const char*)value); + // maybe safer to add CommitRow method, if we cannot rely on last row index + if (column_index + 1 == row->NumCells()) + { + ProfilerHubTableHandler* table = (ProfilerHubTableHandler*)row->TableHandle(); + rocprofvis_db_query_callback_parameters callback_params; + std::vector columnNames(row->NumCells()); + for (int i = 0; i < row->NumCells(); i++) + { + columnNames[i] = row->CellAt(i)->name; + } + callback_params.db = db; + callback_params.future = table->dm_future; + callback_params.handle = table->handle; + callback_params.db_instance = table->instance; + callback_params.operation = table->op; + callback_params.track_id = table->chunk; + if (TableProcessor::CallbackRunCompoundQuery(&callback_params, row->NumCells(), row, (char**)columnNames.data()) == 0) + return kProfilerHubStatusSuccess; + } + return kProfilerHubStatusUnknownError; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddEventDataFlowEndPoint( + client_trace_handle_t trace, + profiler_hub_flowtrace_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id, + profiler_hub_track_id_t track_id, + profiler_hub_flow_direction_t direction, + uint64_t timestamp, + uint64_t duration, + profiler_hub_string_id_t category_id, + profiler_hub_string_id_t symbol_id, + profiler_hub_event_level_t level + ) { + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + rocprofvis_db_flow_data_t record; + record.id.bitfield.event_id = event_id; + record.time = timestamp; + record.category_id = category_id; + record.symbol_id = symbol_id; + record.level = level; + record.end_time = timestamp + duration; + record.track_id = track_id; + if (db->BindObject()->FuncAddFlow(container,record) != kRocProfVisDmResultSuccess) return kProfilerHubStatusUnknownError; + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddEventExtendedInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_string_t category, + profiler_hub_string_t name, + profiler_hub_value_type_t type, + profiler_hub_string_t value + ) + { + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + rocprofvis_db_ext_data_t record; + record.category = category; + record.name = name; + record.type = (rocprofvis_db_data_type_t)type; + record.data = value; + record.db_instance = instance; + if (db->BindObject()->FuncAddExtDataRecord(container,record) != kRocProfVisDmResultSuccess) return kProfilerHubStatusUnknownError; + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddEventEssentialInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_track_id_t track_id, + profiler_hub_track_id_t stream_track_id, + profiler_hub_event_level_t level, + profiler_hub_event_level_t stream_level + ) { + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + rocprofvis_db_ext_data_t record; + record.category = "Track"; + record.name = "trackId"; + record.type = kRPVDataTypeInt; + std::string str = std::to_string(track_id); + record.data = str.c_str(); + record.category_enum = kRocProfVisEventEssentialDataTrack; + record.db_instance = instance; + if (db->BindObject()->FuncAddExtDataRecord(container, record) != + kRocProfVisDmResultSuccess) + return kProfilerHubStatusUnknownError; + record.category = "Track"; + record.name = "levelForTrack"; + record.type = kRPVDataTypeInt; + str = std::to_string(level); + record.data = str.c_str(); + record.category_enum = kRocProfVisEventEssentialDataLevel; + record.db_instance = instance; + if (db->BindObject()->FuncAddExtDataRecord(container, record) != + kRocProfVisDmResultSuccess) + return kProfilerHubStatusUnknownError; + if (stream_track_id != -1) + { + record.category = "Track"; + record.name = "streamTrackId"; + record.type = kRPVDataTypeInt; + std::string str = std::to_string(stream_track_id); + record.data = str.c_str(); + record.category_enum = kRocProfVisEventEssentialDataStreamTrack; + record.db_instance = instance; + if (db->BindObject()->FuncAddExtDataRecord(container, record) != + kRocProfVisDmResultSuccess) + return kProfilerHubStatusUnknownError; + record.category = "Track"; + record.name = "levelForStreamTrack"; + record.type = kRPVDataTypeInt; + str = std::to_string(stream_level); + record.data = str.c_str(); + record.category_enum = kRocProfVisEventEssentialDataStreamLevel; + record.db_instance = instance; + if (db->BindObject()->FuncAddExtDataRecord(container, record) != + kRocProfVisDmResultSuccess) + return kProfilerHubStatusUnknownError; + } + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddEventArgumentsInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + uint32_t position, + profiler_hub_string_t name, + profiler_hub_string_t type, + profiler_hub_string_t value + ) { + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + rocprofvis_db_argument_data_t record; + record.position = position; + record.type = type; + record.name = name; + record.value = value; + if (db->BindObject()->FuncAddArgDataRecord(container, record) != kRocProfVisDmResultSuccess) return kProfilerHubStatusUnknownError; + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t ProfilerHubClientMethods::AddEventCallStackFrame( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_call_stack_handle_t container, + profiler_hub_string_t function, + profiler_hub_string_t file, + profiler_hub_string_t line, + profiler_hub_string_t address, + uint32_t depth + ) { + ProfilerHub* db = (ProfilerHub*)trace; + if (db == nullptr) + return kProfilerHubStatusInvalidArgument; + rocprofvis_db_stack_data_t record; + record.symbol = function; + record.line = line; + record.depth = depth; + record.args = file; + if (db->BindObject()->FuncAddStackFrame(container, record) != + kRocProfVisDmResultSuccess) + return kProfilerHubStatusUnknownError; + return kProfilerHubStatusSuccess; + } + +} + +//---------------------------------------------------------------------------------------------------------------------------------------- + +profiler_hub_result_t profiler_hub::client::interface::AddInstance( + client_trace_handle_t trace, + profiler_hub_instance_id_t id, + profiler_hub_string_t file, + profiler_hub_string_t uuid +) { + return RocProfVis::DataModel::ProfilerHubClientMethods::AddInstance(trace, id, file, uuid); +} + +profiler_hub_result_t profiler_hub::client::interface::AddString( + client_trace_handle_t trace, + profiler_hub_string_t string, + uint32_t string_id + +) { + return RocProfVis::DataModel::ProfilerHubClientMethods::AddString(trace, string, string_id); +} + +profiler_hub_result_t profiler_hub::client::interface::AddTrack( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_track_id_t track_id, + profiler_hub_string_t track_name, + profiler_hub_track_category_t category, + profiler_hub_optional_int_t node_id, + profiler_hub_string_t node_name, + profiler_hub_optional_int_t process_id, + profiler_hub_string_t process_name, + profiler_hub_optional_int_t thread_id, + profiler_hub_string_t thread_name, + profiler_hub_optional_int_t stream_id, + profiler_hub_string_t stream_name, + profiler_hub_agent_type_t agent_type, + profiler_hub_optional_int_t agent_id, + profiler_hub_string_t agent_name, + profiler_hub_optional_int_t queue_id, + profiler_hub_string_t queue_name, + profiler_hub_optional_int_t counter_id, + profiler_hub_string_t counter_name, + uint32_t records_count, + uint64_t min_timestamp, + uint64_t max_timestamp, + double min_level_or_value, + double max_level_or_value +){ + return RocProfVis::DataModel::ProfilerHubClientMethods::AddTrack( + trace, instance, track_id, track_name, category, node_id, node_name, process_id, process_name, + thread_id, thread_name, stream_id, stream_name, agent_type, agent_id, agent_name, queue_id, queue_name, + counter_id, counter_name, records_count, min_timestamp, max_timestamp, min_level_or_value, max_level_or_value); +} + +profiler_hub_result_t profiler_hub::client::interface::AddTrackHistogramBucket( + client_trace_handle_t trace, + profiler_hub_track_id_t track_id, + uint32_t bucket_number, + uint32_t events_count, + double bucket_value +) { + return RocProfVis::DataModel::ProfilerHubClientMethods::AddTrackHistogramBucket(trace, track_id, bucket_number, events_count, bucket_value); +} + +profiler_hub_result_t profiler_hub::client::interface::AddInfoProperty( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_string_t category, + profiler_hub_string_t name, + profiler_hub_value_type_t type, + profiler_hub_row_id_t row_id, + profiler_hub_value_handle_t value +) { + return RocProfVis::DataModel::ProfilerHubClientMethods::AddInfoProperty(trace, instance, category, name, type, row_id, value); +} + +profiler_hub_result_t profiler_hub::client::interface::AddEventRecord( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_timeslice_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id, + uint64_t timestamp, + uint64_t duration, + profiler_hub_string_id_t category_id, + profiler_hub_string_id_t symbol_id, + profiler_hub_event_level_t level +) +{ + return RocProfVis::DataModel::ProfilerHubClientMethods::AddEventRecord(trace, instance, container, operation, event_id, timestamp, duration, category_id, symbol_id, level); +} + +profiler_hub_result_t profiler_hub::client::interface::AddPmcRecord( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_timeslice_handle_t container, + uint64_t timestamp, + double value +) +{ + return RocProfVis::DataModel::ProfilerHubClientMethods::AddPmcRecord(trace, instance, container, timestamp, value); +} + +profiler_hub_table_row_handle_t profiler_hub::client::interface::AddTableRowContainer( + client_trace_handle_t trace, + profiler_hub_table_handle_t container, + size_t num_columns +) { + return RocProfVis::DataModel::ProfilerHubClientMethods::AddTableRowContainer(trace, container, num_columns); +} + +profiler_hub_result_t profiler_hub::client::interface::AddTableCell( + client_trace_handle_t trace, + profiler_hub_table_row_handle_t container, + uint32_t column_index, + profiler_hub_string_t column_name, + profiler_hub_value_type_t column_type, + profiler_hub_value_handle_t value +) +{ + return RocProfVis::DataModel::ProfilerHubClientMethods::AddTableCell(trace, container, column_index, column_name, column_type, value); +} + + +profiler_hub_result_t profiler_hub::client::interface::AddEventDataFlowEndPoint( + client_trace_handle_t trace, + profiler_hub_flowtrace_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id, + profiler_hub_track_id_t track_id, + profiler_hub_flow_direction_t direction, + uint64_t timestamp, + uint64_t duration, + profiler_hub_string_id_t category_id, + profiler_hub_string_id_t symbol_id, + profiler_hub_event_level_t level +) { + return RocProfVis::DataModel::ProfilerHubClientMethods::AddEventDataFlowEndPoint(trace, container, operation, event_id,track_id, direction, timestamp, duration, category_id, symbol_id, level); +} + +profiler_hub_result_t profiler_hub::client::interface::AddEventExtendedInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_string_t category, + profiler_hub_string_t name, + profiler_hub_value_type_t type, + profiler_hub_string_t value +) { + return RocProfVis::DataModel::ProfilerHubClientMethods::AddEventExtendedInfo(trace, instance, container, category, name, type, value); +} + +profiler_hub_result_t profiler_hub::client::interface::AddEventEssentialInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_track_id_t track_id, + profiler_hub_track_id_t stream_track_id, + profiler_hub_event_level_t level, + profiler_hub_event_level_t stream_level +) { + return RocProfVis::DataModel::ProfilerHubClientMethods::AddEventEssentialInfo(trace, instance, container, track_id, stream_track_id, level, stream_level); +} + +profiler_hub_result_t profiler_hub::client::interface::AddEventArgumentsInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + uint32_t position, + profiler_hub_string_t name, + profiler_hub_string_t type, + profiler_hub_string_t value +) { + return RocProfVis::DataModel::ProfilerHubClientMethods::AddEventArgumentsInfo(trace, instance, container, position, name, type, value); +} + +profiler_hub_result_t profiler_hub::client::interface::AddEventCallStackFrame( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_call_stack_handle_t container, + profiler_hub_string_t function, + profiler_hub_string_t file, + profiler_hub_string_t line, + profiler_hub_string_t address, + uint32_t depth +) { + return RocProfVis::DataModel::ProfilerHubClientMethods::AddEventCallStackFrame(trace, instance, container, function, file, line, address, depth); +} \ No newline at end of file diff --git a/src/model/src/database/rocprofvis_db_profiler_hub_client.h b/src/model/src/database/rocprofvis_db_profiler_hub_client.h new file mode 100644 index 000000000..291f8946a --- /dev/null +++ b/src/model/src/database/rocprofvis_db_profiler_hub_client.h @@ -0,0 +1,287 @@ +#pragma once + +#ifdef USE_PROFILER_HUB +#include "profiler_hub_interface_types.h" + +namespace RocProfVis +{ +namespace DataModel +{ + +class ProfilerHubClientMethods +{ + public: + // report a single trace instance. Usually based on GUID (for rocpd 3.x and later), otherwise just synthesized single instance. + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // id - instance index + // file - which file the instance comes from, in case of multi-file trace + // uuid - UUID of the instance, if applicable + static + profiler_hub_result_t AddInstance( + client_trace_handle_t trace, + profiler_hub_instance_id_t id, + profiler_hub_string_t file, + profiler_hub_string_t uuid + ); + + // helps to collect strings table for a trace. It's better to replace any string passed to data-model in future requests with integer id. To do so profiler-hub must have keep remapping look-up tables. + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // string - a string from any column + // string_id - string index, expected to start with 0 when first string submitted, but can be re-purposed. The idea is data-model will match any string id from future calls to the strin in the table. + static + profiler_hub_result_t AddString( + client_trace_handle_t trace, + profiler_hub_string_t string, + uint32_t string_id + ); + + // This will convert profiler-hub track representation to data-model + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // instance - what trace instance the track belong to + // category - track category, as enumerated in profiler_hub_track_category_t + // track_id - expected to be sequential number, but can be re-mapped if needed + // track_name - optional track name + // node_id - full node id from database, 0 if not applicable + // node_name - node name if applicable + // process_id - full process id, not table index + // process_name - process name, if applicable + // thread_id - full thread id, not table index + // thread_name - thread name, if applicable + // stream_id - stream id + // thread_name - stream name, if applicable + // agent_type - agent type as defined in profiler_hub_agent_type_t + // agent_id - agent typed id + // agent_name - agent name, if applicable + // queue_id - queue id + // queue_name - queue name, if applicable + // counter_id - counter id (for rocpd 3.x it will be pmc info id, for original rocpd schema - synthesized, for perfetto - counter track id) + // counter_name - counter name, required, if counter track + // records_count - how many records the track have + // min_timestamp - minimum timestamp + // max_timestamp - maximum timestamp + // min_level_or_value - minimum level for event track - expected 0, minimum value for counter track + // max_level_or_value - maximum level for event track, maximum value for counter track + static + profiler_hub_result_t AddTrack( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_track_id_t track_id, + profiler_hub_string_t track_name, + profiler_hub_track_category_t category, + profiler_hub_optional_int_t node_id, + profiler_hub_string_t node_name, + profiler_hub_optional_int_t process_id, + profiler_hub_string_t process_name, + profiler_hub_optional_int_t thread_id, + profiler_hub_string_t thread_name, + profiler_hub_optional_int_t stream_id, + profiler_hub_string_t stream_name, + profiler_hub_agent_type_t agent_type, + profiler_hub_optional_int_t agent_id, + profiler_hub_string_t agent_name, + profiler_hub_optional_int_t queue_id, + profiler_hub_string_t queue_name, + profiler_hub_optional_int_t counter_id, + profiler_hub_string_t counter_name, + uint32_t records_count, + uint64_t min_timestamp, + uint64_t max_timestamp, + double min_level_or_value, + double max_level_or_value + ); + + // Reports info property to a caller + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // category - can be table name from where extended data has been taken, or synthesized + // name - can be column name from where extended data has been taken, or synthesized + // type - SQL type, as described in profiler_hub_value_type_t + // instance - what trace instance the track belongs to + // value - value as a void pointer to a value , will be cast based on type + static + profiler_hub_result_t AddInfoProperty( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_string_t category, + profiler_hub_string_t name, + profiler_hub_value_type_t type, + profiler_hub_row_id_t row_id, + profiler_hub_value_handle_t value + ); + + // Adds histogram bucket to track histogram + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // track_id - track id, the combination of trace+track_id can be replaced with track_handler, but then AddTrack has to return the handler and profiler-hub must keep it for reference + // bucket_number - bucket number, no need to send zero count buckets, so the value is not sequential + // events_count - number of events in the bucket, skip zero event buckets + // bucket_value - average value for counter sample buckets, for events can be max level (tbd) + static + profiler_hub_result_t AddTrackHistogramBucket( + client_trace_handle_t trace, + profiler_hub_track_id_t track_id, + uint32_t bucket_number, + uint32_t events_count, + double bucket_value + ); + + // Add Event record to time slice container + // container - time slice container handle + // operation - event operation, as described in profiler_hub_event_operation_t + // event_id - event id relative to operation + // timestamp - start time of the event + // duration - event duration + // category_id - index of category in string table, be ready to remap database table string index into data-model string table + // symbol_id - index of symbol in string table, be ready to remap database table string index into data-model string table + // level - event level for event stacking. The same level should be used to generate call stack trace. It seems to be most reliable and universal way, unlike parent_id/parent_stack_id combination + static + profiler_hub_result_t AddEventRecord( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_timeslice_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id, + uint64_t timestamp, + uint64_t duration, + profiler_hub_string_id_t category_id, + profiler_hub_string_id_t symbol_id, + profiler_hub_event_level_t level + ); + + // Add PMC record to time slice container + // container - time slice container handle + // timestamp - start time of the counter sample + // value - counter sample value + static + profiler_hub_result_t AddPmcRecord( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_timeslice_handle_t container, + uint64_t timestamp, + double value + ); + + // Add an empty row to table processor container + // container - table processor container handle + static + profiler_hub_table_row_handle_t AddTableRowContainer( + client_trace_handle_t trace, + profiler_hub_table_handle_t container, + size_t num_columns + ); + + // Add table cell value + // container - table row container handle + // column_index - column index + // column_name - column name + // column_type - column value type + // value - value as string + static + profiler_hub_result_t AddTableCell( + client_trace_handle_t trace, + profiler_hub_table_row_handle_t container, + uint32_t column_index, + profiler_hub_string_t column_name, + profiler_hub_value_type_t column_type, + profiler_hub_value_handle_t value + ); + + + // Add flow trace endpoint + // container - flow trace container handle + // operation - event operation, as described in profiler_hub_event_operation_t + // event_id - event id relative to operation + // direction - incoming/outgoing + // timestamp - endpoint timestamp + // duration - endpoint duration + // category_id - endpoint category id + // symbol_id - endpoint symbol id + // level - endpoint level + static + profiler_hub_result_t AddEventDataFlowEndPoint( + client_trace_handle_t trace, + profiler_hub_flowtrace_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id, + profiler_hub_track_id_t track_id, + profiler_hub_flow_direction_t direction, + uint64_t timestamp, + uint64_t duration, + profiler_hub_string_id_t category_id, + profiler_hub_string_id_t symbol_id, + profiler_hub_event_level_t level + ); + + + // Add extended data record + // container - extended data container + // category - data category. Can be table name + // name - can be column name + // type - SQL type, as described in profiler_hub_value_type_t + // value - pointer to a value of type + static + profiler_hub_result_t AddEventExtendedInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_string_t category, + profiler_hub_string_t name, + profiler_hub_value_type_t type, + profiler_hub_string_t value + ); + + // Add essential data to extended data table + // container - extended data container + // track_id - event track id + // stream_track_id - event stream track id, set to -1 if event does not belong to any stream + // level - event level + // stream_level - event level on stream track, set to -1 if event doesn't belong to any stream + static + profiler_hub_result_t AddEventEssentialInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_track_id_t track_id, + profiler_hub_track_id_t stream_track_id, + profiler_hub_event_level_t level, + profiler_hub_event_level_t stream_level + ); + + // Add event arguments info + // container - extended data container + // position - argument position + // name - argument name + // type - argument type + static + profiler_hub_result_t AddEventArgumentsInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + uint32_t position, + profiler_hub_string_t name, + profiler_hub_string_t type, + profiler_hub_string_t value + ); + + + // Add call stack frame to container + // container - call stack container + // symbol - stack frame symbol + // file - stack frame file + // line - stack frame code line + // depth - stack frame depth + static + profiler_hub_result_t AddEventCallStackFrame( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_call_stack_handle_t container, + profiler_hub_string_t function, + profiler_hub_string_t file, + profiler_hub_string_t line, + profiler_hub_string_t address, + uint32_t depth + ); + +}; + +} +} +#endif \ No newline at end of file diff --git a/src/model/src/database/rocprofvis_db_query_builder.cpp b/src/model/src/database/rocprofvis_db_query_builder.cpp index d5ba40ca7..0e5ffaa19 100644 --- a/src/model/src/database/rocprofvis_db_query_builder.cpp +++ b/src/model/src/database/rocprofvis_db_query_builder.cpp @@ -349,5 +349,64 @@ namespace DataModel return 0; } + std::unordered_map RocProfVis::DataModel::Builder::table_view_schema = { + {OPERATION_SERVICE_NAME, {OPERATION_SERVICE_NAME, ColumnType::Byte, SCHEMA_INDEX_OPERATION}}, + {DB_ID_PUBLIC_NAME, {DB_ID_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_EVENT_DB_ID}}, + {ID_PUBLIC_NAME, {ID_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_EVENT_ID}}, + {CATEGORY_REFERENCE, {CATEGORY_PUBLIC_NAME, ColumnType::Word, SCHEMA_INDEX_CATEGORY}}, + {CATEGORY_REFERENCE_RPD, {CATEGORY_PUBLIC_NAME, ColumnType::Word, SCHEMA_INDEX_CATEGORY_RPD}}, + {CATEGORY_REFERENCE_PERFETTO, {CATEGORY_PUBLIC_NAME, ColumnType::Word, SCHEMA_INDEX_CATEGORY_PERFETTO}}, + {EVENT_NAME_REFERENCE, {NAME_PUBLIC_NAME, ColumnType::Dword, SCHEMA_INDEX_EVENT_NAME}}, + {SYMBOL_NAME_REFERENCE, {NAME_PUBLIC_NAME, ColumnType::Dword, SCHEMA_INDEX_EVENT_SYMBOL}}, + {EVENT_NAME_REFERENCE_RPD, {NAME_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_EVENT_NAME_RPD}}, + {EVENT_NAME_REFERENCE_PERFETTO, {NAME_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_EVENT_NAME_PERFETTO}}, + {EVENT_ARGS_RPD, {ARGS_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_EVENT_ARGS_RPD}}, + {M_TYPE_REFERENCE, {NAME_PUBLIC_NAME, ColumnType::Byte, SCHEMA_INDEX_MEM_TYPE}}, + {STREAM_NAME_REFERENCE, {STREAM_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_STREAM_NAME}}, + {QUEUE_NAME_REFERENCE, {QUEUE_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_QUEUE_NAME}}, + {NODE_ID_SERVICE_NAME, {NODE_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_NODE_ID}}, + {PROCESS_ID_PUBLIC_NAME, {PROCESS_ID_PUBLIC_NAME, ColumnType::Dword,SCHEMA_INDEX_PROCESS_ID}}, + {THREAD_ID_PUBLIC_NAME, {THREAD_ID_PUBLIC_NAME, ColumnType::Dword,SCHEMA_INDEX_THREAD_ID}}, + {AGENT_ABS_INDEX_REFERENCE, {AGENT_ABS_INDEX_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_ABS_INDEX}}, + {AGENT_TYPE_REFERENCE, {AGENT_TYPE_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_TYPE}}, + {AGENT_TYPE_INDEX_REFERENCE, {AGENT_TYPE_INDEX_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_TYPE_INDEX}}, + {AGENT_NAME_REFERENCE, {AGENT_NAME_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_NAME}}, + {START_SERVICE_NAME, {START_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_START}}, + {END_SERVICE_NAME, {END_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_END}}, + {DURATION_PUBLIC_NAME, {DURATION_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_DURATION}}, + {GRID_SIZEX_PUBLIC_NAME, {GRID_SIZEX_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_GRID_SIZEX}}, + {GRID_SIZEY_PUBLIC_NAME, {GRID_SIZEY_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_GRID_SIZEY}}, + {GRID_SIZEZ_PUBLIC_NAME, {GRID_SIZEZ_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_GRID_SIZEZ}}, + {WORKGROUP_SIZEX_PUBLIC_NAME, {WORKGROUP_SIZEX_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_WORKGROUP_SIZEX}}, + {WORKGROUP_SIZEY_PUBLIC_NAME, {WORKGROUP_SIZEY_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_WORKGROUP_SIZEY}}, + {WORKGROUP_SIZEZ_PUBLIC_NAME, {WORKGROUP_SIZEZ_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_WORKGROUP_SIZEZ}}, + {LDS_SIZE_PUBLIC_NAME, {LDS_SIZE_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_LDS_SIZE}}, + {SCRATCH_SIZE_PUBLIC_NAME, {SCRATCH_SIZE_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_SCRATCH_SIZE}}, + {STATIC_LDS_SIZE_PUBLIC_NAME, {STATIC_LDS_SIZE_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_STATIC_LDS_SIZE}}, + {STATIC_SCRATCH_SIZE_PUBLIC_NAME, {STATIC_SCRATCH_SIZE_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_STATIC_SCRATCH_SIZE}}, + {SIZE_PUBLIC_NAME, {SIZE_PUBLIC_NAME, ColumnType::Dword,SCHEMA_INDEX_SIZE}}, + {ADDRESS_PUBLIC_NAME, {ADDRESS_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_ADDRESS}}, + {LEVEL_REFERENCE, {LEVEL_REFERENCE, ColumnType::Byte,SCHEMA_INDEX_LEVEL}}, + {AGENT_SRC_ABS_INDEX_REFERENCE, {AGENT_SRC_ABS_INDEX_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_SRC_ABS_INDEX}}, + {AGENT_SRC_TYPE_REFERENCE, {AGENT_SRC_TYPE_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_SRC_TYPE}}, + {AGENT_SRC_TYPE_INDEX_REFERENCE, {AGENT_SRC_TYPE_INDEX_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_SRC_TYPE_INDEX}}, + {AGENT_SRC_NAME_REFERENCE, {AGENT_SRC_NAME_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_SRC_NAME}}, + {SRC_ADDRESS_PUBLIC_NAME, {SRC_ADDRESS_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_SRC_ADDRESS}}, + {COUNTER_ID_SERVICE_NAME,{COUNTER_ID_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_COUNTER_ID}}, + {COUNTER_NAME_REFERENCE_RPD,{COUNTER_ID_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_COUNTER_ID_RPD}}, + {COUNTER_VALUE_SERVICE_NAME,{COUNTER_VALUE_PUBLIC_NAME, ColumnType::Double,SCHEMA_INDEX_COUNTER_VALUE}}, + {TRACK_ID_PUBLIC_NAME,{TRACK_ID_PUBLIC_NAME, TRACK_ID_TYPE,SCHEMA_INDEX_TRACK_ID}}, + {STREAM_TRACK_ID_PUBLIC_NAME,{STREAM_TRACK_ID_PUBLIC_NAME, TRACK_ID_TYPE,SCHEMA_INDEX_STREAM_TRACK_ID}}, + }; + + + std::vector RocProfVis::DataModel::Builder::mem_alloc_types = { + "ALLOC", "FREE", "REALLOC", "RECLAIM" + }; + + std::vector RocProfVis::DataModel::Builder::mem_alloc_levels = { + "REAL", "VIRTUAL", "SCRATCH" + }; + } // namespace DataModel } // namespace RocProfVis \ No newline at end of file diff --git a/src/model/src/database/rocprofvis_db_query_builder.h b/src/model/src/database/rocprofvis_db_query_builder.h index 11c077cbe..624871fce 100644 --- a/src/model/src/database/rocprofvis_db_query_builder.h +++ b/src/model/src/database/rocprofvis_db_query_builder.h @@ -300,13 +300,9 @@ class Builder static constexpr const char* NOT_APLICABLE = "N/A"; - inline static std::vector mem_alloc_types = { - "ALLOC", "FREE", "REALLOC", "RECLAIM" - }; + static std::vector mem_alloc_types; - inline static std::vector mem_alloc_levels = { - "REAL", "VIRTUAL", "SCRATCH" - }; + static std::vector mem_alloc_levels; typedef struct ColumnData { @@ -368,57 +364,7 @@ class Builder SCHEMA_INDEX_STREAM_TRACK_ID, } table_view_schema_index_t; - inline static std::unordered_map table_view_schema = { - {OPERATION_SERVICE_NAME, {OPERATION_SERVICE_NAME, ColumnType::Byte, SCHEMA_INDEX_OPERATION}}, - {DB_ID_PUBLIC_NAME, {DB_ID_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_EVENT_DB_ID}}, - {ID_PUBLIC_NAME, {ID_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_EVENT_ID}}, - {CATEGORY_REFERENCE, {CATEGORY_PUBLIC_NAME, ColumnType::Word, SCHEMA_INDEX_CATEGORY}}, - {CATEGORY_REFERENCE_RPD, {CATEGORY_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_CATEGORY_RPD}}, - {CATEGORY_REFERENCE_PERFETTO, {CATEGORY_PUBLIC_NAME, ColumnType::Word, SCHEMA_INDEX_CATEGORY_PERFETTO}}, - {EVENT_NAME_REFERENCE, {NAME_PUBLIC_NAME, ColumnType::Dword, SCHEMA_INDEX_EVENT_NAME}}, - {SYMBOL_NAME_REFERENCE, {NAME_PUBLIC_NAME, ColumnType::Dword, SCHEMA_INDEX_EVENT_SYMBOL}}, - {EVENT_NAME_REFERENCE_RPD, {NAME_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_EVENT_NAME_RPD}}, - {EVENT_NAME_REFERENCE_PERFETTO, {NAME_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_EVENT_NAME_PERFETTO}}, - {EVENT_ARGS_RPD, {ARGS_PUBLIC_NAME, ColumnType::Qword, SCHEMA_INDEX_EVENT_ARGS_RPD}}, - {M_TYPE_REFERENCE, {NAME_PUBLIC_NAME, ColumnType::Byte, SCHEMA_INDEX_MEM_TYPE}}, - {STREAM_NAME_REFERENCE, {STREAM_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_STREAM_NAME}}, - {QUEUE_NAME_REFERENCE, {QUEUE_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_QUEUE_NAME}}, - {NODE_ID_SERVICE_NAME, {NODE_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_NODE_ID}}, - {PROCESS_ID_PUBLIC_NAME, {PROCESS_ID_PUBLIC_NAME, ColumnType::Dword,SCHEMA_INDEX_PROCESS_ID}}, - {THREAD_ID_PUBLIC_NAME, {THREAD_ID_PUBLIC_NAME, ColumnType::Dword,SCHEMA_INDEX_THREAD_ID}}, - {AGENT_ABS_INDEX_REFERENCE, {AGENT_ABS_INDEX_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_ABS_INDEX}}, - {AGENT_TYPE_REFERENCE, {AGENT_TYPE_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_TYPE}}, - {AGENT_TYPE_INDEX_REFERENCE, {AGENT_TYPE_INDEX_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_TYPE_INDEX}}, - {AGENT_NAME_REFERENCE, {AGENT_NAME_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_NAME}}, - {START_SERVICE_NAME, {START_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_START}}, - {END_SERVICE_NAME, {END_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_END}}, - {DURATION_PUBLIC_NAME, {DURATION_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_DURATION}}, - {GRID_SIZEX_PUBLIC_NAME, {GRID_SIZEX_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_GRID_SIZEX}}, - {GRID_SIZEY_PUBLIC_NAME, {GRID_SIZEY_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_GRID_SIZEY}}, - {GRID_SIZEZ_PUBLIC_NAME, {GRID_SIZEZ_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_GRID_SIZEZ}}, - {WORKGROUP_SIZEX_PUBLIC_NAME, {WORKGROUP_SIZEX_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_WORKGROUP_SIZEX}}, - {WORKGROUP_SIZEY_PUBLIC_NAME, {WORKGROUP_SIZEY_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_WORKGROUP_SIZEY}}, - {WORKGROUP_SIZEZ_PUBLIC_NAME, {WORKGROUP_SIZEZ_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_WORKGROUP_SIZEZ}}, - {LDS_SIZE_PUBLIC_NAME, {LDS_SIZE_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_LDS_SIZE}}, - {SCRATCH_SIZE_PUBLIC_NAME, {SCRATCH_SIZE_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_SCRATCH_SIZE}}, - {STATIC_LDS_SIZE_PUBLIC_NAME, {STATIC_LDS_SIZE_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_STATIC_LDS_SIZE}}, - {STATIC_SCRATCH_SIZE_PUBLIC_NAME, {STATIC_SCRATCH_SIZE_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_STATIC_SCRATCH_SIZE}}, - {SIZE_PUBLIC_NAME, {SIZE_PUBLIC_NAME, ColumnType::Dword,SCHEMA_INDEX_SIZE}}, - {ADDRESS_PUBLIC_NAME, {ADDRESS_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_ADDRESS}}, - {LEVEL_REFERENCE, {LEVEL_REFERENCE, ColumnType::Byte,SCHEMA_INDEX_LEVEL}}, - {AGENT_SRC_ABS_INDEX_REFERENCE, {AGENT_SRC_ABS_INDEX_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_SRC_ABS_INDEX}}, - {AGENT_SRC_TYPE_REFERENCE, {AGENT_SRC_TYPE_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_SRC_TYPE}}, - {AGENT_SRC_TYPE_INDEX_REFERENCE, {AGENT_SRC_TYPE_INDEX_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_SRC_TYPE_INDEX}}, - {AGENT_SRC_NAME_REFERENCE, {AGENT_SRC_NAME_PUBLIC_NAME, ColumnType::Byte,SCHEMA_INDEX_AGENT_SRC_NAME}}, - {SRC_ADDRESS_PUBLIC_NAME, {SRC_ADDRESS_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_SRC_ADDRESS}}, - {COUNTER_ID_SERVICE_NAME,{COUNTER_ID_PUBLIC_NAME, ColumnType::Qword,SCHEMA_INDEX_COUNTER_ID}}, - {COUNTER_NAME_REFERENCE_RPD,{COUNTER_ID_PUBLIC_NAME, ColumnType::Word,SCHEMA_INDEX_COUNTER_ID_RPD}}, - {COUNTER_VALUE_SERVICE_NAME,{COUNTER_VALUE_PUBLIC_NAME, ColumnType::Double,SCHEMA_INDEX_COUNTER_VALUE}}, - {TRACK_ID_PUBLIC_NAME,{TRACK_ID_PUBLIC_NAME, TRACK_ID_TYPE,SCHEMA_INDEX_TRACK_ID}}, - {STREAM_TRACK_ID_PUBLIC_NAME,{STREAM_TRACK_ID_PUBLIC_NAME, TRACK_ID_TYPE,SCHEMA_INDEX_STREAM_TRACK_ID}}, - }; - - + static std::unordered_map table_view_schema; static std::optional FindColumnNameByIndex(const std::unordered_map& m, const uint8_t& index) { for (const auto& [key, val] : m) { diff --git a/src/model/src/database/rocprofvis_db_query_manager.cpp b/src/model/src/database/rocprofvis_db_query_manager.cpp index 01dcf0105..fd6d822f0 100644 --- a/src/model/src/database/rocprofvis_db_query_manager.cpp +++ b/src/model/src/database/rocprofvis_db_query_manager.cpp @@ -39,7 +39,7 @@ int QueryManager::CallbackGetValue(void* data, int argc, sqlite3_stmt* stmt, cha ROCPROFVIS_ASSERT_MSG_RETURN(argc==1, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackGetValue; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; QueryManager* db = (QueryManager*) callback_params->db; std::string * string_ptr = (rocprofvis_dm_string_t*)callback_params->handle; ROCPROFVIS_ASSERT_MSG_RETURN(string_ptr, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); @@ -52,8 +52,8 @@ int QueryManager::CallbackMakeHistogramPerTrack(void* data, int argc, sqlite3_st ROCPROFVIS_ASSERT_MSG_RETURN(argc == 4, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void *func = (void*)&CallbackMakeHistogramPerTrack; - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) data; QueryManager* db = (QueryManager*) callback_params->db; if(callback_params->future->Interrupted()) return SQLITE_ABORT; uint32_t index = db->Sqlite3ColumnInt(func, stmt, azColName, 3); @@ -67,7 +67,7 @@ int QueryManager::CallbackMakeHistogramPerTrack(void* data, int argc, sqlite3_st int QueryManager::CallbackRunQuery(void *data, int argc, sqlite3_stmt* stmt, char **azColName){ ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; QueryManager* db = (QueryManager*)callback_params->db; void* func = (void*)&CallbackRunQuery; if (callback_params->future->Interrupted()) return 1; @@ -216,9 +216,6 @@ rocprofvis_dm_event_operation_t QueryManager::GetTableQueryOperation(std::string return kRocProfVisDmOperationNoOp; } - - - bool QueryManager::IsEmptyRange(uint32_t track, uint64_t start, uint64_t end) { if (TABLE_QUERY_UNPACK_OP_TYPE(track) != 0) return false; @@ -269,6 +266,54 @@ QueryManager::BuildCompoundQuery( { query = ""; + BuildTableSqlSubQuery( + start, + end, + tracks, + slice_query_map_array, + where, + query); + if(query.empty()) + { + return kRocProfVisDmResultSuccess; + } + bool sample_query = false; + if(TABLE_QUERY_UNPACK_OP_TYPE(tracks[0]) == 0) + { + sample_query = + TrackPropertiesAt(tracks[0])->track_indentifiers.category == + kRocProfVisDmPmcTrack; + } + else + { + sample_query = + (rocprofvis_dm_event_operation_t) TABLE_QUERY_UNPACK_OP_TYPE( + tracks[0]) == kRocProfVisDmOperationNoOp; + } + return TableProcessor::BuildTableSemanticSubQuery( + use_case, + filter, + group, + group_cols, + sort_column, + sort_order, + max_count, + offset, + count_only, + sample_query, + query); + +} + +rocprofvis_dm_result_t +QueryManager::BuildTableSqlSubQuery( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_db_track_selection_t tracks, + std::vector& slice_query_map_array, + rocprofvis_dm_charptr_t where, + rocprofvis_dm_string_t& query) +{ size_t thread_count = std::thread::hardware_concurrency(); bool event_table = false; for(int i = 0; i < slice_query_map_array.size(); i++) @@ -340,119 +385,7 @@ QueryManager::BuildCompoundQuery( } } } - if(query.empty()) - { - return kRocProfVisDmResultSuccess; - } - query += "-- CMD: TYPE "; - switch(use_case) - { - case kRPVDMTableUseCaseEventTrackTable: - { - query += std::to_string(kRPVTableDataTypeEvent); - break; - } - case kRPVDMTableUseCaseSampleTrackTable: - { - query += std::to_string(kRPVTableDataTypeSample); - break; - } - case kRPVDMTableUseCaseEventSearch: - { - query += std::to_string(kRPVTableDataTypeSearch); - break; - } - default: - { - return kRocProfVisDmResultInvalidParameter; - break; - } - } - query += "\n"; - - if(group && strlen(group)) - { - query += "-- CMD: GROUP "; - if(group_cols && strlen(group_cols)) - { - if(!FilterExpression::StartsWithSubstring(group, group_cols)) - { - query += group_cols; - query += ", "; - } - query += group; - } - else - { - query += group; - bool sample_query = false; - if(TABLE_QUERY_UNPACK_OP_TYPE(tracks[0]) == 0) - { - sample_query = - TrackPropertiesAt(tracks[0])->track_indentifiers.category == - kRocProfVisDmPmcTrack; - } - else - { - sample_query = - (rocprofvis_dm_event_operation_t) TABLE_QUERY_UNPACK_OP_TYPE( - tracks[0]) == kRocProfVisDmOperationNoOp; - } - if(sample_query) - { - query += ", COUNT(*) as count, AVG(value) as avg_value, MIN(value) as " - "min_value, MAX(value) as max_value"; - } - else - { - query += ", COUNT(*) as num_invocations, AVG(duration) as avg_duration, " - "MIN(duration) as min_duration, MAX(duration) as max_duration"; - } - } - query += "\n"; - } - - if(filter && strlen(filter)) - { - query += "-- CMD: FILTER "; - query += filter; - query += "\n"; - } - - if(sort_column && strlen(sort_column)) - { - query += "-- CMD: SORT"; - if(sort_order == kRPVDMSortOrderAsc) - { - query += " ASC "; - } - else - { - query += " DESC "; - } - query += sort_column; - query += "\n"; - } - if(count_only) - { - query += "-- CMD: COUNT"; - query += "\n"; - } - else - { - if(max_count) - { - query += "-- CMD: LIMIT "; - query += std::to_string(max_count); - query += "\n"; - } - if(offset) - { - query += "-- CMD: OFFSET "; - query += std::to_string(offset); - query += "\n"; - } - } + return kRocProfVisDmResultSuccess; } @@ -462,7 +395,7 @@ QueryManager::BuildTableQuery( rocprofvis_dm_table_use_case_enum_t use_case, rocprofvis_dm_timestamp_t start, rocprofvis_dm_timestamp_t end, rocprofvis_db_num_of_tracks_t num, rocprofvis_db_track_selection_t tracks, - rocprofvis_dm_charptr_t where, rocprofvis_dm_charptr_t filter, + rocprofvis_dm_processor_identifiers_ptr processor, rocprofvis_dm_charptr_t filter, rocprofvis_dm_charptr_t group, rocprofvis_dm_charptr_t group_cols, rocprofvis_dm_charptr_t sort_column, rocprofvis_dm_sort_order_t sort_order, uint64_t max_count, uint64_t offset, bool count_only, rocprofvis_dm_string_t& query) @@ -502,7 +435,7 @@ QueryManager::BuildTableQuery( return kRocProfVisDmResultSuccess; } return BuildCompoundQuery(use_case, start, end, num, tracks, slice_query_map_array, - where, filter, group, group_cols, sort_column, sort_order, + GetProcessorIDSubquery(processor).c_str(), filter, group, group_cols, sort_column, sort_order, max_count, offset, count_only, query); } @@ -510,7 +443,7 @@ rocprofvis_dm_result_t QueryManager::BuildEventSearchQuery( rocprofvis_dm_timestamp_t start, rocprofvis_dm_timestamp_t end, rocprofvis_db_num_of_tracks_t num, rocprofvis_db_track_selection_t ops, - rocprofvis_dm_charptr_t where, + rocprofvis_dm_processor_identifiers_ptr processor, rocprofvis_dm_num_string_table_filters_t num_string_table_filters, rocprofvis_dm_string_table_filters_t string_table_filters, bool include_substring, bool include_category, bool partial_matching, rocprofvis_dm_charptr_t sort_column, rocprofvis_dm_sort_order_t sort_order, @@ -551,7 +484,7 @@ QueryManager::BuildEventSearchQuery( return kRocProfVisDmResultSuccess; } return BuildCompoundQuery(kRPVDMTableUseCaseEventSearch, start, end, num, ops, slice_query_map_array, - where, nullptr, nullptr, nullptr, sort_column, sort_order, + GetProcessorIDSubquery(processor).c_str(), nullptr, nullptr, nullptr, sort_column, sort_order, max_count, offset, count_only, query); } @@ -864,11 +797,17 @@ QueryManager::ExecuteQueryForAllTracksAsync( try { - futures.back()->SetWorker(std::move( - std::thread(QueryManager::ExecuteSQLQueryStatic, this, - futures.back(), + Future* future = futures.back(); + const char* query = future->GetAsyncQueryPtr(); + future->SetWorker(std::move( + std::thread([this, + future, db_instance, - futures.back()->GetAsyncQueryPtr(), callback))); + query, callback]{ + return future->SetPromise( + ExecuteSQLQuery(future, db_instance, query, callback) + ); + }))); } catch (const std::exception& ex) { @@ -898,11 +837,11 @@ QueryManager::ExecuteQueryForAllTracksAsync( } rocprofvis_dm_result_t -QueryManager::ExecuteQueriesAsync( +QueryManager::GetEventTablesAsync( std::vector>& queries, Future* parent, rocprofvis_dm_handle_t handle, - RpvSqliteExecuteQueryCallback callback) + RpvCallback callback) { rocprofvis_dm_result_t result = kRocProfVisDmResultSuccess; // Register each worker as a sub-future of the parent so it is reachable by @@ -915,11 +854,15 @@ QueryManager::ExecuteQueriesAsync( futures[i] = parent->AddSubFuture(); try { + Future* future = futures[i]; + DbInstance* db_instance = queries[i].first; + const char* query = queries[i].second.c_str(); futures[i]->SetWorker(std::move( - std::thread(ExecuteSQLQueryStaticWithHandle, this, - futures[i], - queries[i].first, - queries[i].second.c_str(), handle, i, callback))); + std::thread([this, future, db_instance, query, handle, i, callback]{ + return future->SetPromise( + ExecuteSQLQuery(future, db_instance, query, handle, i, (RpvSqliteExecuteQueryCallback)callback) + ); + }))); } catch(const std::exception& ex) { // The worker thread never started, so the sub-future's promise will @@ -1063,6 +1006,19 @@ rocprofvis_dm_result_t QueryManager::ExportTableCSV(rocprofvis_dm_charptr_t quer return future->SetPromise(result); } +std::string QueryManager::TableColumnText(void* func, void* handle, char** azColName, int index) { + return Sqlite3ColumnText(func, (sqlite3_stmt*)handle, azColName, index); +} +int QueryManager::TableColumnInt(void* func, void* handle, char** azColName, int index) { + return Sqlite3ColumnInt(func, (sqlite3_stmt*)handle, azColName, index); +} +int64_t QueryManager::TableColumnInt64(void* func, void* handle, char** azColName, int index) { + return Sqlite3ColumnInt64(func, (sqlite3_stmt*)handle, azColName, index); +} +double QueryManager::TableColumnDouble(void* func, void* handle, char** azColName, int index) { + return Sqlite3ColumnDouble(func, (sqlite3_stmt*)handle, azColName, index); +} + } // namespace DataModel } // namespace RocProfVis diff --git a/src/model/src/database/rocprofvis_db_query_manager.h b/src/model/src/database/rocprofvis_db_query_manager.h index fd3acb56f..dc8881603 100644 --- a/src/model/src/database/rocprofvis_db_query_manager.h +++ b/src/model/src/database/rocprofvis_db_query_manager.h @@ -4,6 +4,7 @@ #pragma once #include "rocprofvis_db_sqlite.h" +#include "rocprofvis_db_systems.h" #include "sqlite3.h" #include #include @@ -48,42 +49,34 @@ typedef enum rocprofvis_db_sqlite_query_type_t kRPVNumSourceQueryTypes = 6 } rocprofvis_db_sqlite_query_type_t; -typedef enum rocprofvis_dm_track_search_id_t -{ - kRPVTrackSearchIdThreads, - kRPVTrackSearchIdThreadSamples, - kRPVTrackSearchIdDispatches, - kRPVTrackSearchIdMemAllocs, - kRPVTrackSearchIdMemCopies, - kRPVTrackSearchIdCounters, - kRPVTrackSearchIdStreams, - kRPVTrackSearchIdUnknown, -} rocprofvis_dm_track_search_id_t; - -typedef enum rocprofvis_db_compound_table_type { - kRPVTableDataTypeEvent, - kRPVTableDataTypeSample, - kRPVTableDataTypeSearch, - kRPVTableDataTypesNum -} rocprofvis_db_compound_table_type; - -typedef std::map> rocprofvis_event_data_category_map_t; +// type of map array for generating time slice query for multiple tracks +typedef std::unordered_map> slice_query_map_t; +// type of map array for storing string id filters for op table queries +typedef std::unordered_map> table_string_id_filter_map_t; // class for any Sqlite database methods and properties -class QueryManager : public SqliteDatabase +class QueryManager : public SystemDatabase, public SqliteDatabase { friend class TableProcessor; friend class PackedTable; friend class TrackLookup; + public: // Database constructor // @param path - full path to database file QueryManager( rocprofvis_db_filename_t path, RpvSqliteExecuteQueryCallback callback_add_any_record) : - SqliteDatabase(path), m_callback_add_any_record(callback_add_any_record), + SystemDatabase(path), SqliteDatabase(this), m_callback_add_any_record(callback_add_any_record), m_table_processor{TableProcessor(this),TableProcessor(this),TableProcessor(this)} {}; // SqliteDatabase destructor, must be defined as virtual to free resources of derived classes virtual ~QueryManager() {}; + // Method to open sqlite database + // @return status of operation + rocprofvis_dm_result_t Open() override { return OpenAsSqlite(); }; + // Method to close sqlite database + // @return status of operation + rocprofvis_dm_result_t Close() override { return CloseAsSqlite(); }; + // worker method to execute database query // @param query - database query // @param description - database description @@ -153,6 +146,15 @@ class QueryManager : public SqliteDatabase bool count_only, rocprofvis_dm_string_t& query); + rocprofvis_dm_result_t + BuildTableSqlSubQuery( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_db_track_selection_t tracks, + std::vector& slice_query_map_array, + rocprofvis_dm_charptr_t where, + rocprofvis_dm_string_t& query); + // method to build a query to read time slice of records for table view // @param use_case - the method is multi-use, this is enumeration of use cases // @param start - start timestamp of time slice @@ -176,7 +178,7 @@ class QueryManager : public SqliteDatabase rocprofvis_dm_timestamp_t end, rocprofvis_db_num_of_tracks_t num, rocprofvis_db_track_selection_t tracks, - rocprofvis_dm_charptr_t where, + rocprofvis_dm_processor_identifiers_ptr processor, rocprofvis_dm_charptr_t filter, rocprofvis_dm_charptr_t group, rocprofvis_dm_charptr_t group_cols, @@ -207,7 +209,7 @@ class QueryManager : public SqliteDatabase rocprofvis_dm_timestamp_t end, rocprofvis_db_num_of_tracks_t num, rocprofvis_db_track_selection_t ops, - rocprofvis_dm_charptr_t where, + rocprofvis_dm_processor_identifiers_ptr processor, rocprofvis_dm_num_string_table_filters_t num_string_table_filters, rocprofvis_dm_string_table_filters_t string_table_filters, bool include_substring, @@ -241,12 +243,9 @@ class QueryManager : public SqliteDatabase rocprofvis_dm_track_params_t* props, rocprofvis_db_query_type_t query_type) = 0; - // Searches for strings matching the passed in list of filter strings and builds a WHERE IN clause for the table query. + // Searches for strings containing the passed in list of filter strings and builds a WHERE IN clause for the table query. // @param num_string_table_filters - number of filter strings // @param string_table_filters - array of filter strings - // @param include_substring - when true a string matches if it contains the filter, when false it has to equal the filter. - // @param include_category - when true the filters are matched against the event category as well as the event name, when false only against the event name. - // @param partial_matching - when true a string matches if it matches any of the filters, when false it has to match all of them. // @param filter - output string containing WHERE clause // @return status of operation virtual rocprofvis_dm_result_t BuildTableStringIdFilter( @@ -257,17 +256,6 @@ class QueryManager : public SqliteDatabase bool partial_matching, table_string_id_filter_map_t& filter) = 0; - // needs to be overriden in all adapters. Used by public interface method. - rocprofvis_dm_result_t BuildComputeQuery( - rocprofvis_db_compute_use_case_enum_t use_case, rocprofvis_db_num_of_params_t num, rocprofvis_db_compute_params_t params, - rocprofvis_dm_string_t& query) override { - (void) use_case; - (void) num; - (void) params; - (void) query; - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN("Systems database does not build compute query", kRocProfVisDmResultNotSupported); - } - // Get prefix an suffix part of histogram calculation query std::string GetHistogramQueryPrefix(uint64_t bucket_size); std::string GetHistogramQuerySuffix(); @@ -345,32 +333,23 @@ class QueryManager : public SqliteDatabase std::function func_prepare, std::function func_clear, guid_list_t run_for_db_instances); - + // executes set of queries asynchronously - rocprofvis_dm_result_t ExecuteQueriesAsync( + rocprofvis_dm_result_t GetEventTablesAsync( std::vector>& queries, Future* parent, rocprofvis_dm_handle_t handle, - RpvSqliteExecuteQueryCallback callback); + RpvCallback callback) override; - // needs to be defined as override. Used by public interface method - rocprofvis_dm_result_t ExecuteComputeQuery( - rocprofvis_db_compute_use_case_enum_t use_case, - rocprofvis_dm_charptr_t query, - Future* future) override { - (void) use_case; - (void) query; - (void) future; - ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN("Systems database does not support compute query", kRocProfVisDmResultNotSupported); - } - - StringTable& StringTableReference() { return m_string_table; }; - - virtual rocprofvis_dm_result_t RemapStringId(uint64_t id, rocprofvis_db_string_type_t type, uint32_t node, uint64_t & result) = 0; - virtual void GetTrackIdentifierIndices(int column_index, char** azColName, rocprofvis_db_sqlite_track_identifier_index_t& track_ids_indices) = 0; - virtual bool FindTrack(rocprofvis_dm_track_category_t category, uint64_t id_process, uint64_t id_subprocess, uint32_t db_instance, uint32_t& out_track) = 0; virtual rocprofvis_dm_track_category_t GetRegionTrackCategory() = 0; virtual const rocprofvis_event_data_category_map_t* GetCategoryEnumMap() = 0; + virtual std::string GetProcessorIDSubquery(rocprofvis_dm_processor_identifiers_ptr processor) { return std::string(); } + + //--------------------------------------Table accessors----------------------------------------------------------------- + std::string TableColumnText(void* func, void* handle, char** azColName, int index) override; + int TableColumnInt(void* func, void* handle, char** azColName, int index) override; + int64_t TableColumnInt64(void* func, void* handle, char** azColName, int index) override; + double TableColumnDouble(void* func, void* handle, char** azColName, int index) override; private: @@ -382,7 +361,7 @@ class QueryManager : public SqliteDatabase protected: TableProcessor m_table_processor[kRPVTableDataTypesNum]; - StringTable m_string_table; + }; diff --git a/src/model/src/database/rocprofvis_db_rocpd.cpp b/src/model/src/database/rocprofvis_db_rocpd.cpp index f033ca406..3fe1bf9f5 100644 --- a/src/model/src/database/rocprofvis_db_rocpd.cpp +++ b/src/model/src/database/rocprofvis_db_rocpd.cpp @@ -174,7 +174,7 @@ int RocpdDatabase::CallBackAgentToProcess(void *data, int argc, sqlite3_stmt* st ROCPROFVIS_ASSERT_MSG_RETURN(argc==3, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallBackAgentToProcess; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; RocpdDatabase* db = (RocpdDatabase*)callback_params->db; if(callback_params->future->Interrupted()) return SQLITE_ABORT; rocprofvis_dm_process_id pid = db->Sqlite3ColumnInt(func, stmt, azColName, 0); @@ -188,7 +188,7 @@ int RocpdDatabase::CallBackAddString(void *data, int argc, sqlite3_stmt* stmt, c ROCPROFVIS_ASSERT_MSG_RETURN(argc==2, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallBackAddString; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; RocpdDatabase* db = (RocpdDatabase*)callback_params->db; if(callback_params->future->Interrupted()) return SQLITE_ABORT; std::stringstream ids((char*)db->Sqlite3ColumnText(func, stmt, azColName, 1)); @@ -227,6 +227,19 @@ std::string RocpdDatabase::GetLevelSchemaHashStr() return hash_str; } +std::string RocpdDatabase::GetProcessorIDSubquery(rocprofvis_dm_processor_identifiers_ptr processor) +{ + std::string where_str; + if(processor) + { + if(processor->agent_id) + { + where_str += " gpuId = " + std::to_string(*processor->agent_id & TOPOLOGY_ID_MASK); + } + } + return where_str; +} + rocprofvis_dm_result_t RocpdDatabase::ReadTraceMetadata(Future* future) { ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); diff --git a/src/model/src/database/rocprofvis_db_rocpd.h b/src/model/src/database/rocprofvis_db_rocpd.h index 57f05a508..2704b5c01 100644 --- a/src/model/src/database/rocprofvis_db_rocpd.h +++ b/src/model/src/database/rocprofvis_db_rocpd.h @@ -106,6 +106,8 @@ class RocpdDatabase : public ProfileDatabase std::string GetLevelSchemaHashStr(); + std::string GetProcessorIDSubquery(rocprofvis_dm_processor_identifiers_ptr processor) override; + private: // ------------------------------SQL query callbacks-------------------------------------- diff --git a/src/model/src/database/rocprofvis_db_rocprof.cpp b/src/model/src/database/rocprofvis_db_rocprof.cpp index 1ec8535d9..6b02eaed3 100644 --- a/src/model/src/database/rocprofvis_db_rocprof.cpp +++ b/src/model/src/database/rocprofvis_db_rocprof.cpp @@ -78,7 +78,7 @@ int RocprofDatabase::CallbackParseMetadata(void* data, int argc, sqlite3_stmt* s ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackParseMetadata; rocprofvis_dm_track_params_t track_params = {0}; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; RocprofDatabase* db = (RocprofDatabase*)callback_params->db; std::string tag = db->Sqlite3ColumnText(func, stmt, azColName, 1); if (tag == "schema_version") @@ -191,7 +191,7 @@ int RocprofDatabase::CallbackCaptureMemoryActivity(void* data, int argc, sqlite3 ROCPROFVIS_ASSERT_MSG_RETURN(argc==rocprofvis_db_sqlite_memory_alloc_activity_query_format::NUM_PARAMS, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackCaptureMemoryActivity; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); RocprofDatabase* db = (RocprofDatabase*)callback_params->db; if(callback_params->future->Interrupted()) return SQLITE_ABORT; @@ -522,7 +522,7 @@ int RocprofDatabase::CallBackAddString(void *data, int argc, sqlite3_stmt* stmt, ROCPROFVIS_ASSERT_MSG_RETURN(argc==3, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallBackAddString; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); RocprofDatabase* db = (RocprofDatabase*)callback_params->db; if(callback_params->future->Interrupted()) return SQLITE_ABORT; @@ -547,8 +547,8 @@ RocprofDatabase::CallbackNodeEnumeration(void* data, int argc, sqlite3_stmt* stm (void) argc; ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackNodeEnumeration; - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) data; RocprofDatabase* db = (RocprofDatabase*) callback_params->db; guid_list_t* guid_list = (guid_list_t*) callback_params->handle; std::string table_name_befor_guid = callback_params->query[kRPVCacheTableName]; @@ -641,6 +641,23 @@ RocprofDatabase::CreateIndexes() } +std::string RocprofDatabase::GetProcessorIDSubquery(rocprofvis_dm_processor_identifiers_ptr processor) +{ + std::string where_str; + if(processor) + { + if(processor->node_id) + { + where_str = std::string(Builder::NODE_ID_SERVICE_NAME) +" = " + std::to_string(*processor->node_id); + if(processor->agent_id) + { + where_str += std::string(" AND ") + Builder::AGENT_ID_SERVICE_NAME + " = " + std::to_string(*processor->agent_id & TOPOLOGY_ID_MASK); + } + } + } + return where_str; +} + rocprofvis_dm_result_t RocprofDatabase::GenerateInterdependencyTables(Future* future) { diff --git a/src/model/src/database/rocprofvis_db_rocprof.h b/src/model/src/database/rocprofvis_db_rocprof.h index 453b69b4f..a4433d442 100644 --- a/src/model/src/database/rocprofvis_db_rocprof.h +++ b/src/model/src/database/rocprofvis_db_rocprof.h @@ -11,21 +11,6 @@ namespace RocProfVis namespace DataModel { -typedef struct rocprofvis_db_string_id_hash_t -{ - size_t operator()(const rocprofvis_db_string_id_t& s) const noexcept - { - size_t h1 = std::hash{}(s.m_string_id); - size_t h2 = std::hash{}(s.m_guid_id); - size_t h3 = std::hash{}(s.m_string_type); - - size_t seed = h1; - seed ^= h2 + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); - seed ^= h3 + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); - return seed; - } -} rocprofvis_db_string_id_hash_t; - typedef enum rocprofvis_db_memalloc_type_t : uint8_t { kRPVMemActivityAlloc, @@ -140,6 +125,7 @@ class RocprofDatabase : public ProfileDatabase uint64_t GetMemoryActivityTableSchemaHash(); std::string GetLevelSchemaHashStr(); + std::string GetProcessorIDSubquery(rocprofvis_dm_processor_identifiers_ptr processor) override; private: diff --git a/src/model/src/database/rocprofvis_db_sqlite.cpp b/src/model/src/database/rocprofvis_db_sqlite.cpp index a8684dc2a..8d8940815 100644 --- a/src/model/src/database/rocprofvis_db_sqlite.cpp +++ b/src/model/src/database/rocprofvis_db_sqlite.cpp @@ -138,62 +138,6 @@ SqliteDatabase::Sqlite3ColumnDouble(void* func, sqlite3_stmt* stmt, char** azCol } } - - -int SqliteDatabase::CallbackRunQuery(void *data, int argc, sqlite3_stmt* stmt, char **azColName){ - ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; - SqliteDatabase* db = (SqliteDatabase*)callback_params->db; - void* func = (void*)&CallbackRunQuery; - if (callback_params->future->Interrupted()) return 1; - rocprofvis_dm_table_row_t row = - db->BindObject()->FuncAddTableRow(callback_params->handle); - ROCPROFVIS_ASSERT_MSG_RETURN(row, ERROR_TABLE_ROW_CANNOT_BE_NULL, 1); - - if(0 == callback_params->future->GetProcessedRowsCount()) - { - for (int i=0; i < argc; i++) - { - if (kRocProfVisDmResultSuccess != db->BindObject()->FuncAddTableColumn(callback_params->handle,azColName[i])) return 1; - } - } - for (int i=0; i < argc; i++) - { - std::string column_text = db->Sqlite3ColumnText(func, stmt, azColName, i); - if (kRocProfVisDmResultSuccess != db->BindObject()->FuncAddTableRowCell(row, column_text.c_str())) return 1; - } - - callback_params->future->CountThisRow(); - return 0; -} - - -rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQueryStatic( - SqliteDatabase* db, - Future* future, - DbInstance* db_instance, - const char* query, - RpvSqliteExecuteQueryCallback callback) -{ - return future->SetPromise( - db->ExecuteSQLQuery(future, db_instance, query, callback) - ); -} - -rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQueryStaticWithHandle( - SqliteDatabase* db, - Future* future, - DbInstance* db_instance, - const char* query, - rocprofvis_dm_handle_t handle, - uint32_t query_index, - RpvSqliteExecuteQueryCallback callback) -{ - return future->SetPromise( - db->ExecuteSQLQuery(future, db_instance, query, handle, query_index, callback) - ); -} - int SqliteDatabase::DetectTable(sqlite3* conn, const char* table, bool is_view) { @@ -233,7 +177,7 @@ SqliteDatabase::DetectTable(sqlite3* conn, const char* table, bool is_view) } -rocprofvis_dm_result_t SqliteDatabase::Open() +rocprofvis_dm_result_t SqliteDatabase::OpenAsSqlite() { for (auto & node : m_db_nodes) { @@ -266,7 +210,7 @@ rocprofvis_dm_result_t SqliteDatabase::OpenConnection(uint32_t db_node_id, sqlit return kRocProfVisDmResultSuccess; } -rocprofvis_dm_result_t SqliteDatabase::Close() +rocprofvis_dm_result_t SqliteDatabase::CloseAsSqlite() { rocprofvis_dm_result_t result = kRocProfVisDmResultSuccess; for (auto & node : m_db_nodes) @@ -372,8 +316,8 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery( DbInstance* db_instance, const char* query ){ - rocprofvis_db_sqlite_callback_parameters params = { - this, + rocprofvis_db_query_callback_parameters params = { + m_db, future, nullptr, nullptr, @@ -387,13 +331,13 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery( Future* future, DbInstance* db_instance, const char* query, - RpvSqliteExecuteQueryCallback callback) + RpvSqliteCallback callback) { - rocprofvis_db_sqlite_callback_parameters params = { - this, + rocprofvis_db_query_callback_parameters params = { + m_db, future, nullptr, - callback, + (RpvCallback)callback, { query }, INVALID_INDEX }; @@ -405,13 +349,13 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery( Future* future, DbInstance* db_instance, const char* query, - RpvSqliteExecuteQueryCallback callback, + RpvSqliteCallback callback, rocprofvis_dm_string_t* value){ - rocprofvis_db_sqlite_callback_parameters params = { - this, + rocprofvis_db_query_callback_parameters params = { + m_db, future, (rocprofvis_dm_handle_t) value, - callback, + (RpvCallback)callback, { query }, INVALID_INDEX }; @@ -422,7 +366,7 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery( Future* future, DbInstance* db_instance, const char* query, - RpvSqliteExecuteQueryCallback callback, + RpvSqliteCallback callback, uint64_t & value){ std::string str_value; rocprofvis_dm_result_t result = ExecuteSQLQuery(future, db_instance, query, callback, &str_value); @@ -437,7 +381,7 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery( Future* future, DbInstance* db_instance, const char* query, - RpvSqliteExecuteQueryCallback callback, + RpvSqliteCallback callback, uint32_t & value){ std::string str_value; rocprofvis_dm_result_t result = ExecuteSQLQuery(future, db_instance, query, callback, &str_value); @@ -453,12 +397,12 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery( DbInstance* db_instance, const char* query, rocprofvis_dm_handle_t handle, - RpvSqliteExecuteQueryCallback callback){ - rocprofvis_db_sqlite_callback_parameters params = { - this, + RpvSqliteCallback callback){ + rocprofvis_db_query_callback_parameters params = { + m_db, future, handle, - callback, + (RpvCallback)callback, { query }, INVALID_INDEX }; @@ -471,12 +415,12 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery( const char* query, rocprofvis_dm_handle_t handle, uint32_t index, - RpvSqliteExecuteQueryCallback callback){ - rocprofvis_db_sqlite_callback_parameters params = { - this, + RpvSqliteCallback callback){ + rocprofvis_db_query_callback_parameters params = { + m_db, future, handle, - callback, + (RpvCallback)callback, { query }, static_cast(index) }; @@ -490,12 +434,12 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery( const char* cache_table_name, rocprofvis_dm_handle_t handle, rocprofvis_dm_event_operation_t op, - RpvSqliteExecuteQueryCallback callback){ - rocprofvis_db_sqlite_callback_parameters params = { - this, + RpvSqliteCallback callback){ + rocprofvis_db_query_callback_parameters params = { + m_db, future, handle, - callback, + (RpvCallback)callback, { query, cache_table_name }, INVALID_INDEX, op @@ -510,13 +454,13 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery( const char* query, const char* cache_table_name, rocprofvis_dm_handle_t handle, - RpvSqliteExecuteQueryCallback callback) + RpvSqliteCallback callback) { - rocprofvis_db_sqlite_callback_parameters params = { - this, + rocprofvis_db_query_callback_parameters params = { + m_db, future, handle, - callback, + (RpvCallback)callback, { query, cache_table_name }, INVALID_INDEX, }; @@ -528,14 +472,14 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery(Future* future, DbInstance* db_instance, uint32_t load_id, std::vector query, - RpvSqliteExecuteQueryCallback find_callback, - RpvSqliteExecuteQueryCallback load_callback) + RpvSqliteCallback find_callback, + RpvSqliteCallback load_callback) { - rocprofvis_db_sqlite_callback_parameters params = { - this, + rocprofvis_db_query_callback_parameters params = { + m_db, future, nullptr, - load_callback, + (RpvCallback)load_callback, query, static_cast(load_id) }; @@ -546,7 +490,7 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery(Future* future, if (false == GetMetadataVersionControl()->MustRebuildTrackInfo(db_instance->FileIndex())) { std::string load_query = std::string("SELECT * FROM ") + load_table_name + " WHERE load_id = " + std::to_string(load_id); - std::string guid = GuidAt(db_instance->GuidIndex()); + std::string guid = m_db->GuidAt(db_instance->GuidIndex()); if (!guid.empty()) { load_query += " AND guid = '" + guid+"'"; @@ -554,8 +498,8 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery(Future* future, result = ExecuteSQLQuery(db_instance, load_query.c_str(), ¶ms); } else { - TraceProperties()->tracks_info_restored = false; - params.callback = find_callback; + m_db->TraceProperties()->tracks_info_restored = false; + params.callback = (RpvCallback)find_callback; for (int i = 0; i < 2; i++) { if (query[i].length() > 0) @@ -575,11 +519,11 @@ int SqliteDatabase::Sqlite3Exec(sqlite3* db, const char* query, { int rc=0; sqlite3_stmt* stmt = nullptr; - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) user_data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) user_data; if (callback_params->future != nullptr) { - callback_params->future->LinkDatabase(this, db); + callback_params->future->LinkDatabase(m_db, db); } sqlite3_mutex_enter(sqlite3_db_mutex(db)); rc = sqlite3_prepare_v2(db, query, -1, &stmt, nullptr); @@ -648,7 +592,7 @@ void SqliteDatabase::ReplaceAllSubstrings(std::string& str, const std::string& f } } -rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery(DbInstance* db_instance, const char* query, rocprofvis_db_sqlite_callback_parameters * params) +rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery(DbInstance* db_instance, const char* query, rocprofvis_db_query_callback_parameters * params) { PROFILE; ROCPROFVIS_ASSERT_MSG_RETURN(db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); @@ -656,10 +600,10 @@ rocprofvis_dm_result_t SqliteDatabase::ExecuteSQLQuery(DbInstance* db_instance, params->db_instance = db_instance; sqlite3* conn = GetConnection(db_instance->FileIndex()); std::string query_str = query; - std::string guid_str = GuidAt(db_instance->GuidIndex()); + std::string guid_str = m_db->GuidAt(db_instance->GuidIndex()); ReplaceAllSubstrings(query_str, "%GUID%", guid_str); query = query_str.c_str(); - int rc = Sqlite3Exec(conn, query, params->callback, params); + int rc = Sqlite3Exec(conn, query, (RpvSqliteCallback)params->callback, params); if(rc != SQLITE_OK) { if (rc == SQLITE_ABORT) diff --git a/src/model/src/database/rocprofvis_db_sqlite.h b/src/model/src/database/rocprofvis_db_sqlite.h index 462bc601e..d64b3c6b9 100644 --- a/src/model/src/database/rocprofvis_db_sqlite.h +++ b/src/model/src/database/rocprofvis_db_sqlite.h @@ -21,7 +21,8 @@ namespace DataModel // type of sqlite3_exec callback function -typedef int (*RpvSqliteExecuteQueryCallback)(void*, int, sqlite3_stmt*, char**); +typedef int (*RpvSqliteCallback)(void*, int, sqlite3_stmt*, char**); + typedef struct SQLInsertParam { const char* column; @@ -30,24 +31,6 @@ typedef struct SQLInsertParam typedef std::vector SQLInsertParams; - -// structure to pass parameters to sqlite3_exec callbacks -typedef struct{ - // pointer tp Database object - Database* db; - // pointer to Future object, to check if thread has been interrupted - Future* future; - // pointer to container object handle, to add processed rows data to the container - rocprofvis_dm_handle_t handle; - // callback method pointer - RpvSqliteExecuteQueryCallback callback; - // pointer to query string, convenient for multiuse callback debugging - std::vector query; - rocprofvis_dm_track_id_t track_id; - rocprofvis_dm_event_operation_t operation; - DbInstance* db_instance; -} rocprofvis_db_sqlite_callback_parameters; - typedef struct rocprofvis_db_sqlite_db_node_t { uint32_t node_id; @@ -63,26 +46,23 @@ typedef std::map> rocprofvis_null_data typedef std::map> rocprofvis_null_data_exceptions_skip; // class for any Sqlite database methods and properties -class SqliteDatabase : public Database +class SqliteDatabase { public: // Database constructor // @param path - full path to database file - SqliteDatabase( rocprofvis_db_filename_t path) : - Database(path) {}; + SqliteDatabase(Database* db): m_db(db) {}; // SqliteDatabase destructor, must be defined as virtual to free resources of derived classes - virtual ~SqliteDatabase() {Close();} - // Method to open sqlite database - // @return status of operation - rocprofvis_dm_result_t Open() override; - // Method to close sqlite database - // @return status of operation - rocprofvis_dm_result_t Close() override; - void InterruptQuery(void* connection) override; + virtual ~SqliteDatabase() {CloseAsSqlite();} + void InterruptQuery(void* connection); // check if table present in database bool CheckTableExists(const std::string& table_name, uint32_t db_node_id); protected: + + rocprofvis_dm_result_t OpenAsSqlite(); + rocprofvis_dm_result_t CloseAsSqlite(); + // ---------------------------------------SQL operations----------------------------------------- // Method to create SQL table // @param table_name - table name @@ -118,8 +98,8 @@ class SqliteDatabase : public Database DbInstance* db_instance, uint32_t load_id, std::vector query, - RpvSqliteExecuteQueryCallback find_callback, - RpvSqliteExecuteQueryCallback load_callback); + RpvSqliteCallback find_callback, + RpvSqliteCallback load_callback); // Method for single row and column SQL query execution returning result of the query as string // @param future - future object for asynchronous execution status // @param query - SQL query @@ -129,7 +109,7 @@ class SqliteDatabase : public Database rocprofvis_dm_result_t ExecuteSQLQuery(Future* future, DbInstance* db_instance, const char* query, - RpvSqliteExecuteQueryCallback callback, + RpvSqliteCallback callback, rocprofvis_dm_string_t* value); // Method for single row and column SQL query execution returning result of the query as uint64 // @param future - future object for asynchronous execution status @@ -140,7 +120,7 @@ class SqliteDatabase : public Database rocprofvis_dm_result_t ExecuteSQLQuery(Future* future, DbInstance* db_instance, const char* query, - RpvSqliteExecuteQueryCallback callback, + RpvSqliteCallback callback, uint64_t & value); // Method for single row and column SQL query execution returning result of the query as uint32 // @param future - future object for asynchronous execution status @@ -151,7 +131,7 @@ class SqliteDatabase : public Database rocprofvis_dm_result_t ExecuteSQLQuery(Future* future, DbInstance* db_instance, const char* query, - RpvSqliteExecuteQueryCallback callback, + RpvSqliteCallback callback, uint32_t & value); // Method for SQL query execution with handle parameter. // Used for callbacks storing data into container with rocprofvis_dm_handle_t handle @@ -164,7 +144,7 @@ class SqliteDatabase : public Database DbInstance* db_instance, const char* query, rocprofvis_dm_handle_t handle, - RpvSqliteExecuteQueryCallback callback); + RpvSqliteCallback callback); // Method for SQL query execution with handle and query index parameter. // Used for callbacks storing data into container with rocprofvis_dm_handle_t handle // @param future - future object for asynchronous execution status @@ -178,7 +158,7 @@ class SqliteDatabase : public Database const char* query, rocprofvis_dm_handle_t handle, uint32_t index, - RpvSqliteExecuteQueryCallback callback); + RpvSqliteCallback callback); // Method for SQL query execution with multi-use subquery parameter. // Used for callbacks storing data into container with rocprofvis_dm_handle_t handle // @param future - future object for asynchronous execution status @@ -189,7 +169,7 @@ class SqliteDatabase : public Database rocprofvis_dm_result_t ExecuteSQLQuery(Future* future, DbInstance* db_instance, const char* query, - RpvSqliteExecuteQueryCallback callback); + RpvSqliteCallback callback); // Method for SQL query execution with multi-use subquery and handle parameter. // Used for callbacks storing data into container with rocprofvis_dm_handle_t handle // @param future - future object for asynchronous execution status @@ -203,7 +183,7 @@ class SqliteDatabase : public Database const char* query, const char* cache_table_name, rocprofvis_dm_handle_t handle, - RpvSqliteExecuteQueryCallback callback); + RpvSqliteCallback callback); rocprofvis_dm_result_t ExecuteSQLQuery(Future* future, DbInstance* db_instance, @@ -211,7 +191,7 @@ class SqliteDatabase : public Database const char* cache_table_name, rocprofvis_dm_handle_t handle, rocprofvis_dm_event_operation_t op, - RpvSqliteExecuteQueryCallback callback); + RpvSqliteCallback callback); // method to run SQL query // @param db_conn - database connection // @param query - SQL query @@ -219,7 +199,7 @@ class SqliteDatabase : public Database rocprofvis_dm_result_t ExecuteSQLQuery( DbInstance* db_instance, const char* query, - rocprofvis_db_sqlite_callback_parameters * params); + rocprofvis_db_query_callback_parameters * params); rocprofvis_dm_result_t ExecuteTransaction( std::vector queries, @@ -231,22 +211,6 @@ class SqliteDatabase : public Database // @param conn - connection static int DetectTable(sqlite3* conn, const char* table, bool is_view = true); - // ---------------------------------------Thread workers-------------------------------------------- - static rocprofvis_dm_result_t ExecuteSQLQueryStatic( - SqliteDatabase* db, - Future* future, - DbInstance* db_instance, - const char* query, - RpvSqliteExecuteQueryCallback callback); - static rocprofvis_dm_result_t ExecuteSQLQueryStaticWithHandle( - SqliteDatabase* db, - Future* future, - DbInstance* db_instance, - const char* query, - rocprofvis_dm_handle_t handle, - uint32_t query_index, - RpvSqliteExecuteQueryCallback callback); - // ------------------------------Wrappers around SQL getters------------------------------------- char* Sqlite3ColumnText(void* func, sqlite3_stmt* stmt, char** azColName, int index); @@ -254,11 +218,6 @@ class SqliteDatabase : public Database int64_t Sqlite3ColumnInt64(void* func, sqlite3_stmt* stmt, char** azColName, int index); double Sqlite3ColumnDouble(void* func, sqlite3_stmt* stmt, char** azColName, int index); - // ---------------------------------------Callbacks-------------------------------------------- - static int CallbackGetValue(void* data, int argc, sqlite3_stmt* stmt, char** azColName); - static int CallbackRunQuery(void *data, int argc, sqlite3_stmt* stmt, char **azColName); - static int CallbackMakeHistogramPerTrack(void* data, int argc, sqlite3_stmt* stmt, char** azColName); - // ---------------------------------------Helpers-------------------------------------------- sqlite3* GetServiceConnection(uint32_t db_node_id=0); @@ -277,7 +236,6 @@ class SqliteDatabase : public Database private: - // method to mimic slite3_exec using sqlite3_prepare_v2 // @param db - database connection // @param query - SQL query @@ -301,6 +259,7 @@ class SqliteDatabase : public Database protected: std::vector> m_db_nodes; + Database* m_db; }; diff --git a/src/model/src/database/rocprofvis_db_systems.cpp b/src/model/src/database/rocprofvis_db_systems.cpp new file mode 100644 index 000000000..52bcb9a91 --- /dev/null +++ b/src/model/src/database/rocprofvis_db_systems.cpp @@ -0,0 +1,309 @@ +// Copyright Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include "rocprofvis_db.h" +#include "rocprofvis_db_profile.h" +#include +#include +#include +#include +#include + +namespace RocProfVis +{ +namespace DataModel +{ + + +rocprofvis_dm_result_t SystemDatabase::AddTrackProperties( + rocprofvis_dm_track_params_t& props) { + try { + m_track_properties.push_back(std::make_unique(props)); + } + catch (const std::exception&) + { + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ERROR_MEMORY_ALLOCATION_FAILURE, kRocProfVisDmResultAllocFailure); + } + return kRocProfVisDmResultSuccess; +} + + +rocprofvis_dm_result_t SystemDatabase::ReadTraceSliceAsync( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_hashed_timestamp_tag_t tag, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t tracks, + rocprofvis_db_future_t object){ + Future* future = (Future*) object; + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, kRocProfVisDmResultResourceBusy); + rocprofvis_dm_result_t result = BindObject()->FuncCheckSliceExists(BindObject()->trace_object, start, end, tag, num, tracks); + if(result != kRocProfVisDmResultNotLoaded) + { + spdlog::debug("Slice ({},{}) exists!", start, end); + return future->SetPromise(result); + } + try { + future->SetWorker(std::move(std::thread([this, start, end, tag, num, tracks, future] { + return ReadTraceSlice(start, end, tag, num, tracks, future); + }))); + } + catch (const std::exception& ex) + { + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); + } + return kRocProfVisDmResultSuccess; +} + +rocprofvis_dm_result_t +SystemDatabase::ReadTracePMCSliceAsync( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_hashed_timestamp_tag_t tag, + rocprofvis_db_track_selection_t track, + bool left_neighbor, + bool right_neighbor, + rocprofvis_db_future_t object){ + Future* future = (Future*) object; + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, kRocProfVisDmResultResourceBusy); + rocprofvis_dm_result_t result = BindObject()->FuncCheckSliceExists(BindObject()->trace_object, start, end, tag, 1, track); + if(result != kRocProfVisDmResultNotLoaded) + { + spdlog::debug("Slice ({},{}) exists!", start, end); + return future->SetPromise(result); + } + try { + future->SetWorker(std::move(std::thread([this, start, end, tag, track, left_neighbor, right_neighbor, future] { + return ReadTracePMCSlice(start, end, tag, track, left_neighbor, right_neighbor, future); + }))); + } + catch (const std::exception& ex) + { + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); + } + return kRocProfVisDmResultSuccess; +} + +rocprofvis_dm_result_t SystemDatabase::ReadEventPropertyAsync( + rocprofvis_dm_event_property_type_t type, + rocprofvis_dm_event_id_t event_id, + rocprofvis_db_future_t object){ + Future* future = (Future*) object; + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, kRocProfVisDmResultResourceBusy); + rocprofvis_dm_result_t result = BindObject()->FuncCheckEventPropertyExists(BindObject()->trace_object, type, event_id); + if(result != kRocProfVisDmResultNotLoaded) + { + return future->SetPromise(kRocProfVisDmResultResourceBusy); + } + try { + future->SetWorker(std::move(std::thread([this, type, event_id, future]{ + switch (type) { + case kRPVDMEventFlowTrace: + return ReadFlowTraceInfo(event_id,future); + case kRPVDMEventStackTrace: + return ReadStackTraceInfo(event_id,future); + case kRPVDMEventExtData: + return ReadExtEventInfo(event_id,future); + } + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ERROR_UNSUPPORTED_PROPERTY, kRocProfVisDmResultNotSupported); + }))); + } + catch (std::exception ex) + { + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); + } + return kRocProfVisDmResultSuccess; +} + + +rocprofvis_dm_result_t SystemDatabase::ExportTableCSVAsync(rocprofvis_dm_string_t query, + rocprofvis_dm_string_t file_path, + rocprofvis_db_future_t object) +{ + Future* future = (Future*) object; + ROCPROFVIS_ASSERT_MSG_RETURN(!file_path.empty(), "Output path cannot be empty.", + kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, + kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, + kRocProfVisDmResultResourceBusy); + try + { + future->SetWorker(std::move(std::thread([this, query, file_path, future]{ + ROCPROFVIS_ASSERT_MSG_RETURN(!file_path.empty(), "New DB path cannot be empty.", + kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, + kRocProfVisDmResultInvalidParameter); + return ExportTableCSV(query.c_str(), file_path.c_str(), future); + }))); + } catch(std::exception ex) + { + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); + } + return kRocProfVisDmResultSuccess; +} + +rocprofvis_dm_result_t +SystemDatabase::SaveTrimmedDataAsync(rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_string_t new_db_path, + rocprofvis_db_future_t object) +{ + Future* future = (Future*) object; + ROCPROFVIS_ASSERT_MSG_RETURN(!new_db_path.empty(), "New DB path cannot be empty.", + kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, + kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, + kRocProfVisDmResultResourceBusy); + try + { + future->SetWorker(std::move(std::thread([this, start, end, new_db_path, future] { + ROCPROFVIS_ASSERT_MSG_RETURN(!new_db_path.empty(), "New DB path cannot be empty.", + kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, + kRocProfVisDmResultInvalidParameter); + + + //check if a db file exists and if it does delete it (we will overwrite it) + std::ifstream file(new_db_path); + if(file.good()) + { + file.close(); + int remove_result = std::remove(new_db_path.c_str()); + if(remove_result != 0) + { + spdlog::error("Failed to overwrite existing file: {}, code: {}", new_db_path, + remove_result); + + ShowProgress(0, "Failed to trim track! Could not overwrite existing file.", kRPVDbError, future); + future->SetPromise(kRocProfVisDmResultDbAccessFailed); + return kRocProfVisDmResultDbAccessFailed; + } + } + + return SaveTrimmedData(start, end, new_db_path.c_str(), future); + }))); + } catch(std::exception ex) + { + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); + } + return kRocProfVisDmResultSuccess; +} + +rocprofvis_dm_result_t SystemDatabase::ExecuteQueryAsync( + rocprofvis_dm_charptr_t query, + rocprofvis_dm_charptr_t description, + rocprofvis_db_future_t object, + rocprofvis_dm_table_id_t* id) +{ + Future* future = (Future*) object; + ROCPROFVIS_ASSERT_MSG_RETURN(future, ERROR_FUTURE_CANNOT_BE_NULL, kRocProfVisDmResultInvalidParameter); + ROCPROFVIS_ASSERT_MSG_RETURN(!future->IsWorking(), ERROR_FUTURE_CANNOT_BE_USED, kRocProfVisDmResultResourceBusy); + *id = std::hash{}(query); + rocprofvis_dm_result_t result = BindObject()->FuncCheckTableExists(BindObject()->trace_object, *id); + if(result != kRocProfVisDmResultNotLoaded) + { + return future->SetPromise(result); + } + try { + future->SetWorker(std::move(std::thread([this, query, description, future] { + return ExecuteQuery(query,description,future); + }))); + } + catch (std::exception ex) + { + ROCPROFVIS_ASSERT_ALWAYS_MSG_RETURN(ex.what(), kRocProfVisDmResultUnknownError); + } + return kRocProfVisDmResultSuccess; +} + + +const char* SystemDatabase::SubProcessNameSuffixFor(rocprofvis_dm_track_category_t category){ + switch(category){ + case kRocProfVisDmPmcTrack: + case kRocProfVisDmKernelDispatchTrack: + case kRocProfVisDmMemoryAllocationTrack: + case kRocProfVisDmMemoryCopyTrack: + return "Queue "; + case kRocProfVisDmRegionTrack: + case kRocProfVisDmRegionMainTrack: + case kRocProfVisDmRegionSampleTrack: + return "Thread "; + } + return ""; +} + + +void SystemDatabase::CreateTracksOrderRanking() { + + + using Track = rocprofvis_dm_track_params_t; + + std::map, std::vector> partitions; + + for (auto& track_prop : m_track_properties) { + Track* t = track_prop.get(); + partitions[t->load_id].push_back(t); + } + + uint32_t order_base = 0; + + for (auto& [load_set, partition_tracks] : partitions) { + + + std::map> groups; + + for (auto* t : partition_tracks) { + DbInstance* db_instance = (DbInstance*)t->track_indentifiers.db_instance; + uint32_t file_index = db_instance ? db_instance->FileIndex() : 0; + groups[file_index].push_back(t); + } + + for (auto& [db, vec] : groups) { + std::sort(vec.begin(), vec.end(),[]( + const Track* a, const Track* b) { + return a->track_indentifiers.track_id < + b->track_indentifiers.track_id; + }); + } + + size_t max_len = 0; + for (const auto& [_, vec] : groups) { + max_len = std::max(max_len, vec.size()); + } + + uint32_t local_order = 0; + + for (size_t i = 0; i < max_len; ++i) { + for (auto& [db, vec] : groups) { + if (i < vec.size()) { + vec[i]->order_id = order_base + local_order; + ++local_order; + } + } + } + + // Advance the base past every track assigned in this partition so + // order_ids stay globally unique regardless of how many tracks a + // partition holds (a fixed stride per partition would collide once a + // partition exceeds that stride). + order_base += local_order; + } + +} + +rocprofvis_dm_size_t SystemDatabase::GetMemoryFootprint() +{ + rocprofvis_dm_size_t size = Database::GetMemoryFootprint(); + size+=NumTracks()*(sizeof(rocprofvis_dm_track_params_t)+sizeof(std::unique_ptr)); + size+=strlen(Path()); + return size; +} + + +} // namespace DataModel +} // namespace RocProfVis \ No newline at end of file diff --git a/src/model/src/database/rocprofvis_db_systems.h b/src/model/src/database/rocprofvis_db_systems.h new file mode 100644 index 000000000..7f1c00c9d --- /dev/null +++ b/src/model/src/database/rocprofvis_db_systems.h @@ -0,0 +1,384 @@ +// Copyright Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +#include "rocprofvis_db_future.h" +#include "rocprofvis_db_cache.h" +#include "rocprofvis_db_track.h" +#include "rocprofvis_db_version.h" +#include "rocprofvis_db.h" +#include "rocprofvis_db_packed_storage.h" +#include +#include +#include +#include +#include +#include +#include + +namespace RocProfVis +{ +namespace DataModel +{ + +typedef std::map> rocprofvis_event_data_category_map_t; +// type of map array for storing slice handlers for multi-track request +typedef std::unordered_map slice_array_t; + +class Database; + +// Helper class to lock processes in order of database instances +class OrderedMutex { +public: + void init(uint32_t num_instances) { for (uint32_t i = 0; i < num_instances; i++) { m_instances.insert(i); } } + + void lock(uint32_t id) { + std::unique_lock lock(m_lock); + m_cv.wait(lock, [&] { return id == *m_instances.begin(); }); + } + + void unlock(uint32_t id) { + { + std::lock_guard lock(m_lock); + m_instances.erase(id); + } + m_cv.notify_all(); + } + + +private: + std::set m_instances; + std::mutex m_lock; + std::condition_variable m_cv; +}; + +typedef enum class rocprofvis_db_string_type:uint32_t +{ + kRPVStringTypeNameOrCategory, + kRPVStringTypeKernelSymbol +} rocprofvis_db_string_type_t; + +typedef struct rocprofvis_db_string_id_t +{ + uint64_t m_string_id; + uint32_t m_guid_id; + rocprofvis_db_string_type_t m_string_type; + + bool operator==(const rocprofvis_db_string_id_t& other) const { + return m_string_id == other.m_string_id && m_guid_id == other.m_guid_id && m_string_type == other.m_string_type; + } +} rocprofvis_db_string_id_t; + +typedef struct rocprofvis_db_string_id_hash_t +{ + size_t operator()(const rocprofvis_db_string_id_t& s) const noexcept + { + size_t h1 = std::hash{}(s.m_string_id); + size_t h2 = std::hash{}(s.m_guid_id); + size_t h3 = std::hash{}(s.m_string_type); + + size_t seed = h1; + seed ^= h2 + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); + seed ^= h3 + 0x9e3779b97f4a7c15ULL + (seed << 6) + (seed >> 2); + return seed; + } +} rocprofvis_db_string_id_hash_t; + +class SystemDatabase : public Database +{ + public: + // Database constructor + // @param path - full path to database file + SystemDatabase( + rocprofvis_db_filename_t path): + Database(path), + m_track_lookup(this) { + }; + // Database destructor, must be defined as virtual to free resources of derived classes + virtual ~SystemDatabase(){}; + + // Get amount of memory used by database resource + // @return memory size + virtual rocprofvis_dm_size_t GetMemoryFootprint(void) override; + + // Method to cleanup database + // @return status of operation + rocprofvis_dm_result_t CleanupAsync(rocprofvis_db_future_t object, bool rebuild); + // Asynchronously read trace metadata from database + // @param object - future object providing asynchronous execution mechanism + // @return status of operation + rocprofvis_dm_result_t ReadTraceMetadataAsync( + rocprofvis_db_future_t object); + // Asynchronously read a time slice (records from specified number of tracks for specified time frame) from database + // @param start - start timestamp of time slice + // @param end - end timestamp of time slice + // @param num - number of tracks + // @param tracks - uint32_t array with track IDs + // @param object - future object providing asynchronous execution mechanism + // @return status of operation + rocprofvis_dm_result_t ReadTraceSliceAsync( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_hashed_timestamp_tag_t tag, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t tracks, + rocprofvis_db_future_t object); + + // Asynchronously read a PMC time slice (records from specified track for specified time frame) from database + // @param start - start timestamp of time slice + // @param end - end timestamp of time slice + // @param track - track ID + // @param left_neighbor - include the left neighbor of the time range + // @param right_neighbor - include the right neighbor of the time range + // @param object - future object providing asynchronous execution mechanism + // @return status of operation + rocprofvis_dm_result_t ReadTracePMCSliceAsync( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_hashed_timestamp_tag_t tag, + rocprofvis_db_track_selection_t track, + bool left_neighbor, + bool right_neighbor, + rocprofvis_db_future_t object); + + // Asynchronously read different types of event properties (flowtrace, stacktrace, extdata) for event ID + // @param type - event property type (flowtrace, stacktrace, extdata) + // @param event_id - 60-bit event id and 4-bit operation type + // @param object - future object providing asynchronous execution mechanism + // @return status of operation + rocprofvis_dm_result_t ReadEventPropertyAsync( + rocprofvis_dm_event_property_type_t type, + rocprofvis_dm_event_id_t event_id, + rocprofvis_db_future_t object); + // Asynchronously run any table query and store results into Table object + // @param query - database query + // @param description - database description + // @param object - future object providing asynchronous execution mechanism + // @param id new id is assigned to the table and returned using this reference pointer + // @return status of operation + rocprofvis_dm_result_t ExecuteQueryAsync( + rocprofvis_dm_charptr_t query, + rocprofvis_dm_charptr_t description, + rocprofvis_db_future_t object, + rocprofvis_dm_table_id_t* id); + + virtual rocprofvis_dm_result_t BuildTableQuery( + rocprofvis_dm_table_use_case_enum_t use_case, + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t tracks, + rocprofvis_dm_processor_identifiers_ptr processor, + rocprofvis_dm_charptr_t filter, + rocprofvis_dm_charptr_t group, + rocprofvis_dm_charptr_t group_cols, + rocprofvis_dm_charptr_t sort_column, + rocprofvis_dm_sort_order_t sort_order, + uint64_t max_count, + uint64_t offset, + bool count_only, + rocprofvis_dm_string_t& query) = 0; + + virtual rocprofvis_dm_result_t BuildEventSearchQuery( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t ops, + rocprofvis_dm_processor_identifiers_ptr processor, + rocprofvis_dm_num_string_table_filters_t num_string_table_filters, + rocprofvis_dm_string_table_filters_t string_table_filters, + bool include_substring, + bool include_category, + bool partial_matching, + rocprofvis_dm_charptr_t sort_column, + rocprofvis_dm_sort_order_t sort_order, + uint64_t max_count, + uint64_t offset, + bool count_only, + rocprofvis_dm_string_t& query) = 0; + + + // Asynchronously writes the results of a table query to .CSV + // @param query - database query + // @param file_path - .CSV output path + // @param object - future object providing asynchronous execution mechanism + // @return status of operation + rocprofvis_dm_result_t ExportTableCSVAsync(rocprofvis_dm_string_t query, + rocprofvis_dm_string_t file_path, + rocprofvis_db_future_t object); + + virtual rocprofvis_dm_result_t SaveTrimmedData(rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_charptr_t new_db_path, + Future* future) = 0; + + rocprofvis_dm_result_t SaveTrimmedDataAsync(rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_string_t new_db_path, + rocprofvis_db_future_t object); + + + virtual rocprofvis_dm_result_t GetEventTablesAsync( + std::vector>& queries, + Future* parent, + rocprofvis_dm_handle_t handle, + RpvCallback callback) { + return kRocProfVisDmResultNotSupported; + }; + + + + private: + /************************static methods to be used as a parameter to std::thread**********************/ + + //static method to read metadata. Required to launch a unique thread for asynchronous metadata read + // @param db - pointer to database object + // @param object - future object providing asynchronous execution mechanism + static rocprofvis_dm_result_t ReadTraceMetadataStatic( + Database* db, + Future* object); + + + + + static rocprofvis_dm_result_t CleanupStatic(Database* db, Future* future, bool rebuild); + + /************************pure virtual worker methods to be implemented in derived classes**********************/ + + // worker method to read time slice + // @param start - start timestamp of time slice + // @param end - end timestamp of time slice + // @param num - number of tracks + // @param tracks - uint32_t array with track IDs + // @param object - future object providing asynchronous execution mechanism + // @return status of operation + virtual rocprofvis_dm_result_t ReadTraceSlice( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_hashed_timestamp_tag_t tag, + rocprofvis_db_num_of_tracks_t num, + rocprofvis_db_track_selection_t tracks, + Future* object) = 0; + + virtual rocprofvis_dm_result_t ReadTracePMCSlice( + rocprofvis_dm_timestamp_t start, + rocprofvis_dm_timestamp_t end, + rocprofvis_dm_hashed_timestamp_tag_t tag, + rocprofvis_db_track_selection_t track, + bool left_neighbor, + bool right_neighbor, + Future* object) = 0; + + // worker method to read flow trace info, called from ReadEventPropertyStatic + // @param event_id - 60-bit event id and 4-bit operation type + // @param object - future object providing asynchronous execution mechanism + // @return status of operation + virtual rocprofvis_dm_result_t ReadFlowTraceInfo( + rocprofvis_dm_event_id_t event_id, + Future* object) = 0; + // worker method to read stack trace info, called from ReadEventPropertyStatic + // @param event_id - 60-bit event id and 4-bit operation type + // @param object - future object providing asynchronous execution mechanism + // @return status of operation + virtual rocprofvis_dm_result_t ReadStackTraceInfo( + rocprofvis_dm_event_id_t event_id, + Future* object) = 0; + // worker method to read extended info, called from ReadEventPropertyStatic + // @param event_id - 60-bit event id and 4-bit operation type + // @param object - future object providing asynchronous execution mechanism + // @return status of operation + virtual rocprofvis_dm_result_t ReadExtEventInfo( + rocprofvis_dm_event_id_t event_id, + Future* object) = 0; + // worker method to execute any database query + // @param query - database query + // @param description - database description + // @param object - future object providing asynchronous execution mechanism + // @return status of operation + virtual rocprofvis_dm_result_t ExecuteQuery( + rocprofvis_dm_charptr_t query, + rocprofvis_dm_charptr_t description, + Future* object) = 0; + + // method to export the results of a table query to .CSV + // @param query - database query + // @param file_path - .CSV output path + // @return status of operation + virtual rocprofvis_dm_result_t ExportTableCSV( rocprofvis_dm_charptr_t query, + rocprofvis_dm_charptr_t file_path, + Future* future) = 0; + + virtual rocprofvis_dm_result_t Cleanup(Future* future, bool rebuild) { (void) future; (void) rebuild; return kRocProfVisDmResultSuccess; }; + + private: + // vector array of track parameters. Used as a reference for data model Track objects and for Database component to generate proper database queries + std::vector> m_track_properties; + TrackLookup m_track_lookup; + StringTable m_string_table; + + + protected: + // ---------------------------------------------Getters--------------------------------------- + // returns pointer to last registered Track properties structure + rocprofvis_dm_track_params_t* TrackPropertiesLast() { return m_track_properties.back().get(); } + // returns track properties begin iterator + rocprofvis_dm_track_params_it TrackPropertiesBegin() { return m_track_properties.begin(); } + // returns track properties end iterator + rocprofvis_dm_track_params_it TrackPropertiesEnd() { return m_track_properties.end(); } + + TrackLookup* TrackTracker() { return& m_track_lookup; } + // return current number of tracks + rocprofvis_dm_size_t NumTracks() { return m_track_properties.size(); } + // returns pointer to track properties structure. Takes index of track as a parameter + rocprofvis_dm_track_params_t* TrackPropertiesAt(rocprofvis_dm_index_t index) { return m_track_properties[index].get(); } + // validated track index + bool IsTrackIndexValid(rocprofvis_dm_index_t index) { return index < m_track_properties.size(); } + StringTable& StringTableReference() { return m_string_table; }; + + // ---------------------------------------------Helpers--------------------------------------- + // register new track + // @param props - track properties structure + // @return status of operation + rocprofvis_dm_result_t AddTrackProperties( + rocprofvis_dm_track_params_t& props); + // remap string IDs in new event record structure + // @param record - event data record + // @return status of operation + virtual rocprofvis_dm_result_t RemapStringIds( + rocprofvis_db_record_data_t & record) { (void) record; return kRocProfVisDmResultSuccess;}; + virtual rocprofvis_dm_result_t RemapStringIds( + rocprofvis_db_flow_data_t & record) { (void) record; return kRocProfVisDmResultSuccess;}; + virtual rocprofvis_dm_result_t StringIndexToId( + rocprofvis_dm_index_t index, std::vector& id) { (void) index; (void) id; return kRocProfVisDmResultSuccess;}; + + virtual rocprofvis_dm_result_t RemapStringId(uint64_t id, rocprofvis_db_string_type_t type, uint32_t node, uint64_t & result) = 0; + + // return suffix to sub-process name for provided track category ('TID', 'Queue') + // @param category - track category + // @return track sub-process name suffix ('TID', 'Queue') + static const char* SubProcessNameSuffixFor(rocprofvis_dm_track_category_t category); + + // create tracks ranking so they can be sorted accordingly in UI + void CreateTracksOrderRanking(); + + virtual void GetTrackIdentifierIndices(int column_index, char** azColName, rocprofvis_db_track_identifier_index_t& track_ids_indices) = 0; + virtual bool FindTrack(rocprofvis_dm_track_category_t category, uint64_t id_process, uint64_t id_subprocess, uint32_t db_instance, uint32_t& out_track) = 0; + + //--------------------------------------Table accessors----------------------------------------------------------------- + virtual std::string TableColumnText(void* func, void* handle, char** azColName, int index) = 0; + virtual int TableColumnInt(void* func, void* handle, char** azColName, int index) = 0; + virtual int64_t TableColumnInt64(void* func, void* handle, char** azColName, int index) = 0; + virtual double TableColumnDouble(void* func, void* handle, char** azColName, int index) = 0; + + public: + // declare DatabaseCache as friend class, for having access to protected members + friend class DatabaseCache; + friend class TableProcessor; + friend class TrackLookup; + friend class PackedTable; + friend class SqliteDatabase; +}; + +} // namespace DataModel +} // namespace RocProfVis diff --git a/src/model/src/database/rocprofvis_db_table_processor.cpp b/src/model/src/database/rocprofvis_db_table_processor.cpp index c355e6fca..68c974df1 100644 --- a/src/model/src/database/rocprofvis_db_table_processor.cpp +++ b/src/model/src/database/rocprofvis_db_table_processor.cpp @@ -100,7 +100,7 @@ namespace DataModel if (new_queries.size()) { - result = m_db->ExecuteQueriesAsync(new_queries, future, (rocprofvis_dm_handle_t)this, &CallbackRunCompoundQuery); + result = m_db->GetEventTablesAsync(new_queries, future, (rocprofvis_dm_handle_t)this, (RpvCallback) & CallbackRunCompoundQuery); if (kRocProfVisDmResultSuccess == result) { try { @@ -161,6 +161,119 @@ namespace DataModel return query_without_commands; } + rocprofvis_dm_result_t + TableProcessor::BuildTableSemanticSubQuery( + rocprofvis_dm_table_use_case_enum_t use_case, + rocprofvis_dm_charptr_t filter, + rocprofvis_dm_charptr_t group, + rocprofvis_dm_charptr_t group_cols, + rocprofvis_dm_charptr_t sort_column, + rocprofvis_dm_sort_order_t sort_order, + uint64_t max_count, + uint64_t offset, + bool count_only, + bool sample_query, + rocprofvis_dm_string_t& query) + { + query += "-- CMD: TYPE "; + switch(use_case) + { + case kRPVDMTableUseCaseEventTrackTable: + { + query += std::to_string(kRPVTableDataTypeEvent); + break; + } + case kRPVDMTableUseCaseSampleTrackTable: + { + query += std::to_string(kRPVTableDataTypeSample); + break; + } + case kRPVDMTableUseCaseEventSearch: + { + query += std::to_string(kRPVTableDataTypeSearch); + break; + } + default: + { + return kRocProfVisDmResultInvalidParameter; + break; + } + } + query += "\n"; + + if(group && strlen(group)) + { + query += "-- CMD: GROUP "; + if(group_cols && strlen(group_cols)) + { + if(!FilterExpression::StartsWithSubstring(group, group_cols)) + { + query += group_cols; + query += ", "; + } + query += group; + } + else + { + query += group; + if(sample_query) + { + query += ", COUNT(*) as count, AVG(value) as avg_value, MIN(value) as " + "min_value, MAX(value) as max_value"; + } + else + { + query += ", COUNT(*) as num_invocations, AVG(duration) as avg_duration, " + "MIN(duration) as min_duration, MAX(duration) as max_duration"; + } + } + query += "\n"; + } + + if(filter && strlen(filter)) + { + query += "-- CMD: FILTER "; + query += filter; + query += "\n"; + } + + if(sort_column && strlen(sort_column)) + { + query += "-- CMD: SORT"; + if(sort_order == kRPVDMSortOrderAsc) + { + query += " ASC "; + } + else + { + query += " DESC "; + } + query += sort_column; + query += "\n"; + } + if(count_only) + { + query += "-- CMD: COUNT"; + query += "\n"; + } + else + { + if(max_count) + { + query += "-- CMD: LIMIT "; + query += std::to_string(max_count); + query += "\n"; + } + if(offset) + { + query += "-- CMD: OFFSET "; + query += std::to_string(offset); + query += "\n"; + } + } + return kRocProfVisDmResultSuccess; + } + bool TableProcessor::IsCompoundQuery(const char* query, std::unordered_map>& queries, std::set& tracks, std::vector& commands) { std::istringstream stream(query); @@ -250,7 +363,7 @@ namespace DataModel { if (op == kRocProfVisDmOperationDispatch || op == kRocProfVisDmOperationMemoryAllocate || op == kRocProfVisDmOperationMemoryCopy) { - Numeric val = m_merged_table.GetMergeTableValue(op, row_index, column_index, m_db); + Numeric val = m_merged_table.GetMergeTableValue(op, row_index, column_index); result = m_db->BindObject()->FuncAddTableRowCell(row, std::to_string(val.data.u64).c_str()); if (result != kRocProfVisDmResultSuccess) break; @@ -284,7 +397,7 @@ namespace DataModel else if (columns[column_index].m_schema_index[op] == Builder::SCHEMA_INDEX_COUNTER_VALUE) { - Numeric val = m_merged_table.GetMergeTableValue(op, row_index, column_index, m_db); + Numeric val = m_merged_table.GetMergeTableValue(op, row_index, column_index); std::string output = std::to_string(val.data.d); if (to_file) { @@ -304,7 +417,7 @@ namespace DataModel } else { - Numeric val = m_merged_table.GetMergeTableValue(op, row_index, column_index, m_db); + Numeric val = m_merged_table.GetMergeTableValue(op, row_index, column_index); bool numeric_string = false; const char* str = columns[column_index].m_type[op] == ColumnType::Null ? "" : PackedTable::ConvertSqlStringReference(m_db, columns[column_index].m_schema_index[op], val.data.u64, db_instance->GuidIndex(), numeric_string); @@ -481,12 +594,12 @@ namespace DataModel else if (columns[column_index].m_schema_index[op] == Builder::SCHEMA_INDEX_COUNTER_VALUE) { - Numeric val = m_merged_table.GetMergeTableValue(op, row_index, column_index, m_db); + Numeric val = m_merged_table.GetMergeTableValue(op, row_index, column_index); map[columns[column_index].m_name] = val.data.d; } else { - Numeric val = m_merged_table.GetMergeTableValue(op, row_index, column_index, m_db); + Numeric val = m_merged_table.GetMergeTableValue(op, row_index, column_index); bool numeric_string = false; const char* str = columns[column_index].m_type[op] == ColumnType::Null ? "" : PackedTable::ConvertSqlStringReference(m_db, columns[column_index].m_schema_index[op], val.data.u64, db_instance->GuidIndex(), numeric_string); @@ -695,7 +808,7 @@ namespace DataModel std::string sort_column = ParseSortCommand(cmd_it->parameter, sort_order); if (sort_order != m_sort_order || sort_column != m_sort_column) { - m_merged_table.SortAggregationByColumn(m_db, sort_column, sort_order); + m_merged_table.SortAggregationByColumn( sort_column, sort_order); m_sort_order = sort_order; m_sort_column = sort_column; } @@ -813,11 +926,11 @@ namespace DataModel return kRocProfVisDmResultSuccess; } - int TableProcessor::CallbackRunCompoundQuery(void* data, int argc, sqlite3_stmt* stmt, char** azColName) { + int TableProcessor::CallbackRunCompoundQuery(void* data, int argc, void* stmt, char** azColName) { ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); - QueryManager* db = (QueryManager*)callback_params->db; + SystemDatabase* db = (SystemDatabase*)callback_params->db; TableProcessor* table_processor = (TableProcessor*)callback_params->handle; void* func = (void*)&CallbackRunCompoundQuery; if (callback_params->future->Interrupted()) @@ -865,25 +978,25 @@ namespace DataModel if (columns[column_index].m_schema_index == Builder::SCHEMA_INDEX_MEM_TYPE) { - uint64_t value = Builder::TypeEnumToInt(db->Sqlite3ColumnText(func, stmt, azColName, - columns[column_index].m_orig_index), Builder::mem_alloc_types); + uint64_t value = Builder::TypeEnumToInt(db->TableColumnText(func, stmt, azColName, + columns[column_index].m_orig_index).c_str(), Builder::mem_alloc_types); table_processor->m_tables[callback_params->track_id]->PlaceValue(column_index, value); } else if (columns[column_index].m_schema_index == Builder::SCHEMA_INDEX_LEVEL) { - uint64_t value = Builder::TypeEnumToInt(db->Sqlite3ColumnText(func, stmt, azColName, - columns[column_index].m_orig_index), Builder::mem_alloc_levels); + uint64_t value = Builder::TypeEnumToInt(db->TableColumnText(func, stmt, azColName, + columns[column_index].m_orig_index).c_str(), Builder::mem_alloc_levels); table_processor->m_tables[callback_params->track_id]->PlaceValue(column_index, value); } else if (columns[column_index].m_schema_index == Builder::SCHEMA_INDEX_COUNTER_VALUE) { - double value = db->Sqlite3ColumnDouble(func, stmt, azColName, columns[column_index].m_orig_index); + double value = db->TableColumnDouble(func, stmt, azColName, columns[column_index].m_orig_index); table_processor->m_tables[callback_params->track_id]->PlaceValue(column_index, value); } else if (columns[column_index].m_schema_index == Builder::SCHEMA_INDEX_OPERATION) { - op = db->Sqlite3ColumnInt(func, stmt, azColName, columns[column_index].m_orig_index); + op = db->TableColumnInt(func, stmt, azColName, columns[column_index].m_orig_index); table_processor->m_tables[callback_params->track_id]->PlaceValue(column_index, op); } else if (columns[column_index].m_schema_index == Builder::SCHEMA_INDEX_EVENT_ID) { - uint64_t id = db->Sqlite3ColumnInt64(func, stmt, azColName, columns[column_index].m_orig_index); + uint64_t id = db->TableColumnInt64(func, stmt, azColName, columns[column_index].m_orig_index); rocprofvis_dm_event_id_t value; value.bitfield.event_id = id; value.bitfield.event_node = callback_params->db_instance->GuidIndex(); @@ -893,8 +1006,8 @@ namespace DataModel columns[column_index].m_schema_index == Builder::SCHEMA_INDEX_CATEGORY_PERFETTO || columns[column_index].m_schema_index == Builder::SCHEMA_INDEX_EVENT_NAME_PERFETTO) { - uint64_t value = db->StringTableReference().ToInt(db->Sqlite3ColumnText(func, stmt, azColName, - columns[column_index].m_orig_index)); + uint64_t value = db->StringTableReference().ToInt(db->TableColumnText(func, stmt, azColName, + columns[column_index].m_orig_index).c_str()); table_processor->m_tables[callback_params->track_id]->PlaceValue(column_index, value); } else if (columns[column_index].m_schema_index == Builder::SCHEMA_INDEX_NODE_ID) @@ -904,12 +1017,12 @@ namespace DataModel } else if (columns[column_index].m_schema_index == Builder::SCHEMA_INDEX_END || columns[column_index].m_schema_index == Builder::SCHEMA_INDEX_START) { - uint64_t value = db->Sqlite3ColumnInt64(func, stmt, azColName, columns[column_index].m_orig_index); + uint64_t value = db->TableColumnInt64(func, stmt, azColName, columns[column_index].m_orig_index); table_processor->m_tables[callback_params->track_id]->PlaceValue(column_index, value - db->TraceProperties()->db_inst_start_time[callback_params->db_instance->GuidIndex()]); } else { - uint64_t value = db->Sqlite3ColumnInt64(func, stmt, azColName, columns[column_index].m_orig_index); + uint64_t value = db->TableColumnInt64(func, stmt, azColName, columns[column_index].m_orig_index); table_processor->m_tables[callback_params->track_id]->PlaceValue(column_index, value); } @@ -917,8 +1030,8 @@ namespace DataModel uint32_t track_id; if (!db->FindTrack(db->TrackTracker()->SearchCategoryMaskLookup((rocprofvis_dm_event_operation_t)op), - db->Sqlite3ColumnInt64(func, stmt, azColName, table_processor->m_tables[callback_params->track_id]->track_ids_indices.process_index), - db->Sqlite3ColumnInt64(func, stmt, azColName, table_processor->m_tables[callback_params->track_id]->track_ids_indices.sub_process_index), + db->TableColumnInt64(func, stmt, azColName, table_processor->m_tables[callback_params->track_id]->track_ids_indices.process_index), + db->TableColumnInt64(func, stmt, azColName, table_processor->m_tables[callback_params->track_id]->track_ids_indices.sub_process_index), callback_params->db_instance->GuidIndex(), track_id)) { @@ -932,8 +1045,8 @@ namespace DataModel op == kRocProfVisDmOperationLaunchSample || table_processor->m_tables[callback_params->track_id]->track_ids_indices.stream_index == -1 || !db->FindTrack(kRocProfVisDmStreamTrack, - db->Sqlite3ColumnInt(func, stmt, azColName, table_processor->m_tables[callback_params->track_id]->track_ids_indices.pid_index), - db->Sqlite3ColumnInt(func, stmt, azColName, table_processor->m_tables[callback_params->track_id]->track_ids_indices.stream_index), + db->TableColumnInt(func, stmt, azColName, table_processor->m_tables[callback_params->track_id]->track_ids_indices.pid_index), + db->TableColumnInt(func, stmt, azColName, table_processor->m_tables[callback_params->track_id]->track_ids_indices.stream_index), callback_params->db_instance->GuidIndex(), track_id)) { diff --git a/src/model/src/database/rocprofvis_db_table_processor.h b/src/model/src/database/rocprofvis_db_table_processor.h index b9e78d4c6..6e1f1eaa6 100644 --- a/src/model/src/database/rocprofvis_db_table_processor.h +++ b/src/model/src/database/rocprofvis_db_table_processor.h @@ -3,14 +3,34 @@ #pragma once -#include "rocprofvis_db_sqlite.h" +#include "rocprofvis_db_systems.h" +#include "json.h" namespace RocProfVis { namespace DataModel { - class QueryManager; + class Database; + + typedef enum rocprofvis_dm_track_search_id_t + { + kRPVTrackSearchIdThreads, + kRPVTrackSearchIdThreadSamples, + kRPVTrackSearchIdDispatches, + kRPVTrackSearchIdMemAllocs, + kRPVTrackSearchIdMemCopies, + kRPVTrackSearchIdCounters, + kRPVTrackSearchIdStreams, + kRPVTrackSearchIdUnknown, + } rocprofvis_dm_track_search_id_t; + + typedef enum rocprofvis_db_compound_table_type { + kRPVTableDataTypeEvent, + kRPVTableDataTypeSample, + kRPVTableDataTypeSearch, + kRPVTableDataTypesNum + } rocprofvis_db_compound_table_type; typedef struct rocprofvis_db_compound_query_command { std::string name; @@ -22,6 +42,9 @@ namespace DataModel uint32_t guid_id; } rocprofvis_db_compound_query_info; + + typedef std::variant rocprofvis_table_cell_t; + class RestartableTimer { public: RestartableTimer() : m_stop(false), m_paused(false) {} @@ -111,21 +134,130 @@ namespace DataModel bool m_pending = false; }; + class ParamSerializer { + jt::Json obj_; + public: + ParamSerializer() { obj_.setObject(); } + + template + void set(const std::string& tag, const T& val) { + obj_.getObject()[tag] = jt::Json(val); + } + + // Set an array of values + template + void setArray(const std::string& tag, const std::vector& values) { + jt::Json arr; + arr.setArray(); + for (const auto& v : values) + arr.getArray().push_back(jt::Json(v)); + obj_.getObject()[tag] = std::move(arr); + } + + std::string toString() const { return obj_.toString(); } + }; + + + class ParamDeserializer { + jt::Json json_; + public: + explicit ParamDeserializer(const std::string& text) { + auto [status, json] = jt::Json::parse(text); + if (status != jt::Json::success) + throw std::runtime_error( + std::string("JSON parse error: ") + jt::Json::StatusToString(status)); + if (!json.isObject()) + throw std::runtime_error("Expected JSON object"); + json_ = std::move(json); + } + + template + bool get(const std::string& tag, T& val) { + if (!json_.contains(tag)) return false; + jt::Json& j = json_[tag]; + if constexpr (std::is_same_v) { + if (!j.isBool()) return false; + val = j.getBool(); + } else if constexpr (std::is_same_v) { + if (!j.isNumber()) return false; + val = static_cast(j.getNumber()); + } else if constexpr (std::is_same_v) { + if (!j.isNumber()) return false; + val = j.getNumber(); + } else if constexpr (std::is_integral_v) { + if (!j.isLong()) return false; + val = static_cast(j.getLong()); + } else if constexpr (std::is_same_v) { + if (!j.isString()) return false; + val = j.getString(); + } else { + return false; + } + return true; + } + + // Get an array of values + template + bool getArray(const std::string& tag, std::vector& values) { + if (!json_.contains(tag)) return false; + jt::Json& j = json_[tag]; + if (!j.isArray()) return false; + + values.clear(); + for (auto& elem : j.getArray()) { + T val; + if constexpr (std::is_same_v) { + if (!elem.isBool()) return false; + val = elem.getBool(); + } else if constexpr (std::is_same_v) { + if (!elem.isNumber()) return false; + val = static_cast(elem.getNumber()); + } else if constexpr (std::is_same_v) { + if (!elem.isNumber()) return false; + val = elem.getNumber(); + } else if constexpr (std::is_integral_v) { + if (!elem.isLong()) return false; + val = static_cast(elem.getLong()); + } else if constexpr (std::is_same_v) { + if (!elem.isString()) return false; + val = elem.getString(); + } else { + return false; + } + values.push_back(val); + } + return true; + } + }; + class TableProcessor { public: - TableProcessor(QueryManager* db) : m_db(db) { + TableProcessor(SystemDatabase* db) : m_db(db) { m_timer.setAction([this] { std::lock_guard lock(m_lock); - this->m_merged_table.Clear(); + this->m_merged_table.Clear(); this->m_tracks.clear(); }); }; + static rocprofvis_dm_result_t + BuildTableSemanticSubQuery( + rocprofvis_dm_table_use_case_enum_t use_case, + rocprofvis_dm_charptr_t filter, + rocprofvis_dm_charptr_t group, + rocprofvis_dm_charptr_t group_cols, + rocprofvis_dm_charptr_t sort_column, + rocprofvis_dm_sort_order_t sort_order, + uint64_t max_count, + uint64_t offset, + bool count_only, + bool sample_query, + rocprofvis_dm_string_t& query); static bool IsCompoundQuery(const char* query, std::unordered_map>& queries, std::set& tracks, std::vector& commands); static std::string QueryWithoutCommands(const char* query); // Unused @@ -139,10 +271,10 @@ namespace DataModel void SaveCurrentQuery(std::unordered_map>& queries) { m_current_queries = queries; }; bool IsCurrentQuery(std::unordered_map>& queries); rocprofvis_dm_result_t ExportToCSV(rocprofvis_dm_charptr_t file_path); + static int CallbackRunCompoundQuery(void* data, int argc, void* stmt, char** azColName); private: rocprofvis_dm_result_t ProcessCompoundQuery(rocprofvis_dm_table_t table, std::vector& commands, bool updated); - static int CallbackRunCompoundQuery(void* data, int argc, sqlite3_stmt* stmt, char** azColName); static std::string Trim(const std::string& str); rocprofvis_dm_result_t AddTableCells(bool to_file, rocprofvis_dm_handle_t handle, uint32_t row_index); rocprofvis_dm_result_t AddTableColumns(bool to_file, rocprofvis_dm_handle_t handle); @@ -150,7 +282,7 @@ namespace DataModel rocprofvis_dm_result_t AddAggregatedCells(bool to_file, rocprofvis_dm_handle_t handle, uint32_t row_index); private: - QueryManager* m_db; + SystemDatabase* m_db; std::vector> m_tables; std::set m_tracks; PackedTable m_merged_table; diff --git a/src/model/src/database/rocprofvis_db_trace_processor.cpp b/src/model/src/database/rocprofvis_db_trace_processor.cpp index 56774771a..415531fc6 100644 --- a/src/model/src/database/rocprofvis_db_trace_processor.cpp +++ b/src/model/src/database/rocprofvis_db_trace_processor.cpp @@ -402,7 +402,7 @@ namespace RocProfVis int GoogleTraceProcessor::CallbackCacheTable(void *data, int argc, sqlite3_stmt* stmt, char **azColName){ ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackCacheTable; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; GoogleTraceProcessor* db = (GoogleTraceProcessor*)callback_params->db; DatabaseCache * ref_tables = (DatabaseCache *)callback_params->handle; std::lock_guard lock(db->m_lock); @@ -430,7 +430,7 @@ namespace RocProfVis int GoogleTraceProcessor::CallbackAddTrack(void* data, int argc, sqlite3_stmt* stmt, char** azColName) { ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackAddTrack; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; GoogleTraceProcessor* db = (GoogleTraceProcessor*)callback_params->db; uint32_t db_index = callback_params->db_instance->GuidIndex(); @@ -771,8 +771,8 @@ namespace RocProfVis ROCPROFVIS_ASSERT_MSG_RETURN(argc == 5, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void *func = (void*)&CallbackGetTrackProperties; - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); uint32_t db_instance = callback_params->db_instance->GuidIndex(); GoogleTraceProcessor* db = (GoogleTraceProcessor*) callback_params->db; @@ -794,7 +794,7 @@ namespace RocProfVis int GoogleTraceProcessor::CallBackAddString(void *data, int argc, sqlite3_stmt* stmt, char **azColName){ ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallBackAddString; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); GoogleTraceProcessor* db = (GoogleTraceProcessor*)callback_params->db; if(callback_params->future->Interrupted()) return SQLITE_ABORT; @@ -813,7 +813,7 @@ namespace RocProfVis ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void *func = (void*)&CallbackAddAnyRecord; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); uint32_t db_instance = callback_params->db_instance->GuidIndex(); GoogleTraceProcessor* db = (GoogleTraceProcessor*)callback_params->db; @@ -1019,7 +1019,7 @@ namespace RocProfVis void GoogleTraceProcessor::GetTrackIdentifierIndices( int column_index, char** azColName, - rocprofvis_db_sqlite_track_identifier_index_t& track_ids_indices) + rocprofvis_db_track_identifier_index_t& track_ids_indices) { std::string column_name = azColName[column_index]; @@ -1047,7 +1047,7 @@ namespace RocProfVis int GoogleTraceProcessor::CallbackAddExtInfo(void* data, int argc, sqlite3_stmt* stmt, char** azColName) { ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); void* func = (void*)&CallbackAddExtInfo; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); GoogleTraceProcessor* db = (GoogleTraceProcessor*)callback_params->db; std::string value_type; @@ -1155,7 +1155,7 @@ namespace RocProfVis ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); ROCPROFVIS_ASSERT_MSG_RETURN(argc == 8, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); void* func = (void*)&CallbackAddFlowTrace; - rocprofvis_db_sqlite_callback_parameters* callback_params = (rocprofvis_db_sqlite_callback_parameters*)data; + rocprofvis_db_query_callback_parameters* callback_params = (rocprofvis_db_query_callback_parameters*)data; ROCPROFVIS_ASSERT_MSG_RETURN(callback_params->db_instance != nullptr, ERROR_NODE_KEY_CANNOT_BE_NULL, 1); uint32_t db_instance = callback_params->db_instance->GuidIndex(); GoogleTraceProcessor* db = (GoogleTraceProcessor*)callback_params->db; @@ -1174,7 +1174,7 @@ namespace RocProfVis record.time-=db->TraceProperties()->db_inst_start_time[db_instance]; record.end_time = db->Sqlite3ColumnInt64(func, stmt, azColName, 4); record.end_time-=db->TraceProperties()->db_inst_start_time[db_instance]; - std::string category = db->Sqlite3ColumnText(func, stmt, azColName, 5); + std::string category = db->Sqlite3ColumnText(func, stmt, azColName, 6); auto it = db->m_string_map.find(category ); uint32_t string_index = it != db->m_string_map.end() ? it->second : db->BindObject()->FuncAddString(db->BindObject()->trace_object, category.c_str()); record.category_id = string_index; @@ -1324,8 +1324,8 @@ namespace RocProfVis ROCPROFVIS_ASSERT_MSG_RETURN(data, ERROR_SQL_QUERY_PARAMETERS_CANNOT_BE_NULL, 1); ROCPROFVIS_ASSERT_MSG_RETURN(argc == 4, ERROR_DATABASE_QUERY_PARAMETERS_MISMATCH, 1); void* func = (void*)&CallbackAddStackTrace; - rocprofvis_db_sqlite_callback_parameters* callback_params = - (rocprofvis_db_sqlite_callback_parameters*) data; + rocprofvis_db_query_callback_parameters* callback_params = + (rocprofvis_db_query_callback_parameters*) data; GoogleTraceProcessor* db = (GoogleTraceProcessor*) callback_params->db; rocprofvis_db_stack_data_t record = {"","","",0,0}; static const char * empty_blob = "{}"; diff --git a/src/model/src/database/rocprofvis_db_trace_processor.h b/src/model/src/database/rocprofvis_db_trace_processor.h index 360eaf8c5..3a37c1cbd 100644 --- a/src/model/src/database/rocprofvis_db_trace_processor.h +++ b/src/model/src/database/rocprofvis_db_trace_processor.h @@ -145,7 +145,7 @@ class GoogleTraceProcessor : public QueryManager // ---------------------------------- Helpers ---------------------------------------- rocprofvis_dm_result_t RemapStringId(uint64_t id, rocprofvis_db_string_type_t type, uint32_t node, uint64_t& result) override { result = id; return kRocProfVisDmResultSuccess; }; rocprofvis_dm_track_category_t GetRegionTrackCategory() override { return kRocProfVisDmRegionTrack; } - void GetTrackIdentifierIndices(int column_index, char** azColName, rocprofvis_db_sqlite_track_identifier_index_t& track_ids_indices) override; + void GetTrackIdentifierIndices(int column_index, char** azColName, rocprofvis_db_track_identifier_index_t& track_ids_indices) override; bool FindTrack(rocprofvis_dm_track_category_t category, uint64_t id_process, uint64_t id_subprocess, uint32_t db_instance, uint32_t& out_track) override; private: diff --git a/src/model/src/database/rocprofvis_db_track.h b/src/model/src/database/rocprofvis_db_track.h index ea10024c1..a7440a5ff 100644 --- a/src/model/src/database/rocprofvis_db_track.h +++ b/src/model/src/database/rocprofvis_db_track.h @@ -23,14 +23,14 @@ typedef enum rocprofvis_db_async_tracks_flags_t typedef std::vector>::iterator rocprofvis_dm_track_params_it; -class Database; +class SystemDatabase; class TrackLookup { public: - TrackLookup(Database* db) : m_db(db) {}; + TrackLookup(SystemDatabase* db) : m_db(db) {}; struct TrackKey { @@ -94,7 +94,7 @@ class TrackLookup std::unordered_map, TrackKeyHash> m_track_lookup; StringTable m_string_lookup; - Database* m_db; + SystemDatabase* m_db; }; diff --git a/src/model/src/database/rocprofvis_db_version.cpp b/src/model/src/database/rocprofvis_db_version.cpp index 25863eb80..d586e5697 100644 --- a/src/model/src/database/rocprofvis_db_version.cpp +++ b/src/model/src/database/rocprofvis_db_version.cpp @@ -4,6 +4,7 @@ #include "rocprofvis_db_version.h" #include "rocprofvis_db_rocprof.h" #include "rocprofvis_db_rocpd.h" +#include namespace RocProfVis { @@ -302,5 +303,55 @@ namespace DataModel return result; } + void DatabaseVersion::SetVersion(const char* version) { + m_db_version = ConvertVersionStringToInt(version); + } + + std::vector + DatabaseVersion::ConvertVersionStringToInt(const char* version) + { + std::vector version_array; + std::stringstream ss(version); + std::string token; + while(std::getline(ss, token, '.')) + { + version_array.push_back(std::stoi(token)); + } + return version_array; + } + + bool DatabaseVersion::IsVersionEqual(const char* version) + { + std::vector db_version = ConvertVersionStringToInt(version); + + for (int i = 0; i < db_version.size(); i++) + { + uint32_t token = (m_db_version.size() > i) ? m_db_version[i] : 0; + if(db_version[i] != token) + { + return false; + } + } + return true; + } + + bool DatabaseVersion::IsVersionGreaterOrEqual(const char* version) + { + std::vector db_version = ConvertVersionStringToInt(version); + + for(int i = 0; i < db_version.size(); i++) + { + uint32_t token = (m_db_version.size() > i) ? m_db_version[i] : 0; + if(token > db_version[i]) + { + return true; + } else if (token < db_version[i]) + { + return false; + } + } + return true; + } + } // namespace DataModel } // namespace RocProfVis diff --git a/src/model/src/tests/rocprofvis_dm_profiler_hub_tests.cpp b/src/model/src/tests/rocprofvis_dm_profiler_hub_tests.cpp new file mode 100644 index 000000000..b0c4b01be --- /dev/null +++ b/src/model/src/tests/rocprofvis_dm_profiler_hub_tests.cpp @@ -0,0 +1,1329 @@ +// Copyright Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include "rocprofvis_c_interface.h" +#include "profiler_hub_lib_interface.h" +#include "rocprofvis_core.h" +#include "rocprofvis_db_future.h" +#include "rocprofvis_error_handling.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#define MULTI_LINE_LOG_START auto multi_line_log = fmt::memory_buffer() +#define MULTI_LINE_LOG(format, ...) \ + fmt::format_to(std::back_inserter(multi_line_log), format, __VA_ARGS__) +#define MULTI_LINE_LOG_ARGS "{:.{}}", multi_line_log.data(), multi_line_log.size() + +std::string g_input_file = "sample/trace_70b_1024_32.rpd"; +bool g_full_range = false; + +void +ReadSliceData(rocprofvis_dm_trace_t trace, uint32_t num_tracks, + rocprofvis_dm_timestamp_t start_time, rocprofvis_dm_timestamp_t end_time, + uint32_t accessInMillisec, rocprofvis_db_future_t object2wait); +void +DeleteSliceData(rocprofvis_dm_trace_t trace, rocprofvis_dm_timestamp_t start_time, + rocprofvis_dm_timestamp_t end_time, uint32_t accessInMillisec); + +int +main(int argc, char** argv) +{ + Catch::Session session; + + using namespace Catch::Clara; + auto cli = session.cli() | + Opt(g_input_file, "input_file")["--input_file"]("Path to input file") | + Opt(g_full_range, "full_range")["--full_range"]( + "Whether to load/query the full trace range or only a segment"); + + // Now pass the new composite back to Catch2 so it uses that + session.cli(cli); + + // Let Catch2 (using Clara) parse the command line + int returnCode = session.applyCommandLine(argc, argv); + if(returnCode != 0) // Indicates a command line error + return returnCode; + + std::string log_file = "Testing/Temporary/rocprofvis_dm_system_tests/" + + std::filesystem::path(g_input_file).filename().string() + + ".txt"; + rocprofvis_core_enable_log(log_file.c_str(), spdlog::level::trace); + + return session.run(); +} + +void +CheckMemoryFootprint(rocprofvis_dm_trace_t trace) +{ + uint64_t memory_used = 0; + if(trace) + { + memory_used += rocprofvis_dm_get_property_as_uint64( + trace, kRPVDMTraceMemoryFootprintUInt64, 0); + rocprofvis_dm_database_t db = + rocprofvis_dm_get_property_as_handle(trace, kRPVDMDatabaseHandle, 0); + if(db) + { + memory_used += rocprofvis_db_get_memory_footprint(db); + } + } + + spdlog::info("\x1b[0mTotal memory utilization so far = {}", memory_used); +} + +#define LIST_SIZE_LIMIT 2 +#define HEADER_LEN 100 + +void +PrintHeader(const char* fmt, ...) +{ + std::string header; + va_list argptr; + char buffer[256]; + va_start(argptr, fmt); + vsnprintf(buffer, sizeof(buffer), fmt, argptr); + va_end(argptr); + size_t text_len = strlen(buffer); + if(HEADER_LEN > text_len) header.assign((HEADER_LEN - text_len) / 2, '*'); + + spdlog::info("\x1b[0m{0}{1}{2}", header, std::string(buffer), header); +} + +void +progress_callback(rocprofvis_db_filename_t db_name, rocprofvis_db_future_id_t id, + rocprofvis_progress_callback_percent_t progress, rocprofvis_db_status_t status, + rocprofvis_db_status_message_t msg, void* user_data) +{ + (void) id; + (void) user_data; + const char* str = " ERROR "; + const char* color = "\x1b[31m"; + if(status == kRPVDbSuccess) + { + color = "\x1b[32m"; + str = " DONE "; + } + else if(status == kRPVDbBusy) + { + color = "\x1b[33m"; + str = " BUSY "; + } + + spdlog::info("{0}[{1}] {2} - {3} - {4}", color, db_name, progress, str, msg); +} + +void +GenerateRandomSlice(rocprofvis_dm_trace_t trace, rocprofvis_db_num_of_tracks_t& count, + rocprofvis_db_track_selection_t& tracks, + rocprofvis_dm_timestamp_t& start_time, + rocprofvis_dm_timestamp_t& end_time) +{ + static std::vector v; + v.clear(); + uint64_t num_tracks = + rocprofvis_dm_get_property_as_uint64(trace, kRPVDMNumberOfTracksUInt64, 0); + + int rand_num_tracks = 1; + for(int i = 0; i < rand_num_tracks; i++) + { + while(true) + { + uint32_t track_id1 = std::rand() % std::max(num_tracks, (uint64_t) 10); + if(std::find_if(v.begin(), v.end(), [track_id1](uint32_t track_id2) { + return track_id2 == track_id1; + }) == v.end()) + { + v.push_back(track_id1); + break; + } + } + } + count = (rocprofvis_db_num_of_tracks_t) rand_num_tracks; + + tracks = &v[0]; + rocprofvis_dm_timestamp_t tenth_time = (end_time - start_time) / 10; + int pie1 = std::rand() % 10; + int pie2 = std::rand() % 10; + if((pie1 + pie2) > 10) pie2 = 10 - pie1; + if(!g_full_range) + { + start_time = start_time + pie1 * tenth_time; + end_time = start_time + pie2 * tenth_time; + } + spdlog::info("Testing slice for {} ns and {} tracks", end_time - start_time, + (uint32_t) count); + spdlog::info("Track indexes: ["); + for(int i = 0; i < count; i++) + spdlog::info("{}", tracks[i]); + spdlog::info("]"); +} + +void +ReadSliceData(rocprofvis_dm_trace_t trace, uint32_t num_tracks, + rocprofvis_dm_timestamp_t start_time, rocprofvis_dm_timestamp_t end_time, + uint32_t accessInMillisec, rocprofvis_db_future_t object2wait) +{ + uint32_t num_rows = 0; + if(object2wait != nullptr) + { + for(uint32_t i = 0; i < accessInMillisec; i++) + { + num_rows = + ((RocProfVis::DataModel::Future*) object2wait)->GetProcessedRowsCount(); + if(num_rows > 1000) + { + break; + } + std::this_thread::sleep_for(std::chrono::microseconds(1000)); + } + spdlog::info(ANSI_COLOR_MAGENTA + "{0} rows processed or {1} milliseconds passed. Start " + "accessing data" ANSI_COLOR_RESET, + num_rows, accessInMillisec); + } + else + { + std::this_thread::sleep_for(std::chrono::milliseconds(accessInMillisec)); + spdlog::info(ANSI_COLOR_MAGENTA "{0} milliseconds passed. Start " + "accessing data" ANSI_COLOR_RESET, + accessInMillisec); + } + + for(uint32_t i = 0; i < num_tracks; i++) + { + rocprofvis_dm_track_t track = + rocprofvis_dm_get_property_as_handle(trace, kRPVDMTrackHandleIndexed, i); + REQUIRE(track != nullptr); + uint64_t hash_time = rocprofvis_dm_hash_combine_timestamp(start_time, end_time, kRocProfVisDmHashedTimestampTagTrackSlice); + rocprofvis_dm_slice_t slice = rocprofvis_dm_get_property_as_handle( + track, kRPVDMSliceHandleTimed, hash_time); + if(slice != nullptr) + { + uint64_t num_records = rocprofvis_dm_get_property_as_uint64( + slice, kRPVDMNumberOfRecordsUInt64, 0); + spdlog::info(ANSI_COLOR_MAGENTA + "Track({0}) Slice({1}) has {2} records" ANSI_COLOR_RESET, + track, start_time, num_records); + } + } +} + +void +DeleteSliceData(rocprofvis_dm_trace_t trace, rocprofvis_dm_timestamp_t start_time, + rocprofvis_dm_timestamp_t end_time, uint32_t accessInMillisec) +{ + std::this_thread::sleep_for(std::chrono::milliseconds(accessInMillisec)); + spdlog::info(ANSI_COLOR_MAGENTA "{0} milliseconds passed. Start " + "deleting slice({1})" ANSI_COLOR_RESET, + accessInMillisec, start_time); + rocprofvis_dm_delete_time_slice(trace, start_time, end_time); +} + +void ReportTraceType(rocprofvis_db_type_t db_type) +{ + spdlog::info(ANSI_COLOR_CYAN"Database type : {}", + db_type == kAutodetect ? "cannot be detected" : + db_type == kProfilerHub ? "profiler hub supported" : + db_type == kRocpdSqlite ? "legacy rocpd" : + db_type == kRocprofSqlite ? "modern rocpd" : + db_type == kRocprofMultinodeSqlite ? "multi-node rocpd" : + db_type == kComputeSqlite ? "compute database" : + db_type == kChromeTrace ? "chrome trace" : + db_type == kPerfettoTrace ? "perfetto trace" : + db_type == kGoogleSqlite ? "perfetto sqlite" : + "unknown"); +} + +struct RocProfVisDMFixture +{ + mutable rocprofvis_dm_trace_t m_trace = nullptr; + mutable rocprofvis_dm_database_t m_dm_db = nullptr; + mutable rocprofvis_dm_database_t m_ph_db = nullptr; + mutable rocprofvis_dm_timestamp_t m_start_time; + mutable rocprofvis_dm_timestamp_t m_end_time; + mutable rocprofvis_db_num_of_tracks_t m_num_tracks; + mutable rocprofvis_db_track_selection_t m_tracks_selection; + mutable rocprofvis_db_type_t m_dm_db_type; + mutable rocprofvis_db_type_t m_ph_db_type; +}; + + +TEST_CASE_PERSISTENT_FIXTURE(RocProfVisDMFixture, "System Trace Data-Model Tests") +{ + + + // Allocates a new DM trace handle. + // Fixture Writes: m_trace + SECTION("Create Trace") + { + PrintHeader("Create trace"); + m_trace = rocprofvis_dm_create_trace(); + REQUIRE(nullptr != m_trace); + } + + SECTION("Detection using data-model") + { + PrintHeader("Detect database %s", g_input_file.c_str()); + m_dm_db_type = rocprofvis_db_identify_type(g_input_file.c_str()); + REQUIRE(kAutodetect != m_dm_db_type); + ReportTraceType(m_ph_db_type); + } + SECTION("Detection using profiler-hub") + { + PrintHeader("Detect database %s", g_input_file.c_str()); + m_ph_db_type = (rocprofvis_db_type_t)profiler_hub_db_identify_type(g_input_file.c_str()); + REQUIRE(kAutodetect != m_ph_db_type); + ReportTraceType(m_ph_db_type); + } + + // Opens the database file and binds it to the trace handle. + // Fixture Reads: m_trace + // Fixture Writes: m_db + SECTION("Data-model trace initialisation") + { + CheckMemoryFootprint(m_trace); + PrintHeader("Open database %s", g_input_file.c_str()); + m_dm_db = rocprofvis_db_open_database(g_input_file.c_str(), m_dm_db_type); + REQUIRE(nullptr != m_dm_db); + rocprofvis_dm_result_t bind_result = + rocprofvis_dm_bind_trace_to_database(m_trace, m_dm_db, nullptr); + REQUIRE(kRocProfVisDmResultSuccess == bind_result); + } + + SECTION("Profiler-hub trace initialisation") + { + CheckMemoryFootprint(m_trace); + PrintHeader("Open database %s", g_input_file.c_str()); + m_ph_db = rocprofvis_db_open_database(g_input_file.c_str(), m_ph_db_type); + REQUIRE(nullptr != m_ph_db); + rocprofvis_dm_result_t bind_result = + rocprofvis_dm_bind_trace_to_database(m_trace, m_ph_db, nullptr); + REQUIRE(kRocProfVisDmResultSuccess == bind_result); + } + + // Loads database metadata (schema, indexes) asynchronously and waits for completion. + // Fixture Reads: m_db + SECTION("Read Metadata using data-model") + { + PrintHeader("Read Metadata using data-model"); + rocprofvis_db_future_t object2wait = rocprofvis_db_future_alloc(progress_callback); + REQUIRE(object2wait); + spdlog::info("Issue metadata read"); + rocprofvis_dm_result_t meta_result = + rocprofvis_db_read_metadata_async(m_dm_db, object2wait); + REQUIRE(kRocProfVisDmResultSuccess == meta_result); + spdlog::info("Wait for metadata"); + rocprofvis_dm_result_t wait_result = + rocprofvis_db_future_wait(object2wait, UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == wait_result); + rocprofvis_db_future_free(object2wait); + } + + SECTION("Read Metadata using profiler-hub") + { + PrintHeader("Read Metadata using data-model"); + profiler_hub_future_handle_t object2wait = profiler_hub_future_alloc(progress_callback); + REQUIRE(object2wait); + spdlog::info("Issue metadata read"); + rocprofvis_dm_result_t meta_result = + rocprofvis_db_read_metadata_async(m_ph_db, object2wait); + REQUIRE(kRocProfVisDmResultSuccess == meta_result); + spdlog::info("Wait for metadata"); + rocprofvis_dm_result_t wait_result = + rocprofvis_db_future_wait(object2wait, UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == wait_result); + rocprofvis_db_future_free(object2wait); + } + + // Reads per-track metadata including identity, category, process names, + // record counts, time bounds, value range, and instance/process/subprocess IDs. + // Fixture Reads: m_trace + // Fixture Writes: m_start_time, m_end_time, m_num_tracks + SECTION("Read Track Metadata") + { + CheckMemoryFootprint(m_trace); + + m_start_time = + rocprofvis_dm_get_property_as_uint64(m_trace, kRPVDMStartTimeUInt64, 0); + m_end_time = + rocprofvis_dm_get_property_as_uint64(m_trace, kRPVDMEndTimeUInt64, 0); + spdlog::info(ANSI_COLOR_GREEN "Trace start time={0}, end time = {1}", + m_start_time, m_end_time); + m_num_tracks = + (rocprofvis_db_num_of_tracks_t) rocprofvis_dm_get_property_as_uint64( + m_trace, kRPVDMNumberOfTracksUInt64, 0); + for(int i = 0; i < m_num_tracks; i++) + { + rocprofvis_dm_track_t track = rocprofvis_dm_get_property_as_handle( + m_trace, kRPVDMTrackHandleIndexed, i); + REQUIRE(track != nullptr); + + char const* category = rocprofvis_dm_get_property_as_charptr( + track, kRPVDMTrackCategoryEnumCharPtr, 0); + char const* process = rocprofvis_dm_get_property_as_charptr( + track, kRPVDMTrackMainProcessNameCharPtr, 0); + char const* subname = rocprofvis_dm_get_property_as_charptr( + track, kRPVDMTrackSubProcessNameCharPtr, 0); + REQUIRE(category); + REQUIRE(process); + REQUIRE(subname); + + uint64_t track_id = + rocprofvis_dm_get_property_as_uint64(track, kRPVDMTrackIdUInt64, 0); + uint64_t node_id = + rocprofvis_dm_get_property_as_uint64(track, kRPVDMTrackNodeIdUInt64, 0); + + uint64_t num_records = rocprofvis_dm_get_property_as_uint64( + track, kRPVDMTrackNumRecordsUInt64, 0); + uint64_t min_timestamp = rocprofvis_dm_get_property_as_uint64( + track, kRPVDMTrackMinimumTimestampUInt64, 0); + uint64_t max_timestamp = rocprofvis_dm_get_property_as_uint64( + track, kRPVDMTrackMaximumTimestampUInt64, 0); + double min_value = rocprofvis_dm_get_property_as_double( + track, kRPVDMTrackMinimumValueDouble, 0); + double max_value = rocprofvis_dm_get_property_as_double( + track, kRPVDMTrackMaximumValueDouble, 0); + + uint64_t instance_id = rocprofvis_dm_get_property_as_uint64( + track, kRPVDMTrackInstanceIdUInt64, 0); + uint64_t process_id = rocprofvis_dm_get_property_as_uint64( + track, kRPVDMTrackProcessIdUInt64, 0); + uint64_t subprocess_id = rocprofvis_dm_get_property_as_uint64( + track, kRPVDMTrackSubProcessIdUInt64, 0); + + spdlog::info(ANSI_COLOR_CYAN + "Track id={0} node={1} category={2} process={3} subprocess={4} " + "records={5} time=[{6},{7}] value=[{8},{9}] " + "instance={10} pid={11} subpid={12}", + track_id, node_id, category, process, subname, num_records, + min_timestamp, max_timestamp, min_value, max_value, instance_id, + process_id, subprocess_id); + } + } + + // Retrieves the topology root from the trace and recursively walks all nodes, + // reading name, type, optional track binding, keyed properties, and children. + // Fixture Reads: m_trace + SECTION("Read Topology") + { + CheckMemoryFootprint(m_trace); + PrintHeader("Read topology tree"); + + rocprofvis_dm_topology_node root = + rocprofvis_dm_get_property_as_handle(m_trace, kRPVDMTopologyHandle, 0); + REQUIRE(root != nullptr); + + struct NodeVisitor + { + static void Visit(rocprofvis_dm_topology_node node, int depth) + { + char* name = rocprofvis_dm_get_property_as_charptr( + node, kRPVControllerTopologyNodeName, 0); + uint64_t type = rocprofvis_dm_get_property_as_uint64( + node, kRPVControllerTopologyNodeType, 0); + + std::string indent(depth * 2, ' '); + spdlog::info("{}Node: name={}, type={}", indent, name ? name : "(null)", + type); + + rocprofvis_dm_get_property_as_uint64(node, + kRPVControllerTopologyNodeTrack, 0); + + uint64_t num_properties = rocprofvis_dm_get_property_as_uint64( + node, kRPVControllerTopologyNodeNumProperties, 0); + for(uint64_t p = 0; p < num_properties; p++) + { + uint64_t key = rocprofvis_dm_get_property_as_uint64( + node, kRPVControllerTopologyNodePropertyKeyIndexed, p); + uint64_t prop_type = rocprofvis_dm_get_property_as_uint64( + node, kRPVControllerTopologyNodePropertyTypeIndexed, p); + if(prop_type == kRPVDataTypeString) + { + char* str_value = rocprofvis_dm_get_property_as_charptr( + node, kRPVControllerTopologyNodePropertyValueIndexed, p); + spdlog::trace("{} Property {}: key={}, value={}, type=string", + indent, p, key, str_value ? str_value : "(null)"); + } + else if(prop_type == kRPVDataTypeDouble) + { + double dbl_value = rocprofvis_dm_get_property_as_double( + node, kRPVControllerTopologyNodePropertyValueIndexed, p); + spdlog::trace("{} Property {}: key={}, value={}, type=double", + indent, p, key, dbl_value); + } + else + { + uint64_t int_value = rocprofvis_dm_get_property_as_uint64( + node, kRPVControllerTopologyNodePropertyValueIndexed, p); + spdlog::trace("{} Property {}: key={}, value={}, type=int", + indent, p, key, int_value); + } + } + + uint64_t num_children = rocprofvis_dm_get_property_as_uint64( + node, kRPVControllerTopologyNodeNumChildren, 0); + for(uint64_t c = 0; c < num_children; c++) + { + rocprofvis_dm_topology_node child = + rocprofvis_dm_get_property_as_handle( + node, kRPVControllerTopologyNodeChildHandleIndexed, c); + REQUIRE(child != nullptr); + Visit(child, depth + 1); + } + } + }; + + NodeVisitor::Visit(root, 0); + } + + // Reads global histogram bucket dimensions from the trace, then iterates a + // subset of tracks reading per-track bucket event density and bucket value. + // Fixture Reads: m_trace, m_num_tracks + SECTION("Read Histogram Data") + { + CheckMemoryFootprint(m_trace); + PrintHeader("Read histogram dimensions"); + + uint64_t num_buckets = + rocprofvis_dm_get_property_as_uint64(m_trace, kRPVDMHistogramNumBuckets, 0); + uint64_t bucket_size = + rocprofvis_dm_get_property_as_uint64(m_trace, kRPVDMHistogramBucketSize, 0); + spdlog::info(ANSI_COLOR_GREEN "Histogram: num_buckets={}, bucket_size={}", + num_buckets, bucket_size); + + if(num_buckets > 0) + { + int tracks_to_check = std::min((int) m_num_tracks, LIST_SIZE_LIMIT); + for(int i = 0; i < tracks_to_check; i++) + { + rocprofvis_dm_track_t track = rocprofvis_dm_get_property_as_handle( + m_trace, kRPVDMTrackHandleIndexed, i); + REQUIRE(track != nullptr); + + char const* category = rocprofvis_dm_get_property_as_charptr( + track, kRPVDMTrackCategoryEnumCharPtr, 0); + spdlog::trace("Track {} ({}) histogram buckets:", i, + category ? category : "(null)"); + + for(uint64_t b = 0; b < num_buckets; b++) + { + uint64_t density = rocprofvis_dm_get_property_as_uint64( + track, kRPVDMTrackHistogramBucketEventDensityUInt64Indexed, b); + double bucket_value = rocprofvis_dm_get_property_as_double( + track, kRPVDMTrackHistogramBucketValueDoubleIndexed, b); + spdlog::trace(" bucket[{}]: density={}, value={}", b, density, + bucket_value); + } + } + } + } + + // Saves a trimmed time range of the trace to a temporary file, validates + // success, then removes the file. + // Fixture Reads: m_trace, m_db, m_start_time, m_end_time + // Fixture Writes: m_num_tracks, m_tracks_selection + SECTION("Trim Trace") + { + PrintHeader("Trim trace"); + rocprofvis_db_future_t object2wait = rocprofvis_db_future_alloc(progress_callback); + REQUIRE(nullptr != object2wait); + + rocprofvis_dm_timestamp_t start_time = m_start_time; + rocprofvis_dm_timestamp_t end_time = m_end_time; + GenerateRandomSlice(m_trace, m_num_tracks, m_tracks_selection, start_time, + end_time); + rocprofvis_dm_result_t result = rocprofvis_db_trim_save_async( + m_db, start_time, (start_time + ((end_time - start_time) / 2)), + "sample/trimmed.db", object2wait); + REQUIRE(result == kRocProfVisDmResultSuccess); + + rocprofvis_dm_result_t read_wait = + rocprofvis_db_future_wait(object2wait, UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == read_wait); + std::filesystem::remove("sample/trimmed.db"); + + rocprofvis_db_future_free(object2wait); + } + + // Reads each selected track's full time range one at a time, measuring read + // times and row counts per track and in total. + // Fixture Reads: m_trace, m_db, m_start_time, m_end_time, m_num_tracks + // Fixture Writes: m_num_tracks, m_tracks_selection + SECTION("Whole Trace Read") + { + PrintHeader("Whole Trace Read"); + double whole_trace_readtime = 0; + uint32_t total_num_rows = 0; + + rocprofvis_dm_timestamp_t start_time = m_start_time; + rocprofvis_dm_timestamp_t end_time = m_end_time; + GenerateRandomSlice(m_trace, m_num_tracks, m_tracks_selection, start_time, + end_time); + for(int i = 0; i < m_num_tracks; i++) + { + rocprofvis_dm_track_t track = rocprofvis_dm_get_property_as_handle( + m_trace, kRPVDMTrackHandleIndexed, i); + REQUIRE(track != nullptr); + + bool load_slice = false; + for(int j = 0; j < m_num_tracks; j++) + { + if(m_tracks_selection[j] == static_cast(i)) + { + load_slice = true; + break; + } + } + if(load_slice) + { + rocprofvis_db_future_t object2wait = + rocprofvis_db_future_alloc(progress_callback); + REQUIRE(nullptr != object2wait); + + auto t1 = std::chrono::steady_clock::now(); + rocprofvis_dm_result_t read_slice_issue = + rocprofvis_db_read_trace_slice_async(m_db, start_time, end_time, + kRocProfVisDmHashedTimestampTagTrackSlice, 1, + (uint32_t*) &i, object2wait); + REQUIRE(kRocProfVisDmResultSuccess == read_slice_issue); + if(kRocProfVisDmResultSuccess == read_slice_issue) + { + rocprofvis_dm_result_t read_wait = + rocprofvis_db_future_wait(object2wait, UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == read_wait); + if(kRocProfVisDmResultSuccess == read_wait) + { + auto t2 = std::chrono::steady_clock::now(); + std::chrono::duration diff = t2 - t1; + whole_trace_readtime += diff.count(); + uint32_t num_rows = ((RocProfVis::DataModel::Future*) object2wait) + ->GetProcessedRowsCount(); + total_num_rows += num_rows; + uint64_t hash_time = rocprofvis_dm_hash_combine_timestamp(start_time, end_time, kRocProfVisDmHashedTimestampTagTrackSlice); + rocprofvis_dm_slice_t slice = + rocprofvis_dm_get_property_as_handle( + track, kRPVDMSliceHandleTimed, hash_time); + REQUIRE(slice); + uint64_t num_records = rocprofvis_dm_get_property_as_uint64( + slice, kRPVDMNumberOfRecordsUInt64, 0); + spdlog::info( + ANSI_COLOR_MAGENTA + "Track {0} has {1} records, read time - {2:13.9f}, number of " + "rows processed = {3}" ANSI_COLOR_RESET, + i, num_records, diff.count(), num_rows); + rocprofvis_dm_delete_all_time_slices(m_trace); + } + } + + rocprofvis_db_future_free(object2wait); + } + } + + spdlog::info(ANSI_COLOR_MAGENTA + "Whole trace read time - {0:13.9f}, number of rows " + "processed = {1}" ANSI_COLOR_RESET, + whole_trace_readtime, total_num_rows); + } + + // Reads a random time slice for selected tracks, validates record access + // including event properties, flow traces, stack traces, extended data, + // argument records, and PMC sample values. + // Fixture Reads: m_trace, m_db, m_start_time, m_end_time, m_num_tracks, + // m_tracks_selection + SECTION("Read Random Slice Data") + { + PrintHeader("Read Random Slice Data"); + if(m_num_tracks > 0) + { + rocprofvis_db_future_t object2wait = rocprofvis_db_future_alloc(progress_callback); + REQUIRE(nullptr != object2wait); + + spdlog::info("Read time slice for all tracks"); + auto t1 = std::chrono::steady_clock::now(); + rocprofvis_dm_result_t read_slice_result = + rocprofvis_db_read_trace_slice_async(m_db, m_start_time, m_end_time, + kRocProfVisDmHashedTimestampTagTrackSlice, + m_num_tracks, m_tracks_selection, + object2wait); + REQUIRE(kRocProfVisDmResultSuccess == read_slice_result); + rocprofvis_dm_result_t slice_wait_result = + rocprofvis_db_future_wait(object2wait, UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == slice_wait_result); + auto t2 = std::chrono::steady_clock::now(); + std::chrono::duration diff = t2 - t1; + uint32_t num_rows = + ((RocProfVis::DataModel::Future*) object2wait)->GetProcessedRowsCount(); + spdlog::info(ANSI_COLOR_MAGENTA + "Whole trace read time - {0:13.9f}, number of " + "rows processed = {1}" ANSI_COLOR_RESET, + diff.count(), num_rows); + CheckMemoryFootprint(m_trace); + spdlog::info("Time slice content, up to {} records", LIST_SIZE_LIMIT); + int first_track = std::rand() % m_num_tracks; + for(int i = first_track; + (i < m_num_tracks) && (i < first_track + LIST_SIZE_LIMIT); i++) + { + rocprofvis_dm_track_t track = rocprofvis_dm_get_property_as_handle( + m_trace, kRPVDMTrackHandleIndexed, m_tracks_selection[i]); + REQUIRE(track != nullptr); + char* track_category_name = rocprofvis_dm_get_property_as_charptr( + track, kRPVDMTrackCategoryEnumCharPtr, 0); + char* track_process_name = rocprofvis_dm_get_property_as_charptr( + track, kRPVDMTrackMainProcessNameCharPtr, 0); + char* track_sub_process_name = rocprofvis_dm_get_property_as_charptr( + track, kRPVDMTrackSubProcessNameCharPtr, 0); + REQUIRE(track_category_name); + REQUIRE(track_process_name); + REQUIRE(track_sub_process_name); + rocprofvis_dm_track_category_t track_category = + (rocprofvis_dm_track_category_t) rocprofvis_dm_get_property_as_uint64( + track, kRPVDMTrackCategoryEnumUInt64, 0); + uint64_t hash_time = rocprofvis_dm_hash_combine_timestamp(m_start_time, m_end_time, kRocProfVisDmHashedTimestampTagTrackSlice); + rocprofvis_dm_slice_t slice = rocprofvis_dm_get_property_as_handle( + track, kRPVDMSliceHandleTimed, hash_time); + REQUIRE(slice != nullptr); + uint64_t track_id = + rocprofvis_dm_get_property_as_uint64(track, kRPVDMTrackIdUInt64, 0); + uint64_t node_id = rocprofvis_dm_get_property_as_uint64( + track, kRPVDMTrackNodeIdUInt64, 0); + uint64_t memory_usage = rocprofvis_dm_get_property_as_uint64( + track, kRPVDMTrackMemoryFootprintUInt64, 0); + uint64_t num_ext_data = rocprofvis_dm_get_property_as_uint64( + track, kRPVDMNumberOfTrackExtDataRecordsUInt64, 0); + spdlog::info("Track id={} node={} category={} process={} " + "subprocess={}", + track_id, node_id, track_category_name, track_process_name, + track_sub_process_name); + + spdlog::info(ANSI_COLOR_CYAN "\t{0} : {1} : {2}", "Properties", + "Memory usage", memory_usage); + for(int ext_data_index = 0; ext_data_index < num_ext_data; ext_data_index++) + { + char* ext_data_category = rocprofvis_dm_get_property_as_charptr( + track, kRPVDMTrackExtDataCategoryCharPtrIndexed, ext_data_index); + char* ext_data_name = rocprofvis_dm_get_property_as_charptr( + track, kRPVDMTrackExtDataNameCharPtrIndexed, ext_data_index); + char* ext_data_value = rocprofvis_dm_get_property_as_charptr( + track, kRPVDMTrackExtDataValueCharPtrIndexed, ext_data_index); + REQUIRE(ext_data_category); + REQUIRE(ext_data_name); + REQUIRE(ext_data_value); + spdlog::info(ANSI_COLOR_CYAN "\t{0} : {1} : {2}", ext_data_category, + ext_data_name, ext_data_value); + } + if(nullptr != slice) + { + uint64_t num_records = rocprofvis_dm_get_property_as_uint64( + slice, kRPVDMNumberOfRecordsUInt64, 0); + spdlog::info(ANSI_COLOR_BLUE + "Time slice for time {0} - {1} for " + "track {2} [{3}:{4}:{5}] has {6} records", + m_start_time, m_end_time, m_tracks_selection[i], + track_category_name, track_process_name, + track_sub_process_name, num_records); + if(num_records == 0) continue; + int first_record = std::rand() % num_records; + for(int j = first_record; + (j < num_records) && (j < first_record + LIST_SIZE_LIMIT); j++) + { + uint64_t timestamp = rocprofvis_dm_get_property_as_uint64( + slice, kRPVDMTimestampUInt64Indexed, j); + if(track_category == + rocprofvis_dm_track_category_t::kRocProfVisDmRegionTrack || + track_category == rocprofvis_dm_track_category_t:: + kRocProfVisDmKernelDispatchTrack || + track_category == rocprofvis_dm_track_category_t:: + kRocProfVisDmMemoryAllocationTrack || + track_category == rocprofvis_dm_track_category_t:: + kRocProfVisDmMemoryCopyTrack) + + { + int64_t duration = rocprofvis_dm_get_property_as_int64( + slice, kRPVDMEventDurationInt64Indexed, j); + if(duration < 0) + { + spdlog::info(ANSI_COLOR_RED + "Record {0} has invalid duration " + "{1}" ANSI_COLOR_RESET, + j, duration); + } + uint64_t event_id = rocprofvis_dm_get_property_as_uint64( + slice, kRPVDMEventIdUInt64Indexed, j); + uint64_t op = rocprofvis_dm_get_property_as_uint64( + slice, kRPVDMEventIdOperationEnumIndexed, j); + char* op_str = rocprofvis_dm_get_property_as_charptr( + slice, kRPVDMEventIdOperationCharPtrIndexed, j); + char* type_str = rocprofvis_dm_get_property_as_charptr( + slice, kRPVDMEventTypeStringCharPtrIndexed, j); + char* symbol_str = rocprofvis_dm_get_property_as_charptr( + slice, kRPVDMEventSymbolStringCharPtrIndexed, j); + REQUIRE(op_str); + REQUIRE(type_str); + REQUIRE(symbol_str); + uint64_t event_level = rocprofvis_dm_get_property_as_uint64( + slice, kRPVDMEventLevelUInt64Indexed, j); + spdlog::info(ANSI_COLOR_BLUE + "Record id={0}, timestamp={1}, op={2}, " + "op_str={3}, type={4}, symbol={5}, " + "level={6}", + event_id, timestamp, op, op_str, type_str, + symbol_str, event_level); + + spdlog::info("Testing multithreaded access to " + "database retrieving flow, stack and " + "extended data for event id = {}", + event_id); + rocprofvis_dm_event_id_t dm_event_id; + dm_event_id.value = event_id; + rocprofvis_db_future_t object2wait4flowtrace = + rocprofvis_db_future_alloc(progress_callback); + REQUIRE(object2wait4flowtrace); + rocprofvis_db_future_t object2wait4stacktrace = + rocprofvis_db_future_alloc(progress_callback); + REQUIRE(object2wait4stacktrace); + rocprofvis_db_future_t object2wait4extdata = + rocprofvis_db_future_alloc(progress_callback); + REQUIRE(object2wait4extdata); + rocprofvis_dm_result_t result4flowtrace = + rocprofvis_db_read_event_property_async( + m_db, kRPVDMEventFlowTrace, dm_event_id, + object2wait4flowtrace); + REQUIRE(kRocProfVisDmResultSuccess == result4flowtrace); + rocprofvis_dm_result_t result4stacktrace = + rocprofvis_db_read_event_property_async( + m_db, kRPVDMEventStackTrace, dm_event_id, + object2wait4stacktrace); + REQUIRE(kRocProfVisDmResultSuccess == result4stacktrace); + rocprofvis_dm_result_t result4extdata = + rocprofvis_db_read_event_property_async( + m_db, kRPVDMEventExtData, dm_event_id, + object2wait4extdata); + REQUIRE(kRocProfVisDmResultSuccess == result4extdata); + + rocprofvis_dm_result_t result4flowtrace_wait = + rocprofvis_db_future_wait(object2wait4flowtrace, + UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == result4flowtrace_wait); + + if(kRocProfVisDmResultSuccess == result4flowtrace && + kRocProfVisDmResultSuccess == result4flowtrace_wait) + { + rocprofvis_dm_flowtrace_t flowtrace = + rocprofvis_dm_get_property_as_handle( + m_trace, kRPVDMFlowTraceHandleByEventID, + dm_event_id.value); + REQUIRE(flowtrace != nullptr); + spdlog::info("Data flow trace for event id = {}", + (uint64_t) dm_event_id.bitfield.event_id); + uint64_t num_endpoints = + rocprofvis_dm_get_property_as_uint64( + flowtrace, kRPVDMNumberOfEndpointsUInt64, 0); + spdlog::info(ANSI_COLOR_MAGENTA + "Event has {0} data flow link:", + num_endpoints); + for(int k = 0; k < num_endpoints; k++) + { + uint64_t endpoint_track_id = + rocprofvis_dm_get_property_as_uint64( + flowtrace, kRPVDMEndpointTrackIDUInt64Indexed, + k); + rocprofvis_dm_event_id_t endpoint_event_id; + endpoint_event_id.value = + rocprofvis_dm_get_property_as_uint64( + flowtrace, kRPVDMEndpointIDUInt64Indexed, k); + uint64_t event_timestamp = + rocprofvis_dm_get_property_as_uint64( + flowtrace, + kRPVDMEndpointTimestampUInt64Indexed, k); + uint64_t end_timestamp = + rocprofvis_dm_get_property_as_uint64( + flowtrace, + kRPVDMEndpointEndTimestampUInt64Indexed, k); + char* endpoint_category = + rocprofvis_dm_get_property_as_charptr( + flowtrace, + kRPVDMEndpointCategoryCharPtrIndexed, k); + char* endpoint_symbol = + rocprofvis_dm_get_property_as_charptr( + flowtrace, kRPVDMEndpointSymbolCharPtrIndexed, + k); + uint64_t endpoint_level = + rocprofvis_dm_get_property_as_uint64( + flowtrace, kRPVDMEndpointLevelUInt64Indexed, + k); + uint64_t event_id_num = + endpoint_event_id.bitfield.event_id; + spdlog::info( + ANSI_COLOR_MAGENTA "\tEndpoint {0} at track={1}, " + "event_id={2}, timestamp={3}, " + "end_ts={4}, category={5}, " + "symbol={6}, level={7}", + k, endpoint_track_id, event_id_num, + event_timestamp, end_timestamp, + endpoint_category ? endpoint_category : "(null)", + endpoint_symbol ? endpoint_symbol : "(null)", + endpoint_level); + } + rocprofvis_dm_delete_event_property_for( + m_trace, kRPVDMEventFlowTrace, dm_event_id); + } + rocprofvis_db_future_free(object2wait4flowtrace); + + rocprofvis_dm_result_t stacktrack_wait = + rocprofvis_db_future_wait(object2wait4stacktrace, + UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == stacktrack_wait); + if(kRocProfVisDmResultSuccess == result4stacktrace && + kRocProfVisDmResultSuccess == stacktrack_wait) + { + rocprofvis_dm_stacktrace_t stacktrace = + rocprofvis_dm_get_property_as_handle( + m_trace, kRPVDMStackTraceHandleByEventID, + dm_event_id.value); + REQUIRE(stacktrace != nullptr); + spdlog::info("Stack trace for event id = {}", + (uint64_t) dm_event_id.bitfield.event_id); + uint64_t num_frames = + rocprofvis_dm_get_property_as_uint64( + stacktrace, kRPVDMNumberOfFramesUInt64, 0); + spdlog::info(ANSI_COLOR_MAGENTA + "Event has {} stack frames:", + num_frames); + for(int k = 0; k < num_frames; k++) + { + uint64_t frame_region_id = + rocprofvis_dm_get_property_as_uint64( + stacktrace, kRPVDMFrameRegionIdUInt64Indexed, + k); + uint64_t frame_depth = + rocprofvis_dm_get_property_as_uint64( + stacktrace, kRPVDMFrameDepthUInt64Indexed, k); + char* frame_symbol = + rocprofvis_dm_get_property_as_charptr( + stacktrace, kRPVDMFrameSymbolCharPtrIndexed, + k); + char* frame_args = + rocprofvis_dm_get_property_as_charptr( + stacktrace, kRPVDMFrameArgsCharPtrIndexed, k); + char* frame_code = + rocprofvis_dm_get_property_as_charptr( + stacktrace, kRPVDMFrameCodeLineCharPtrIndexed, + k); + REQUIRE(frame_symbol); + REQUIRE(frame_args); + REQUIRE(frame_code); + spdlog::info(ANSI_COLOR_MAGENTA + "\tFrame {0} : region_id={1}, " + "depth={2}, symbol={3}, args={4}, " + "code={5} \n", + k, frame_region_id, frame_depth, + frame_symbol, frame_args, frame_code); + } + rocprofvis_dm_delete_event_property_for( + m_trace, kRPVDMEventStackTrace, dm_event_id); + } + rocprofvis_db_future_free(object2wait4stacktrace); + + rocprofvis_dm_result_t extdata_wait = + rocprofvis_db_future_wait(object2wait4extdata, + UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == extdata_wait); + if(kRocProfVisDmResultSuccess == result4extdata && + kRocProfVisDmResultSuccess == extdata_wait) + { + rocprofvis_dm_extdata_t extdata = + rocprofvis_dm_get_property_as_handle( + m_trace, kRPVDMExtInfoHandleByEventID, + dm_event_id.value); + REQUIRE(extdata != nullptr); + spdlog::info("Extended data for event id = {}", + (uint64_t) dm_event_id.bitfield.event_id); + uint64_t extdata_num_records = + rocprofvis_dm_get_property_as_uint64( + extdata, kRPVDMNumberOfExtDataRecordsUInt64, 0); + spdlog::info(ANSI_COLOR_MAGENTA + "Event has {} extended data " + "properties:", + extdata_num_records); + for(int k = 0; k < extdata_num_records; k++) + { + char* data_category = + rocprofvis_dm_get_property_as_charptr( + extdata, kRPVDMExtDataCategoryCharPtrIndexed, + k); + char* data_name = + rocprofvis_dm_get_property_as_charptr( + extdata, kRPVDMExtDataNameCharPtrIndexed, k); + char* data_value = + rocprofvis_dm_get_property_as_charptr( + extdata, kRPVDMExtDataValueCharPtrIndexed, k); + REQUIRE(data_category); + REQUIRE(data_name); + REQUIRE(data_value); + uint64_t data_type = + rocprofvis_dm_get_property_as_uint64( + extdata, kRPVDMExtDataTypeUint64Indexed, k); + uint64_t data_enum = + rocprofvis_dm_get_property_as_uint64( + extdata, kRPVDMExtDataEnumUint64Indexed, k); + spdlog::info(ANSI_COLOR_MAGENTA + "\tItem {0} : category={1}, " + "name={2}, value={3}, " + "type={4}, enum={5} \n", + k, data_category, data_name, data_value, + data_type, data_enum); + } + + uint64_t num_args = rocprofvis_dm_get_property_as_uint64( + extdata, kRPVDMNumberOfArgumentRecordsUInt64, 0); + spdlog::info(ANSI_COLOR_MAGENTA + "Event has {} argument records:", + num_args); + for(uint64_t a = 0; a < num_args; a++) + { + uint64_t arg_pos = + rocprofvis_dm_get_property_as_uint64( + extdata, kRPVDMArgumentPositionUint64Indexed, + a); + char* arg_type = + rocprofvis_dm_get_property_as_charptr( + extdata, kRPVDMArgumentTypeCharPtrIndexed, a); + char* arg_name = + rocprofvis_dm_get_property_as_charptr( + extdata, kRPVDMArgumentNameCharPtrIndexed, a); + char* arg_value = + rocprofvis_dm_get_property_as_charptr( + extdata, kRPVDMArgumentValueCharPtrIndexed, + a); + spdlog::info(ANSI_COLOR_MAGENTA + "\tArg {0} : pos={1}, type={2}, " + "name={3}, value={4}", + a, arg_pos, + arg_type ? arg_type : "(null)", + arg_name ? arg_name : "(null)", + arg_value ? arg_value : "(null)"); + } + + rocprofvis_dm_delete_event_property_for( + m_trace, kRPVDMEventExtData, dm_event_id); + } + rocprofvis_db_future_free(object2wait4extdata); + } + else if(track_category == + rocprofvis_dm_track_category_t::kRocProfVisDmPmcTrack) + { + double value = rocprofvis_dm_get_property_as_double( + slice, kRPVDMPmcValueDoubleIndexed, j); + spdlog::info(ANSI_COLOR_BLUE + "Record timestamp={0}, value={1}\n", + timestamp, value); + } + } + } + else + { + spdlog::info(ANSI_COLOR_RED + "No time slice at {0} loaded for track {1}", + m_start_time, m_tracks_selection[i]); + } + } + spdlog::info("Delete all slices"); + rocprofvis_dm_delete_all_time_slices(m_trace); + + rocprofvis_db_future_free(object2wait); + } + } + + // Executes a SQL query against the database, validates the resulting table + // has columns and rows, and prints the content. + // Fixture Reads: m_trace, m_db + SECTION("SQL Query") + { + PrintHeader("Test SQL Query"); + CheckMemoryFootprint(m_trace); + + rocprofvis_db_future_t object2wait = rocprofvis_db_future_alloc(progress_callback); + rocprofvis_dm_table_id_t table_id = 0; + REQUIRE(nullptr != object2wait); + spdlog::info("Test SQL table read"); + rocprofvis_dm_result_t query_result = rocprofvis_db_execute_query_async( + m_db, "select * from top;", "Kernel execution summary", object2wait, + &table_id); + + REQUIRE((kRocProfVisDmResultSuccess == query_result || + kRocProfVisDmResultNotSupported == query_result)); + rocprofvis_dm_result_t query_wait = + rocprofvis_db_future_wait(object2wait, UINT64_MAX); + if(kRocProfVisDmResultSuccess == query_wait) + { + uint64_t num_tables = rocprofvis_dm_get_property_as_uint64( + m_trace, kRPVDMNumberOfTablesUInt64, 0); + if(num_tables > 0) + { + rocprofvis_dm_table_t table = rocprofvis_dm_get_property_as_handle( + m_trace, kRPVDMTableHandleByID, table_id); + REQUIRE(table); + char* table_description = rocprofvis_dm_get_property_as_charptr( + table, kRPVDMExtTableDescriptionCharPtr, 0); + REQUIRE(table_description); + char* table_query = rocprofvis_dm_get_property_as_charptr( + table, kRPVDMExtTableQueryCharPtr, 0); + REQUIRE(table_query); + uint64_t num_columns = rocprofvis_dm_get_property_as_uint64( + table, kRPVDMNumberOfTableColumnsUInt64, 0); + uint64_t num_rows = rocprofvis_dm_get_property_as_uint64( + table, kRPVDMNumberOfTableRowsUInt64, 0); + + { + spdlog::info("\x1b[0m{0} - {1}", table_description, table_query); + MULTI_LINE_LOG_START; + std::vector columns; + for(int i = 0; i < num_columns; i++) + { + columns.push_back(rocprofvis_dm_get_property_as_charptr( + table, kRPVDMExtTableColumnNameCharPtrIndexed, i)); + } + for(int i = 0; i < num_columns; i++) + { + MULTI_LINE_LOG(" {0:20s} |", columns[i].c_str()); + } + spdlog::info(MULTI_LINE_LOG_ARGS); + } + for(int i = 0; i < num_rows; i++) + { + MULTI_LINE_LOG_START; + rocprofvis_dm_table_row_t table_row = + rocprofvis_dm_get_property_as_handle( + table, kRPVDMExtTableRowHandleIndexed, i); + REQUIRE(table_row); + if(table_row != nullptr) + { + uint64_t num_cells = rocprofvis_dm_get_property_as_uint64( + table_row, kRPVDMNumberOfTableRowCellsUInt64, 0); + if(num_cells == num_columns) + { + std::vector row; + for(int j = 0; j < num_cells; j++) + { + row.push_back(rocprofvis_dm_get_property_as_charptr( + table_row, kRPVDMExtTableRowCellValueCharPtrIndexed, + j)); + } + for(int j = 0; j < num_cells; j++) + { + MULTI_LINE_LOG("{0:20s}", row[j].substr(0, 20).c_str()); + } + } + else + { + MULTI_LINE_LOG("\x1b[31mError! Number of colums does nt " + "match number of cells in row {}\n", + i); + break; + } + } + spdlog::info(MULTI_LINE_LOG_ARGS); + } + rocprofvis_dm_delete_all_tables(m_trace); + } + } + rocprofvis_db_future_free(object2wait); + } + + // Builds a structured table query via rocprofvis_db_build_table_query and validates + // the resulting table has columns and rows. + // Fixture Reads: m_trace, m_db, m_start_time, m_end_time + SECTION("Structured Table Query") + { + CheckMemoryFootprint(m_trace); + + uint32_t op[1] = { TABLE_QUERY_PACK_OP_TYPE(kRocProfVisDmOperationDispatch) }; + std::string sort_column = "total_duration"; + char* built_query = nullptr; + rocprofvis_dm_result_t build_result = rocprofvis_db_build_table_query( + m_db, kRPVDMTableUseCaseEventTrackTable, m_start_time, m_end_time, 1, + (rocprofvis_db_track_selection_t) op, nullptr, nullptr, + "name, COUNT(*) AS num_invocations, AVG(duration) AS avg_duration, " + "MIN(duration) AS min_duration, MAX(duration) AS max_duration, SUM(duration) " + "AS total_duration", + "name", sort_column.c_str(), kRPVDMSortOrderDesc, 0, 0, false, &built_query); + REQUIRE(kRocProfVisDmResultSuccess == build_result); + REQUIRE(built_query != nullptr); + + spdlog::info(ANSI_COLOR_GREEN "Built table query: {}", built_query); + + rocprofvis_db_future_t object2wait = rocprofvis_db_future_alloc(progress_callback); + REQUIRE(nullptr != object2wait); + + rocprofvis_dm_table_id_t table_id = 0; + rocprofvis_dm_result_t exec_result = rocprofvis_db_execute_query_async( + m_db, built_query, "Structured table query test", object2wait, &table_id); + REQUIRE(kRocProfVisDmResultSuccess == exec_result); + + rocprofvis_dm_result_t wait_result = + rocprofvis_db_future_wait(object2wait, UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == wait_result); + uint64_t num_tables = + rocprofvis_dm_get_property_as_uint64(m_trace, kRPVDMNumberOfTablesUInt64, 0); + if(num_tables > 0) + { + rocprofvis_dm_table_t table = rocprofvis_dm_get_property_as_handle( + m_trace, kRPVDMTableHandleByID, table_id); + REQUIRE(table != nullptr); + char* table_description = rocprofvis_dm_get_property_as_charptr( + table, kRPVDMExtTableDescriptionCharPtr, 0); + REQUIRE(table_description); + uint64_t num_columns = rocprofvis_dm_get_property_as_uint64( + table, kRPVDMNumberOfTableColumnsUInt64, 0); + uint64_t num_rows = rocprofvis_dm_get_property_as_uint64( + table, kRPVDMNumberOfTableRowsUInt64, 0); + spdlog::info(ANSI_COLOR_GREEN + "Structured query result: desc={}, cols={}, rows={}", + table_description, num_columns, num_rows); + REQUIRE(num_columns > 0); + + for(uint64_t r = 0; r < std::min(num_rows, (uint64_t) LIST_SIZE_LIMIT); r++) + { + rocprofvis_dm_table_row_t row = rocprofvis_dm_get_property_as_handle( + table, kRPVDMExtTableRowHandleIndexed, r); + REQUIRE(row != nullptr); + uint64_t num_cells = rocprofvis_dm_get_property_as_uint64( + row, kRPVDMNumberOfTableRowCellsUInt64, 0); + REQUIRE(num_cells == num_columns); + } + + rocprofvis_dm_delete_all_tables(m_trace); + } + free(built_query); + rocprofvis_db_future_free(object2wait); + } + + // Builds a structured table query, exports the result to a temporary CSV file + // via rocprofvis_db_export_table_csv_async, validates the file exists and has + // content, then removes it. + // Fixture Reads: m_trace, m_db, m_start_time, m_end_time + SECTION("Table CSV Export") + { + CheckMemoryFootprint(m_trace); + + uint32_t op[1] = { TABLE_QUERY_PACK_OP_TYPE(kRocProfVisDmOperationDispatch) }; + std::string sort_column = "total_duration"; + char* csv_query = nullptr; + rocprofvis_dm_result_t build_result = rocprofvis_db_build_table_query( + m_db, kRPVDMTableUseCaseEventTrackTable, m_start_time, m_end_time, 1, + (rocprofvis_db_track_selection_t) op, nullptr, nullptr, + "name, COUNT(*) AS num_invocations, AVG(duration) AS avg_duration, " + "MIN(duration) AS min_duration, MAX(duration) AS max_duration, SUM(duration) " + "AS total_duration", + "name", sort_column.c_str(), kRPVDMSortOrderDesc, 0, 0, false, &csv_query); + REQUIRE(kRocProfVisDmResultSuccess == build_result); + REQUIRE(csv_query != nullptr); + + const char* csv_path = "sample/test_export.csv"; + + rocprofvis_db_future_t object2wait = rocprofvis_db_future_alloc(progress_callback); + REQUIRE(nullptr != object2wait); + + rocprofvis_dm_result_t export_result = + rocprofvis_db_export_table_csv_async(m_db, csv_query, csv_path, object2wait); + REQUIRE(kRocProfVisDmResultSuccess == export_result); + rocprofvis_dm_result_t wait_result = + rocprofvis_db_future_wait(object2wait, UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == wait_result); + REQUIRE(std::filesystem::exists(csv_path)); + uint64_t file_size = std::filesystem::file_size(csv_path); + spdlog::info(ANSI_COLOR_GREEN "CSV export file size: {} bytes", file_size); + REQUIRE(file_size > 0); + + std::filesystem::remove(csv_path); + free(csv_query); + rocprofvis_db_future_free(object2wait); + } + + // Stress-tests concurrent read and delete operations on time slices across + // multiple iterations to validate thread safety. + // Fixture Reads: m_trace, m_db, m_start_time, m_end_time, m_num_tracks + // Fixture Writes: m_num_tracks, m_tracks_selection + // Local Reads: total_num_tracks + SECTION("Thread Safety") + { + PrintHeader("Thread safety test"); + g_full_range = true; + for(int test_count = 0; test_count < 10; test_count++) + { + rocprofvis_dm_timestamp_t start_time = m_start_time; + rocprofvis_dm_timestamp_t end_time = m_end_time; + GenerateRandomSlice(m_trace, m_num_tracks, m_tracks_selection, start_time, + end_time); + + rocprofvis_db_future_t object2wait = rocprofvis_db_future_alloc(progress_callback); + REQUIRE(nullptr != object2wait); + + rocprofvis_dm_result_t read_slice_issue = + rocprofvis_db_read_trace_slice_async(m_db, start_time, end_time, + kRocProfVisDmHashedTimestampTagTrackSlice, + m_num_tracks, m_tracks_selection, + object2wait); + REQUIRE(kRocProfVisDmResultSuccess == read_slice_issue); + spdlog::info("Access data while loaded"); + { + std::thread access_slice(ReadSliceData, m_trace, m_num_tracks, start_time, + end_time, m_num_tracks * 20, object2wait); + access_slice.join(); + } + rocprofvis_dm_result_t read_wait = + rocprofvis_db_future_wait(object2wait, UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == read_wait); + spdlog::info("Access data while deleted"); + std::thread delete_slice(DeleteSliceData, m_trace, start_time, end_time, 100); + { + std::thread access_slice(ReadSliceData, m_trace, m_num_tracks, start_time, + end_time, 99, nullptr); + delete_slice.join(); + access_slice.join(); + } + rocprofvis_db_future_free(object2wait); + } + } + + // Cleans the database with rebuild enabled. + // Fixture Reads: m_trace, m_db + SECTION("Database Cleanup") + { + CheckMemoryFootprint(m_trace); + + uint64_t initial_size = std::filesystem::file_size(g_input_file); + spdlog::info("Initial DB file size: {} bytes", initial_size); + REQUIRE(initial_size > 0); + + rocprofvis_db_future_t object2wait = rocprofvis_db_future_alloc(progress_callback); + REQUIRE(nullptr != object2wait); + + rocprofvis_dm_result_t result = + rocprofvis_db_cleanup_async(m_db, object2wait, true); + REQUIRE(kRocProfVisDmResultSuccess == result); + + result = rocprofvis_db_future_wait(object2wait, UINT64_MAX); + REQUIRE(kRocProfVisDmResultSuccess == result); + + uint64_t final_size = std::filesystem::file_size(g_input_file); + spdlog::info("Final DB file size: {} bytes", final_size); + + rocprofvis_db_future_free(object2wait); + } + + // Reports memory footprint and frees the trace handle. + // Fixture Reads: m_trace + SECTION("Delete Trace") + { + CheckMemoryFootprint(m_trace); + PrintHeader("Delete trace"); + rocprofvis_dm_delete_trace(m_trace); + } +} diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index b36890b33..db7d5589a 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -4,6 +4,20 @@ find_package(Vulkan REQUIRED) set(VULKAN_LIB "Vulkan::Vulkan") +option(PROFILER_HUB_BUILD_ROC_OPTIQ_IFACE_LIB "Build RocOptiq interface library" OFF) + +if(PROFILER_HUB_BUILD_ROC_OPTIQ_IFACE_LIB) + message( + STATUS + "Build RocOptiq interface library" + ) + +set(PROFILER_HUB_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(PROFILER_HUB_BUILD_BENCHMARKS OFF CACHE BOOL "" FORCE) +add_subdirectory(profiler-hub) +add_subdirectory(profiler-hub-client/optiq) +set(PROFILER_HUB_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/profiler-hub/include) +endif() add_subdirectory(spdlog) add_subdirectory(catch2) diff --git a/thirdparty/profiler-hub-client/optiq/CMakeLists.txt b/thirdparty/profiler-hub-client/optiq/CMakeLists.txt new file mode 100644 index 000000000..d9ac6ba36 --- /dev/null +++ b/thirdparty/profiler-hub-client/optiq/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.16) + +project(optiq_interface VERSION 1.0.0 LANGUAGES CXX) + +# Create the static library +add_library(optiq_interface STATIC + profiler_hub_interface.cpp + profiler_hub_trace.cpp + profiler_hub_future.cpp +) + +# Set include directories so consumers can find the headers +target_include_directories(optiq_interface + PRIVATE + ${PROFILER_HUB_INCLUDE_DIR} + PUBLIC + $ + $ +) + +# Require C++17 (adjust as needed) +target_compile_features(optiq_interface PUBLIC cxx_std_17) \ No newline at end of file diff --git a/thirdparty/profiler-hub-client/optiq/profiler_hub_client_interface.h b/thirdparty/profiler-hub-client/optiq/profiler_hub_client_interface.h new file mode 100644 index 000000000..58d9b6f7f --- /dev/null +++ b/thirdparty/profiler-hub-client/optiq/profiler_hub_client_interface.h @@ -0,0 +1,269 @@ +#pragma once + +#include "profiler_hub_interface_types.h" + +namespace profiler_hub::client::interface +{ + + extern "C" + { + // report a single trace instance. Usually based on GUID (for rocpd 3.x and later), otherwise just synthesized single instance. + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // id - instance index + // file - which file the instance comes from, in case of multi-file trace + // uuid - UUID of the instance, if applicable + profiler_hub_result_t AddInstance( + client_trace_handle_t trace, + profiler_hub_instance_id_t id, + profiler_hub_string_t file, + profiler_hub_string_t uuid + ); + + // helps to collect strings table for a trace. It's better to replace any string passed to data-model in future requests with integer id. To do so profiler-hub must have keep remapping look-up tables. + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // string - a string from any column + // string_id - string index, expected to start with 0 when first string submitted, but can be re-purposed. The idea is data-model will match any string id from future calls to the strin in the table. + profiler_hub_result_t AddString( + client_trace_handle_t trace, + profiler_hub_string_t string, + uint32_t string_id + ); + + // This will convert profiler-hub track representation to data-model + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // instance - what trace instance the track belong to + // category - track category, as enumerated in profiler_hub_track_category_t + // track_id - expected to be sequential number, but can be re-mapped if needed + // track_name - optional track name + // node_id - full node id from database, 0 if not applicable + // node_name - node name if applicable + // process_id - full process id, not table index + // process_name - process name, if applicable + // thread_id - full thread id, not table index + // thread_name - thread name, if applicable + // stream_id - stream id + // thread_name - stream name, if applicable + // agent_type - agent type as defined in profiler_hub_agent_type_t + // agent_id - agent typed id + // agent_name - agent name, if applicable + // queue_id - queue id + // queue_name - queue name, if applicable + // counter_id - counter id (for rocpd 3.x it will be pmc info id, for original rocpd schema - synthesized, for perfetto - counter track id) + // counter_name - counter name, required, if counter track + // records_count - how many records the track have + // min_timestamp - minimum timestamp + // max_timestamp - maximum timestamp + // min_level_or_value - minimum level for event track - expected 0, minimum value for counter track + // max_level_or_value - maximum level for event track, maximum value for counter track + profiler_hub_result_t AddTrack( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_track_id_t track_id, + profiler_hub_string_t track_name, + profiler_hub_track_category_t category, + profiler_hub_optional_int_t node_id, + profiler_hub_string_t node_name, + profiler_hub_optional_int_t process_id, + profiler_hub_string_t process_name, + profiler_hub_optional_int_t thread_id, + profiler_hub_string_t thread_name, + profiler_hub_optional_int_t stream_id, + profiler_hub_string_t stream_name, + profiler_hub_agent_type_t agent_type, + profiler_hub_optional_int_t agent_id, + profiler_hub_string_t agent_name, + profiler_hub_optional_int_t queue_id, + profiler_hub_string_t queue_name, + profiler_hub_optional_int_t counter_id, + profiler_hub_string_t counter_name, + uint32_t records_count, + uint64_t min_timestamp, + uint64_t max_timestamp, + double min_level_or_value, + double max_level_or_value + ); + + // Adds histogram bucket to track histogram + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // track_id - track id, the combination of trace+track_id can be replaced with track_handler, but then AddTrack has to return the handler and profiler-hub must keep it for reference + // bucket_number - bucket number, no need to send zero count buckets, so the value is not sequential + // events_count - number of events in the bucket, skip zero event buckets + // bucket_value - average value for counter sample buckets, for events can be max level (tbd) + profiler_hub_result_t AddTrackHistogramBucket( + client_trace_handle_t trace, + profiler_hub_track_id_t track_id, + uint32_t bucket_number, + uint32_t events_count, + double bucket_value + ); + + // Reports info property to a caller + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // category - can be table name from where extended data has been taken, or synthesized + // name - can be column name from where extended data has been taken, or synthesized + // type - SQL type, as described in profiler_hub_value_type_t + // instance - what trace instance the track belongs to + // value - value as a void pointer to a value , will be cast based on type + profiler_hub_result_t AddInfoProperty( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_string_t category, + profiler_hub_string_t name, + profiler_hub_value_type_t type, + profiler_hub_row_id_t row_id, + profiler_hub_value_handle_t value + ); + + + // Add Event record to time slice container + // container - time slice container handle + // operation - event operation, as described in profiler_hub_event_operation_t + // event_id - event id relative to operation + // timestamp - start time of the event + // duration - event duration + // category_id - index of category in string table, be ready to remap database table string index into data-model string table + // symbol_id - index of symbol in string table, be ready to remap database table string index into data-model string table + // level - event level for event stacking. The same level should be used to generate call stack trace. It seems to be most reliable and universal way, unlike parent_id/parent_stack_id combination + profiler_hub_result_t AddEventRecord( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_timeslice_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id, + uint64_t timestamp, + uint64_t duration, + profiler_hub_string_id_t category_id, + profiler_hub_string_id_t symbol_id, + profiler_hub_event_level_t level + ); + + // Add PMC record to time slice container + // container - time slice container handle + // timestamp - start time of the counter sample + // value - counter sample value + profiler_hub_result_t AddPmcRecord( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_timeslice_handle_t container, + uint64_t timestamp, + double value + ); + + // Add an empty row to table processor container + // container - table processor container handle + profiler_hub_table_row_handle_t AddTableRowContainer( + client_trace_handle_t trace, + profiler_hub_table_handle_t container, + size_t num_columns + ); + + + // Add table cell value + // container - table row container handle + // column_index - column index + // column_name - column name + // column_type - column value type + // value - value as string + profiler_hub_result_t AddTableCell( + client_trace_handle_t trace, + profiler_hub_table_row_handle_t container, + uint32_t column_index, + profiler_hub_string_t column_name, + profiler_hub_value_type_t column_type, + profiler_hub_value_handle_t value + ); + + + + // Add flow trace endpoint + // container - flow trace container handle + // operation - event operation, as described in profiler_hub_event_operation_t + // event_id - event id relative to operation + // direction - incoming/outgoing + // timestamp - endpoint timestamp + // duration - endpoint duration + // category_id - endpoint category id + // symbol_id - endpoint symbol id + // level - endpoint level + profiler_hub_result_t AddEventDataFlowEndPoint( + client_trace_handle_t trace, + profiler_hub_flowtrace_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id, + profiler_hub_track_id_t track_id, + profiler_hub_flow_direction_t direction, + uint64_t timestamp, + uint64_t duration, + profiler_hub_string_id_t category_id, + profiler_hub_string_id_t symbol_id, + profiler_hub_event_level_t level + ); + + // Add extended data record + // container - extended data container + // category - data category. Can be table name + // name - can be column name + // type - SQL type, as described in profiler_hub_value_type_t + // value - pointer to a value of type + profiler_hub_result_t AddEventExtendedInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_string_t category, + profiler_hub_string_t name, + profiler_hub_value_type_t type, + profiler_hub_string_t value + ); + + // Add essential data to extended data table + // container - extended data container + // track_id - event track id + // stream_track_id - event stream track id, set to -1 if event does not belong to any stream + // level - event level + // stream_level - event level on stream track, set to -1 if event doesn't belong to any stream + profiler_hub_result_t AddEventEssentialInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_track_id_t track_id, + profiler_hub_track_id_t stream_track_id, + profiler_hub_event_level_t level, + profiler_hub_event_level_t stream_level + ); + + // Add event arguments info + // container - extended data container + // position - argument position + // name - argument name + // type - argument type + profiler_hub_result_t AddEventArgumentsInfo( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + uint32_t position, + profiler_hub_string_t name, + profiler_hub_string_t type, + profiler_hub_string_t value + ); + + + // Add call stack frame to container + // container - call stack container + // symbol - stack frame symbol + // file - stack frame file + // line - stack frame code line + // depth - stack frame depth + profiler_hub_result_t AddEventCallStackFrame( + client_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_call_stack_handle_t container, + profiler_hub_string_t function, + profiler_hub_string_t file, + profiler_hub_string_t line, + profiler_hub_string_t address, + uint32_t depth + ); + + } + +} // namespace ProfilerHub \ No newline at end of file diff --git a/thirdparty/profiler-hub-client/optiq/profiler_hub_future.cpp b/thirdparty/profiler-hub-client/optiq/profiler_hub_future.cpp new file mode 100644 index 000000000..de80bc05d --- /dev/null +++ b/thirdparty/profiler-hub-client/optiq/profiler_hub_future.cpp @@ -0,0 +1,96 @@ +#include "profiler_hub_future.hpp" +#include "profiler_hub_missing.hpp" + +#include +#include +#include + +namespace profiler_hub +{ + +// Module-level counter – each future_t gets a unique id at construction time. +static std::atomic s_next_id{1}; + + // ----------------------------------------------------------------------------- + // Construction / destruction + // ----------------------------------------------------------------------------- + + future_t::future_t(progress_callback_t progress_callback, void* user_data) + : m_progress_callback(progress_callback) + , m_user_data(user_data) + , m_id(s_next_id.fetch_add(1, std::memory_order_relaxed)) + , m_progress(0.0) + , m_interrupt_status(false) + , m_future(m_promise.get_future()) + {} + + future_t::~future_t() + { + // Ask a running worker to stop before we block on join. + set_interrupted(); + if (m_worker.joinable()) + m_worker.join(); + } + + // ----------------------------------------------------------------------------- + // Synchronization + // ----------------------------------------------------------------------------- + + profiler_hub_result_t + future_t::wait_for_completion(uint64_t timeout_ms) + { + const auto wait_status = + m_future.wait_for(std::chrono::milliseconds(timeout_ms)); + + if (wait_status == std::future_status::timeout) + { + set_interrupted(); + return profiler_hub_result_t::kProfilerHubStatusTimeout; + } + + return m_future.get(); + } + + profiler_hub_result_t + future_t::set_promise(profiler_hub_result_t status) + { + try + { + m_promise.set_value(status); + } + catch (const std::future_error&) + { + // Promise was already fulfilled (e.g. called twice) – report error. + return profiler_hub_result_t::kProfilerHubStatusUnknownError; + } + return status; + } + + // ----------------------------------------------------------------------------- + // Interruption + // ----------------------------------------------------------------------------- + + void + future_t::set_interrupted() + { + m_interrupt_status.store(true, std::memory_order_release); + missing_t::function("query cancellation propagation"); + } + + // ----------------------------------------------------------------------------- + // Progress reporting + // ----------------------------------------------------------------------------- + + void + future_t::show_progress(const char* db_name, + double progress_percent, + const char* action, + profiler_hub_async_status_t status) + { + m_progress.store(progress_percent, std::memory_order_relaxed); + + if (m_progress_callback) + m_progress_callback(db_name, m_id, progress_percent, status, action, nullptr); + } + +} // namespace profiler_hub diff --git a/thirdparty/profiler-hub-client/optiq/profiler_hub_future.hpp b/thirdparty/profiler-hub-client/optiq/profiler_hub_future.hpp new file mode 100644 index 000000000..eee6f22a0 --- /dev/null +++ b/thirdparty/profiler-hub-client/optiq/profiler_hub_future.hpp @@ -0,0 +1,137 @@ +#pragma once + +#include "profiler_hub_interface_types.h" + +#include +#include +#include +#include + +// These types are assumed to be declared in the project's public API headers. +// Included here symbolically – replace with the real include path. +// progress_callback_t +// timeout_ms_t +// status_t +// charptr_t +// result_t + + +namespace profiler_hub +{ + + typedef void ( *progress_callback_t)( + const char * file_name, + uint64_t future_id, + double progress_percent, + profiler_hub_async_status_t status, + const char* status_message, + void* user_data + ); + + // Represents a handle to an asynchronous database operation. + // The caller creates a future_t, optionally attaches a progress callback, + // moves a worker std::thread into it, and later calls wait_for_completion(). + // The worker thread is responsible for calling set_promise() when it finishes + // and show_progress() as it advances. + class future_t + { + public: + // Constructs a future with an optional progress-reporting callback. + // @param progress_callback – function invoked on each progress update; may be nullptr. + // @param user_data – opaque pointer forwarded to every callback invocation. + future_t(progress_callback_t progress_callback, + void* user_data = nullptr); + + // Joins the worker thread if it is still running, then destroys the object. + ~future_t(); + + // Non-copyable; move is intentionally left undefined – ownership is exclusive. + future_t(const future_t&) = delete; + future_t& operator=(const future_t&) = delete; + + // ------------------------------------------------------------------------- + // Accessors + // ------------------------------------------------------------------------- + + // @return the progress callback supplied at construction (may be nullptr). + progress_callback_t get_progress_callback() const + { + return m_progress_callback; + } + + // @return the unique numeric identifier assigned to this future at construction. + uint64_t get_id() const { return m_id; } + + // @return current operation progress in the range [0.0, 100.0]. + double get_progress() const { return m_progress.load(std::memory_order_relaxed); } + + // ------------------------------------------------------------------------- + // Worker thread management + // ------------------------------------------------------------------------- + + // Takes ownership of the worker thread that carries out the async operation. + // Must be called before the first wait_for_completion() call. + // @param thread – r-value reference to the thread to adopt. + void set_worker(std::thread&& thread) { m_worker = std::move(thread); } + + // @return true while the worker thread is alive (joinable). + bool is_working() const { return m_worker.joinable(); } + + // ------------------------------------------------------------------------- + // Synchronisation + // ------------------------------------------------------------------------- + + // Blocks until the worker completes or the timeout elapses. + // On timeout the interrupted flag is set so the worker can react and exit early. + // @param timeout_ms – maximum wait time in milliseconds. + // @return the operation result reported by the worker, or a timeout status. + profiler_hub_result_t wait_for_completion(uint64_t timeout_ms); + + // Called by the worker thread to deliver its final operation status. + // Safe to call at most once; subsequent calls are silently ignored. + // @param status – result of the completed operation. + // @return the same status value that was stored, or an error code on failure. + profiler_hub_result_t set_promise(profiler_hub_result_t status); + + + // ------------------------------------------------------------------------- + // Interruption + // ------------------------------------------------------------------------- + + // @return true if the operation was cut short by wait_for_completion() timeout. + bool is_interrupted() const + { + return m_interrupt_status.load(std::memory_order_acquire); + } + + // Signals the worker thread that it should stop as soon as possible. + // Idempotent – safe to call multiple times. + void set_interrupted(); + + // ------------------------------------------------------------------------- + // Progress reporting + // ------------------------------------------------------------------------- + + // Updates the stored progress value and, if a callback was supplied, + // forwards the event to the caller. + // @param db_name – path to the database file being processed. + // @param step – progress percentage of the current operation step. + // @param action – human-readable description of the ongoing operation. + // @param status – current operation status. + void show_progress(const char* db_name, + double weight, + const char* action, + profiler_hub_async_status_t status); + + private: + progress_callback_t m_progress_callback; + void* m_user_data; + const uint64_t m_id; + std::atomic m_progress; + std::atomic m_interrupt_status; + std::thread m_worker; + std::promise m_promise; + std::future m_future; + }; + +} // namespace profiler_hub diff --git a/thirdparty/profiler-hub-client/optiq/profiler_hub_interface.cpp b/thirdparty/profiler-hub-client/optiq/profiler_hub_interface.cpp new file mode 100644 index 000000000..336adf9df --- /dev/null +++ b/thirdparty/profiler-hub-client/optiq/profiler_hub_interface.cpp @@ -0,0 +1,323 @@ +#include "profiler_hub_trace.h" +#include "profiler_hub_interface.h" +#include "profiler_hub_future.hpp" + + +namespace profiler_hub::interface +{ + + profiler_hub_future_handle_t FutureAlloc(progress_callback_t progress_callback) { + return new future_t(progress_callback, nullptr); + } + + profiler_hub_result_t FutureFree( + profiler_hub_future_handle_t future_handle + ) { + if (!future_handle) + return kProfilerHubStatusInvalidArgument; + future_t* future = static_cast(future_handle); + delete future; + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t FutureWait( + profiler_hub_future_handle_t future_handle, + uint64_t timeout_ms + ) { + if (!future_handle) + return kProfilerHubStatusInvalidArgument; + future_t* future = static_cast(future_handle); + return future->wait_for_completion(timeout_ms); + } + + profiler_hub_result_t FutureCancel( + profiler_hub_future_handle_t future_handle + ) { + if (!future_handle) + return kProfilerHubStatusInvalidArgument; + future_t* future = static_cast(future_handle); + future->set_interrupted(); + return kProfilerHubStatusSuccess; + } + + profiler_hub_db_type_t DetectTrace( + profiler_hub_string_t trace_file_path + ) { + + return profiler_hub_trace_t::detect_trace(trace_file_path); + } + + profiler_hub_trace_handle_t OpenTrace( + profiler_hub_string_t trace_file_path + ) + { + profiler_hub_trace_t* ph_trace = new profiler_hub_trace_t(trace_file_path); + return (profiler_hub_trace_handle_t)ph_trace; + } + + profiler_hub_result_t SetTraceProperties( + profiler_hub_trace_handle_t trace, + client_trace_handle_t client_trace, + profiler_hub_string_t config_dir_path, + size_t histogram_bucket_count + ) + { + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + ph_trace->set_trace_properties(client_trace, config_dir_path, histogram_bucket_count); + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t ReadTraceMetadata( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace + ) + { + future_t* future = static_cast(future_handle); + if (future == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + std::thread worker([ph_trace, future]() + { + future->set_promise(ph_trace->open_trace(future)); + }); + future->set_worker(std::move(worker)); + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t CloseTrace( + profiler_hub_trace_handle_t trace + ) { + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + delete ph_trace; + return kProfilerHubStatusSuccess; + } + + + profiler_hub_result_t GetTimeSlice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end + ) { + future_t* future = static_cast(future_handle); + if (future == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + std::thread worker([ph_trace, future, track_id, slice_container, timestamp_start, timestamp_end]() + { + future->set_promise(ph_trace->get_time_slice(future, track_id, slice_container, timestamp_start, timestamp_end)); + }); + future->set_worker(std::move(worker)); + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t GetPmcTimeSlice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end, + bool left_neighbor, + bool right_neighbor + ) { + future_t* future = static_cast(future_handle); + if (future == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + std::thread worker([ph_trace, future, track_id, slice_container, timestamp_start, timestamp_end, left_neighbor, right_neighbor]() + { + future->set_promise(ph_trace->get_pmc_time_slice(future, track_id, slice_container, timestamp_start, timestamp_end, left_neighbor, right_neighbor)); + }); + future->set_worker(std::move(worker)); + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t GetTableTimeSlice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_table_handle_t table_handle, + profiler_hub_track_id_t track_id, + uint64_t timestamp_start, + uint64_t timestamp_end + ) { + future_t* future = static_cast(future_handle); + if (future == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + std::thread worker([ph_trace, future, table_handle, track_id, timestamp_start, timestamp_end]() + { + future->set_promise(ph_trace->get_table_time_slice(future, table_handle, track_id, timestamp_start, timestamp_end)); + }); + future->set_worker(std::move(worker)); + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t GetSearchTimeSlice( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_table_handle_t table_handle, + profiler_hub_event_operation_t operation, + uint64_t timestamp_start, + uint64_t timestamp_end, + size_t num_search_strings, + profiler_hub_search_strings_t string_filters + ) { + future_t* future = static_cast(future_handle); + if (future == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + std::thread worker([ph_trace, future, table_handle, operation, timestamp_start, timestamp_end, num_search_strings, string_filters, instance]() + { + future->set_promise(ph_trace->get_search_time_slice(future, instance, table_handle, operation, timestamp_start, timestamp_end, num_search_strings, string_filters)); + }); + future->set_worker(std::move(worker)); + return kProfilerHubStatusSuccess; + + } + + profiler_hub_result_t GetEventDataFlow( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_flowtrace_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id + ) { + future_t* future = static_cast(future_handle); + if (future == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + std::thread worker([ph_trace, future, instance, container, operation, event_id]() + { + future->set_promise(ph_trace->get_data_flow_for_event(future, instance, container, operation, event_id)); + }); + future->set_worker(std::move(worker)); + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t GetEventExtendedData( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id + ) { + future_t* future = static_cast(future_handle); + if (future == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + std::thread worker([ph_trace, future, instance, container, operation, event_id]() + { + future->set_promise(ph_trace->get_event_details(future, instance, container, operation, event_id)); + }); + future->set_worker(std::move(worker)); + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t GetEventStackTrace( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_call_stack_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id + ) { + future_t* future = static_cast(future_handle); + if (future == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + std::thread worker([ph_trace, future, instance, container, operation, event_id]() + { + future->set_promise(ph_trace->get_event_stack_trace(future, instance, container, operation, event_id)); + }); + future->set_worker(std::move(worker)); + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t TrimTraceDatabase( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + uint64_t timestamp_start, + uint64_t timestamp_end, + profiler_hub_string_t new_path) + { + future_t* future = static_cast(future_handle); + if (future == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + profiler_hub_trace_t* ph_trace = static_cast(trace); + if (ph_trace == nullptr) + { + return kProfilerHubStatusInvalidArgument; + } + std::thread worker([ph_trace, future, timestamp_start, timestamp_end, new_path]() + { + future->set_promise(ph_trace->trim_trace_database(future, timestamp_start, timestamp_end, new_path)); + }); + future->set_worker(std::move(worker)); + return kProfilerHubStatusSuccess; + } + +} diff --git a/thirdparty/profiler-hub-client/optiq/profiler_hub_interface.h b/thirdparty/profiler-hub-client/optiq/profiler_hub_interface.h new file mode 100644 index 000000000..9f24194c8 --- /dev/null +++ b/thirdparty/profiler-hub-client/optiq/profiler_hub_interface.h @@ -0,0 +1,193 @@ +#pragma once + +#include "profiler_hub_future.hpp" + +namespace profiler_hub::interface +{ + + extern "C" + { + // Allocate future object for asynchronous operations. + // Returns future object handle + profiler_hub_future_handle_t FutureAlloc(progress_callback_t progress_callback); + + // Delete future object after asynchronous operation is completed + profiler_hub_result_t FutureFree( + profiler_hub_future_handle_t future + ); + + // Wait for asynchronous operation to complete. + // Returns completion status + // timeout in milliseconds + profiler_hub_result_t FutureWait( + profiler_hub_future_handle_t future, + uint64_t timeout_ms + ); + + // Cancel asynchronous operation + profiler_hub_result_t FutureCancel( + profiler_hub_future_handle_t future + ); + + // Detects if trace file format is supported by profiler hub + // trace_file_path - path to the trace file + profiler_hub_db_type_t DetectTrace( + profiler_hub_string_t trace_file_path + ); + // Opens trace, read and compile metadata. + // This method should build tracks, collect node/instance information, cache information tables, calculate levels, build histograms, etc. + // client_trace - client trace reference + // trace_file_path - path to the trace file + + profiler_hub_trace_handle_t OpenTrace( + profiler_hub_string_t trace_file_path + ); + + // Set trace properties, just a setter for trace parameters, historically separated from constructor + // config_dir_path - path to the location temporary files should be stored + // histogram_bucket_count - caller should decide density of the histogram + profiler_hub_result_t SetTraceProperties( + profiler_hub_trace_handle_t trace, + client_trace_handle_t client_trace, + profiler_hub_string_t config_dir_path, + size_t histogram_bucket_count + ); + + // Read trace + profiler_hub_result_t ReadTraceMetadata( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace + ); + + // Close trace, destroy trace-related objects + profiler_hub_result_t CloseTrace( + profiler_hub_trace_handle_t trace + ); + + + // event time slice request + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // track_id - it seems most practical to query time slice for single track, this method does not take instance id, because a single track must belong to single trace instance + // timestamp_start - time-slice start + // timestamp_end - time-slice end + profiler_hub_result_t GetTimeSlice( + profiler_hub_future_handle_t future, + profiler_hub_trace_handle_t trace, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end + ); + + // performance time slice request + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // track_id - it seems most practical to query time slice for single track, this method does not take instance id, because a single track must belong to single trace instance + // timestamp_start - time-slice start + // timestamp_end - time-slice end + // left_neighbor - start from sample to the left of timestamp_start + // right_neighbor - take extra sample to the right of timestamp_end + profiler_hub_result_t GetPmcTimeSlice( + profiler_hub_future_handle_t future, + profiler_hub_trace_handle_t trace, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end, + bool left_neghbor, + bool right_neighbor + ); + + + // get events data for the Table view. The rows will be post processed, aligned, grouped, filtered in the data-model + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // table_handle - data-model table processor instance handle. + // track_id - table data is requested per track. + // timestamp_start - time-slice start + // timestamp_end - time-slice end + profiler_hub_result_t GetTableTimeSlice( + profiler_hub_future_handle_t future, + profiler_hub_trace_handle_t trace, + profiler_hub_table_handle_t table_handle, + profiler_hub_track_id_t track_id, + uint64_t timestamp_start, + uint64_t timestamp_end + ); + + // search for events by provided string filters + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // table_handle - data-model table processor instance handle. + // num_operations - length of operations list. The search is done per operation (e.g. region/kernel/memalloc/memcopy) or list of operations + // operations - list of operations + // timestamp_start - time-slice start + // timestamp_end - time-slice end + // num_search_strings - number of strings in the string filter + // string_filters - array of string to filter events + profiler_hub_result_t GetSearchTimeSlice( + profiler_hub_future_handle_t future, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_table_handle_t table_handle, + profiler_hub_event_operation_t operation, + uint64_t timestamp_start, + uint64_t timestamp_end, + size_t num_search_strings, + profiler_hub_search_strings_t string_filters + ); + + // get data flow end-point events + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // instance - multi-node instance, determined by GUID + // operation - operation type corresponds to specific event table + // event_id - event id. Historically event_id represents primary key relative to a specific event table + profiler_hub_result_t GetEventDataFlow( + profiler_hub_future_handle_t future, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_flowtrace_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id + ); + + // get extended data properties + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // instance - multi-node instance, determined by GUID + // operation - operation type corresponds to specific event table + // event_id - event id. Historically event_id represents primary key relative to a specific event table + profiler_hub_result_t GetEventExtendedData( + profiler_hub_future_handle_t future, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id + ); + + // get event call stack trace + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // instance - multi-node instance, determined by GUID + // operation - operation type corresponds to specific event table + // event_id - event id. Historically event_id represents primary key relative to a specific event table + profiler_hub_result_t GetEventStackTrace( + profiler_hub_future_handle_t future, + profiler_hub_trace_handle_t trace, + profiler_hub_instance_id_t instance, + profiler_hub_call_stack_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id + ); + + // trim trace database to time range specified in parameters + // trace - handle of a trace, considering single profiler hub instance handles multiple traces. + // timestamp_start - trim start + // timestamp_end - trim end + // new_path - file path to save trimmed database + profiler_hub_result_t TrimTraceDatabase( + profiler_hub_future_handle_t future_handle, + profiler_hub_trace_handle_t trace, + uint64_t timestamp_start, + uint64_t timestamp_end, + profiler_hub_string_t new_path); + + } + +} // namespace ProfilerHub \ No newline at end of file diff --git a/thirdparty/profiler-hub-client/optiq/profiler_hub_interface_types.h b/thirdparty/profiler-hub-client/optiq/profiler_hub_interface_types.h new file mode 100644 index 000000000..aca577754 --- /dev/null +++ b/thirdparty/profiler-hub-client/optiq/profiler_hub_interface_types.h @@ -0,0 +1,112 @@ +#pragma once + +#include +#include +#include + +typedef void* profiler_hub_handle_t; +typedef profiler_hub_handle_t profiler_hub_trace_handle_t; +typedef profiler_hub_handle_t client_trace_handle_t; +typedef profiler_hub_handle_t profiler_hub_future_handle_t; +typedef profiler_hub_handle_t profiler_hub_value_handle_t; +typedef profiler_hub_handle_t profiler_hub_timeslice_handle_t; +typedef profiler_hub_handle_t profiler_hub_flowtrace_handle_t; +typedef profiler_hub_handle_t profiler_hub_ext_data_handle_t; +typedef profiler_hub_handle_t profiler_hub_table_handle_t; +typedef profiler_hub_handle_t profiler_hub_table_row_handle_t; +typedef profiler_hub_handle_t profiler_hub_call_stack_handle_t; +typedef const char* profiler_hub_string_t; +typedef const void** profiler_hub_optional_t; +typedef const char** profiler_hub_optional_string_t; +typedef const char** profiler_hub_search_strings_t; +typedef uint64_t* profiler_hub_optional_int_t; +typedef double* profiler_hub_optional_double_t; +typedef uint32_t profiler_hub_instance_id_t; +typedef uint32_t profiler_hub_track_id_t; +typedef uint64_t profiler_hub_event_id_t; +typedef uint32_t profiler_hub_string_id_t; +typedef uint32_t profiler_hub_event_level_t; +typedef uint64_t profiler_hub_row_id_t; + +// Error status, the list to be updated during development +typedef enum profiler_hub_result_t { + kProfilerHubStatusSuccess, + kProfilerHubStatusUnknownError, + kProfilerHubStatusNotSupported, + kProfilerHubStatusInvalidArgument, + kProfilerHubStatusTimeout, + kProfilerHubStatusNotLoaded, +} profiler_hub_result_t; + +typedef enum profiler_hub_async_status_t { + kProfilerHubAsyncSuccess, + kProfilerHubAsyncFailed, + kProfilerHubAsyncBusy +} profiler_hub_async_status_t; + +// value types, inherited from SQL types +typedef enum profiler_hub_value_type_t +{ + kPprofilerHubDataTypeUndefined = 0, + kPprofilerHubDataTypeInt = 1, + kPprofilerHubDataTypeDouble = 2, + kPprofilerHubDataTypeString = 3, + kPprofilerHubDataTypeBlob = 4, + kPprofilerHubDataTypeNull = 5 +} profiler_hub_value_type_t; + +// track categories +typedef enum profiler_hub_track_category_t { + + kPprofilerHubCategoryUndefined = 0, + kPprofilerHubCategoryRegionInstrumented, + kPprofilerHubCategoryRegionSampled, + kPprofilerHubCategoryKernelDispatch, + kPprofilerHubCategoryMemoryAllocate, + kPprofilerHubCategoryMemoryCopy, + kPprofilerHubCategoryPerformanceCounter, + kPprofilerHubCategoryStream, +} profiler_hub_track_category_t; + +// event operations +typedef profiler_hub::reader_types::event_type_t profiler_hub_event_operation_t; + +// property category will determine which info table or join read the property from +// all the info tables are supposed to be cached +typedef enum profiler_hub_property_category_t{ + kProfilerHubPropertyNode, + kProfilerHubPropertyProcess, + kProfilerHubPropertyAgent, + kProfilerHubPropertyThread, + kProfilerHubPropertyQueue, + kProfilerHubPropertyStream, + kProfilerHubPropertyCounter, + +} profiler_hub_property_category_t; + +typedef enum profiler_hub_agent_type_t{ + kProfilerHubNotAgent, + kProfilerHubAgentCPU, + kProfilerHubAgentGPU, + kProfilerHubAgentNIC, +} profiler_hub_agent_type_t; + +typedef enum profiler_hub_flow_direction_t +{ + kProfilerHubDirectionOutgoing, + kProfilerHubDirectionIncoming, +} profiler_hub_flow_direction_t; + +// Database type +typedef enum profiler_hub_db_type_t { + // not supported by profiler hub + kDbNotSupported, + // supported by profiler hub + kDbSupported, + +} profiler_hub_db_type_t; + +typedef enum profiler_hub_string_source_t { + kProfilerHubStringNameOrCategory, + kProfilerHubStringKernelSymbol, +} profiler_hub_string_source_t; diff --git a/thirdparty/profiler-hub-client/optiq/profiler_hub_missing.hpp b/thirdparty/profiler-hub-client/optiq/profiler_hub_missing.hpp new file mode 100644 index 000000000..8346e75ca --- /dev/null +++ b/thirdparty/profiler-hub-client/optiq/profiler_hub_missing.hpp @@ -0,0 +1,224 @@ +#pragma once + +#include "profiler_hub_interface_types.h" +#include +#include +#include +#include +#include + + +namespace profiler_hub::missing_types +{ + /// A column descriptor: carries the name and the uniform type for all non-null cells in + /// this column. Positional index within table_t::columns is the column's identity. + struct table_column_t + { + std::string name{}; ///< Column identifier. + profiler_hub_value_type_t type{}; ///< Declared type; all non-null cells must conform. + }; + + using table_column_list_t = std::vector; + + /// A single cell value. nullptr_t = null (absent); otherwise the held alternative must + /// conform to the type declared by the corresponding table_column_t. monostate is not + /// used here — a cell is either a typed value or null, never present-but-empty. + using table_cell_value_t = std::variant; + + struct table_cell_t + { + table_cell_value_t value{ nullptr }; ///< Null by default; set to a typed value when populated. + }; + + using table_cell_list_t = std::vector; + + /// A single row in a table_t. cells[i] corresponds to columns[i] in the parent table. + /// A null cell (nullptr_t) is valid for any column type. + struct table_row_t + { + size_t row_index{}; ///< Zero-based position of this row in the table. + table_cell_list_t cells{}; ///< One entry per column, positionally aligned. + }; + + using table_row_ptr_t = std::shared_ptr; + using table_row_list_t = std::vector; + + /// A rectangular, typed table. Column identity (name + type) lives in columns; + /// cell values in each row are positionally aligned to columns and must conform + /// to the declared column type unless null. + struct table_t + { + std::string name{}; ///< Optional display name for the table. + table_column_list_t columns{}; ///< Ordered column descriptors (name + type). + table_row_list_t rows{}; ///< Ordered row set, each row aligned to columns. + std::string extdata{}; + }; + + using table_ptr_t = std::shared_ptr; + using table_list_t = std::vector; + + struct event_filter_t + { + reader_types::time_window_t time_window; ///< Time range filter + reader_types::pagination_t pagination; ///< Limit/offset for chunking + std::optional sort{ std::nullopt }; ///< Sort order + + /// Which event types to include (empty = all) + reader_types::event_type_t type; + std::vector search_strings; + profiler_hub_instance_id_t instance; + }; +} + +namespace profiler_hub +{ + + class trace_instance_t; + class track_histogram_bucket_t; + +// Exception type raised by missing_t methods. +// Carries a human-readable description of the unimplemented feature. +class missing_error_t : public std::runtime_error +{ +public: + explicit missing_error_t(std::string_view description) + : std::runtime_error(std::string(description)) + {} +}; + +// Placeholder class whose methods throw missing_error_t. +// Add one method per unimplemented feature; call throw_missing() with a +// clear description of what is absent so callers get an actionable message. +class missing_t +{ +public: + + static std::vector get_trace_instances() + { + throw_missing("multi-trace instances"); + return {}; + } + + static std::unordered_map get_trace_string_table() + { + // string table may have strings from different sqlite tables, e.g. rocpd_string and rocpd_info_kernel_symbol, maybe more + // put all the strings into single string table, then do proper remapping when send string id to a client + + throw_missing("string table"); + return {}; + } + + static profiler_hub_db_type_t detect_trace(std::string& file_name) + { + throw_missing("trace detection"); + return profiler_hub_db_type_t::kDbNotSupported; + } + + static std::vector get_track_histogram(profiler_hub::reader_types::track_info_ptr_t & track, size_t histogram_bucket_count) + { + throw_missing("track histogram"); + return {}; + } + + static profiler_hub_result_t trim_trace_database(uint64_t timestamp_start, uint64_t timestamp_end, profiler_hub_string_t new_path) { + throw_missing("trimming trace database"); + return kProfilerHubStatusNotSupported; + } + + static reader_types::counter_timeline_event_list_t missing_t::get_counter_events_for_track( + reader_types::track_info_ptr_t track, + const reader_types::event_filter_t& filter, + bool left_neighbor, + bool right_neighbor) + { + throw_missing("performance counter events"); + return {}; + } + + static missing_types::table_ptr_t get_event_table( + missing_types::event_filter_t& filter, + size_t start, + size_t stop + ) { + throw_missing("event table"); + return {}; + } + + static missing_types::table_ptr_t get_event_table_for_track( + profiler_hub::reader_types::track_info_ptr_t& track, + missing_types::event_filter_t& filter, + size_t start, + size_t stop + ) { + throw_missing("event table for track"); + return {}; + } + + static void function(std::string_view feature) + { + throw_missing(feature); + } + static uint64_t integer(std::string_view feature) + { + throw_missing(feature); + return 0; + } + static const char * string(std::string_view feature) + { + throw_missing(feature); + return ""; + } + + [[noreturn]] static void check_missing_client(void * client) + { + if (client == nullptr) + { + throw missing_error_t("fatal error: client trace cannot be null!"); + } + } + +protected: + // Throws missing_error_t with a standardised message. + // @param feature – short description of the missing feature or method. + [[noreturn]] static void throw_missing(std::string_view feature) + { + throw missing_error_t("not implemented: " + std::string(feature)); + } + + +}; + +class trace_instance_t +{ +public: + trace_instance_t(std::string& file, std::string& guid, uint32_t index) : + m_file(file), + m_guid(guid), + m_index(index) { + } + const char* get_file() { return m_file.c_str(); } + const char* get_guid() { return m_guid.c_str(); } + uint32_t get_index() { return m_index; } +private: + std::string m_file; + std::string m_guid; + uint32_t m_index; + +}; + +class track_histogram_bucket_t +{ +public: + track_histogram_bucket_t(uint32_t bucket_number, uint32_t events_count, double bucket_value) : + m_bucket_number(bucket_number), m_events_count(events_count), m_bucket_value(bucket_value) { + }; + uint32_t get_bucket_number() { return m_bucket_number; } + uint32_t get_events_count() { return m_events_count; } + double get_bucket_value() { return m_bucket_value; } +private: + uint32_t m_bucket_number; + uint32_t m_events_count; + double m_bucket_value; +}; + +} // namespace profiler_hub diff --git a/thirdparty/profiler-hub-client/optiq/profiler_hub_trace.cpp b/thirdparty/profiler-hub-client/optiq/profiler_hub_trace.cpp new file mode 100644 index 000000000..d2ce447fd --- /dev/null +++ b/thirdparty/profiler-hub-client/optiq/profiler_hub_trace.cpp @@ -0,0 +1,632 @@ +#include "profiler_hub_interface.h" +#include "profiler_hub_client_interface.h" +#include "profiler_hub_trace.h" +#include "profiler_hub_missing.hpp" +#include "profiler_hub_future.hpp" + + +namespace profiler_hub::interface +{ + profiler_hub_track_category_t profiler_hub_trace_t::get_track_category(profiler_hub::reader_types::track_type_t type, profiler_hub::reader_types::region_track_kind_t region_kind) + { + switch (type) + { + case profiler_hub::reader_types::track_type_t::cpu_thread: + if (region_kind == profiler_hub::reader_types::region_track_kind_t::main) + return kPprofilerHubCategoryRegionInstrumented; + else if (region_kind == profiler_hub::reader_types::region_track_kind_t::sample) + return kPprofilerHubCategoryRegionSampled; + else return kPprofilerHubCategoryUndefined; + case profiler_hub::reader_types::track_type_t::gpu_queue: return kPprofilerHubCategoryKernelDispatch; + case profiler_hub::reader_types::track_type_t::memory : return kPprofilerHubCategoryMemoryCopy; + case profiler_hub::reader_types::track_type_t::memory_activity : return kPprofilerHubCategoryMemoryAllocate; + case profiler_hub::reader_types::track_type_t::kernel_dispatch_pmc : return kPprofilerHubCategoryPerformanceCounter; + case profiler_hub::reader_types::track_type_t::stream : return kPprofilerHubCategoryStream; + default: return kPprofilerHubCategoryUndefined; + } + } + + profiler_hub_agent_type_t profiler_hub_trace_t::get_agent_type(std::string& type) + { + if (type == "CPU") + return kProfilerHubAgentCPU; + else if (type == "GPU") + return kProfilerHubAgentGPU; + else if (type == "NIC") + return kProfilerHubAgentNIC; + else + return kProfilerHubNotAgent; + } + + template + profiler_hub_result_t profiler_hub_trace_t::report_info_properties( + ListT& collection, + const char* category, + profiler_hub_instance_id_t instance_id, + const std::unordered_map>& properties_dispatcher) + { + for (auto it = collection.begin(); it != collection.end(); ++it) + { + size_t* id_value = nullptr; + auto entry = properties_dispatcher.find("id"); + if (entry != properties_dispatcher.end()) + { + auto type = entry->second(it, (profiler_hub_optional_t)&id_value); + if (type == kPprofilerHubDataTypeInt) + { + for (auto& [name, fn] : properties_dispatcher) + { + void* cell_value = nullptr; + auto type = fn(it, (profiler_hub_optional_t)&cell_value); + if (type != kPprofilerHubDataTypeUndefined) + { + auto result = client::interface::AddInfoProperty(m_client_trace, instance_id, category, name.c_str(), type, *id_value, cell_value); + if (result != kProfilerHubStatusSuccess) return result; + } + } + } + } + } + return kProfilerHubStatusSuccess; + } + + profiler_hub_result_t profiler_hub_trace_t::report_node_info(profiler_hub_instance_id_t instance_id) { + using it_t = profiler_hub::reader_types::node_info_list_t::iterator; + using fn_t = std::function; + static const std::unordered_map properties_dispatcher = + { + { "id", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->node_id; return kPprofilerHubDataTypeInt; }}, + { "hash", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->hash; return kPprofilerHubDataTypeInt; }}, + { "machine_id", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->machine_id.c_str(); return kPprofilerHubDataTypeString; }}, + { "system_name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->system_name.c_str(); return kPprofilerHubDataTypeString; }}, + { "hostname", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->hostname.c_str(); return kPprofilerHubDataTypeString; }}, + { "release", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->release.c_str(); return kPprofilerHubDataTypeString; }}, + { "version", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->version.c_str(); return kPprofilerHubDataTypeString; }}, + { "hardware_name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->hardware_name.c_str(); return kPprofilerHubDataTypeString; }}, + { "domain_name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->domain_name.c_str(); return kPprofilerHubDataTypeString; }}, + }; + return report_info_properties(get_reader()->get_all_nodes(), "Node", instance_id, properties_dispatcher); + } + + profiler_hub_result_t profiler_hub_trace_t::report_process_info(profiler_hub_instance_id_t instance_id) { + using it_t = profiler_hub::reader_types::process_info_list_t::iterator; + using fn_t = std::function; + static const std::unordered_map properties_dispatcher = + { + { "id", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->pid; return kPprofilerHubDataTypeInt; }}, + { "nid", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->node_info->node_id; return kPprofilerHubDataTypeInt; }}, + { "init", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->init.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->init; return kPprofilerHubDataTypeInt; }}, + { "fini", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->fini.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->fini; return kPprofilerHubDataTypeInt; }}, + { "start", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->start.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->start; return kPprofilerHubDataTypeInt; }}, + { "end", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->end.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->end; return kPprofilerHubDataTypeInt; }}, + { "command", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->command.c_str(); return kPprofilerHubDataTypeString; }}, + { "environment", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->environment.c_str(); return kPprofilerHubDataTypeString; }}, + { "extdata", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->extdata.c_str(); return kPprofilerHubDataTypeString; }}, + }; + return report_info_properties(get_reader()->get_all_processes(), "Process", instance_id, properties_dispatcher); + } + + profiler_hub_result_t profiler_hub_trace_t::report_agent_info(profiler_hub_instance_id_t instance_id) { + using it_t = profiler_hub::reader_types::agent_info_list_t::iterator; + using fn_t = std::function; + static const std::unordered_map properties_dispatcher = + { + { "id", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->id; return kPprofilerHubDataTypeInt; }}, + { "nid", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->node_info->node_id; return kPprofilerHubDataTypeInt; }}, + { "type_index", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->type_index; return kPprofilerHubDataTypeInt; }}, + { "absolute_index", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->absolute_index.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->absolute_index; return kPprofilerHubDataTypeInt; }}, + { "logical_index", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->logical_index.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->logical_index; return kPprofilerHubDataTypeInt; }}, + { "uuid", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->uuid.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->uuid; return kPprofilerHubDataTypeInt; }}, + { "type", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->agent_type.c_str(); return kPprofilerHubDataTypeString; }}, + { "name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->name.c_str(); return kPprofilerHubDataTypeString; }}, + { "model_name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->model_name.c_str(); return kPprofilerHubDataTypeString; }}, + { "vendor_name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->vendor_name.c_str(); return kPprofilerHubDataTypeString; }}, + { "product_name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->product_name.c_str(); return kPprofilerHubDataTypeString; }}, + { "user_name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->user_name.c_str(); return kPprofilerHubDataTypeString; }}, + { "extdata", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->extdata.c_str(); return kPprofilerHubDataTypeString; }}, + }; + return report_info_properties(get_reader()->get_all_agents(), "Agent", instance_id, properties_dispatcher); + } + + profiler_hub_result_t profiler_hub_trace_t::report_pmc_info(profiler_hub_instance_id_t instance_id) { + using it_t = profiler_hub::reader_types::pmc_info_list_t::iterator; + using fn_t = std::function; + static const std::unordered_map properties_dispatcher = + { + { "id", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->pmc_id; return kPprofilerHubDataTypeInt; }}, + { "nid", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->node_info->node_id; return kPprofilerHubDataTypeInt; }}, + { "pid", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->process_info->pid; return kPprofilerHubDataTypeInt; }}, + { "agent_id", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->agent_info->id; return kPprofilerHubDataTypeInt; }}, + { "event_code", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->event_code.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->event_code; return kPprofilerHubDataTypeInt; }}, + { "instance_id", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->instance_id.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->instance_id; return kPprofilerHubDataTypeInt; }}, + { "is_constant", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->is_constant.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->is_constant; return kPprofilerHubDataTypeInt; }}, + { "is_derived", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->is_derived.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->is_derived; return kPprofilerHubDataTypeInt; }}, + { "name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->name.c_str(); return kPprofilerHubDataTypeString; }}, + { "target_arch", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->target_arch.c_str(); return kPprofilerHubDataTypeString; }}, + { "symbol", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->symbol.c_str(); return kPprofilerHubDataTypeString; }}, + { "description", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->description.c_str(); return kPprofilerHubDataTypeString; }}, + { "long_description", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->long_description.c_str(); return kPprofilerHubDataTypeString; }}, + { "component", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->component.c_str(); return kPprofilerHubDataTypeString; }}, + { "units", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->units.c_str(); return kPprofilerHubDataTypeString; }}, + { "value_type", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->value_type.c_str(); return kPprofilerHubDataTypeString; }}, + { "block", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->block.c_str(); return kPprofilerHubDataTypeString; }}, + { "expression", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->expression.c_str(); return kPprofilerHubDataTypeString; }}, + { "extdata", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->extdata.c_str(); return kPprofilerHubDataTypeString; }}, + }; + return report_info_properties(get_reader()->get_all_pmc_info(), "PMC", instance_id, properties_dispatcher); + } + + profiler_hub_result_t profiler_hub_trace_t::report_thread_info(profiler_hub_instance_id_t instance_id) { + using it_t = profiler_hub::reader_types::thread_info_list_t::iterator; + using fn_t = std::function; + static const std::unordered_map properties_dispatcher = + { + { "id", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->thread_id; return kPprofilerHubDataTypeInt; }}, + { "nid", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->node_info->node_id; return kPprofilerHubDataTypeInt; }}, + { "pid", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->process_info->pid; return kPprofilerHubDataTypeInt; }}, + { "ppid", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->parent_process_id.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->parent_process_id; return kPprofilerHubDataTypeInt; }}, + { "start", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->start.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->start; return kPprofilerHubDataTypeInt; }}, + { "end", [](it_t it, profiler_hub_optional_t value) { if (!it->get()->end.has_value()) return kPprofilerHubDataTypeUndefined; *value = &it->get()->end; return kPprofilerHubDataTypeInt; }}, + { "name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->name.c_str(); return kPprofilerHubDataTypeString; }}, + { "extdata", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->extdata.c_str(); return kPprofilerHubDataTypeString; }}, + }; + return report_info_properties(get_reader()->get_all_threads(), "Thread", instance_id, properties_dispatcher); + } + + profiler_hub_result_t profiler_hub_trace_t::report_stream_info(profiler_hub_instance_id_t instance_id) { + using it_t = profiler_hub::reader_types::stream_info_list_t::iterator; + using fn_t = std::function; + static const std::unordered_map properties_dispatcher = + { + { "id", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->stream_id; return kPprofilerHubDataTypeInt; }}, + { "nid", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->node_info->node_id; return kPprofilerHubDataTypeInt; }}, + { "pid", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->process_info->pid; return kPprofilerHubDataTypeInt; }}, + { "name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->name.c_str(); return kPprofilerHubDataTypeString; }}, + { "extdata", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->extdata.c_str(); return kPprofilerHubDataTypeString; }}, + }; + return report_info_properties(get_reader()->get_all_streams(), "Stream", instance_id, properties_dispatcher); + } + + profiler_hub_result_t profiler_hub_trace_t::report_queue_info(profiler_hub_instance_id_t instance_id) { + using it_t = profiler_hub::reader_types::queue_info_list_t::iterator; + using fn_t = std::function; + static const std::unordered_map properties_dispatcher = + { + { "id", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->queue_id; return kPprofilerHubDataTypeInt; }}, + { "nid", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->node_info->node_id; return kPprofilerHubDataTypeInt; }}, + { "pid", [](it_t it, profiler_hub_optional_t value) { *value = &it->get()->process_info->pid; return kPprofilerHubDataTypeInt; }}, + { "name", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->name.c_str(); return kPprofilerHubDataTypeString; }}, + { "extdata", [](it_t it, profiler_hub_optional_t value) { *value = it->get()->extdata.c_str(); return kPprofilerHubDataTypeString; }}, + }; + return report_info_properties(get_reader()->get_all_queues(), "Queue", instance_id, properties_dispatcher); + } + + profiler_hub_db_type_t profiler_hub_trace_t::detect_trace(std::string trace_path) { + return missing_t::detect_trace(trace_path); + } + + profiler_hub_result_t profiler_hub_trace_t::open_trace(future_t * future) { + missing_t::check_missing_client(m_client_trace); + profiler_hub_result_t result = kProfilerHubStatusInvalidArgument; + // initialize and compile trace metadata + future->show_progress(m_trace_path.c_str(), 80, "Read trace", kProfilerHubAsyncBusy); + m_storage = std::make_unique(m_trace_path, ""); + m_reader = std::make_shared(std::move(m_storage)); + + future->show_progress(m_trace_path.c_str(), 1, "Add instances", kProfilerHubAsyncBusy); + auto& trace_instances = missing_t::get_trace_instances(); + for (auto& instance : trace_instances) + { + result = client::interface::AddInstance(m_client_trace, instance.get_index(), instance.get_file(), instance.get_guid()); + if (result != kProfilerHubStatusSuccess) + { + return result; + } + } + + future->show_progress(m_trace_path.c_str(), 1, "Add info properties", kProfilerHubAsyncBusy); + + for (auto& instance : trace_instances) + { + if (kProfilerHubStatusSuccess != report_node_info(instance.get_index())) break; + if (kProfilerHubStatusSuccess != report_process_info(instance.get_index())) break; + if (kProfilerHubStatusSuccess != report_thread_info(instance.get_index())) break; + if (kProfilerHubStatusSuccess != report_agent_info(instance.get_index())) break; + if (kProfilerHubStatusSuccess != report_queue_info(instance.get_index())) break; + if (kProfilerHubStatusSuccess != report_stream_info(instance.get_index())) break; + if (kProfilerHubStatusSuccess != report_pmc_info(instance.get_index())) break; + } + + future->show_progress(m_trace_path.c_str(), 1, "Add strings", kProfilerHubAsyncBusy); + + auto& string_table = missing_t::get_trace_string_table(); + for (auto& [id, string] : string_table) + { + result = client::interface::AddString(m_client_trace, string.c_str(), id); + if (result != kProfilerHubStatusSuccess) + { + return result; + } + } + + + future->show_progress(m_trace_path.c_str(), 3, "Add tracks", kProfilerHubAsyncBusy); + + auto & tracks = m_reader->get_tracks(); + for (auto& track : tracks) + { + result = client::interface::AddTrack( + m_client_trace, + missing_t::integer("track->instance_info->id"), + track->id.value, + track->name.c_str(), + get_track_category(track->type, track->region_kind), + &track->node_info->node_id, + track->node_info->hostname.c_str(), + &track->process_info->pid, + track->process_info->command.c_str(), + track->thread_info ? &track->thread_info->thread_id : nullptr, + nullptr, + track->stream_info ? &track->stream_info->stream_id : nullptr, + track->stream_info ? track->stream_info->name.c_str() : nullptr, + track->agent_info ? get_agent_type(track->agent_info->agent_type) : kProfilerHubNotAgent, + track->agent_info ? &track->agent_info->type_index : nullptr, + track->agent_info ? track->agent_info->name.c_str() : nullptr, + track->queue_info ? &track->queue_info->queue_id : nullptr, + track->queue_info ? track->queue_info->name.c_str() : nullptr, + track->pmc_info ? &track->pmc_info->pmc_id : nullptr, + track->pmc_info ? track->pmc_info->name.c_str() : nullptr, + missing_t::integer("track->records_count"), // records has to be counted using COUNT(*) + missing_t::integer("track->min_timestamp"), // min_timestamp has to be aggregated with MIN(start_ts) + missing_t::integer("track->max_timestamp"), // max_timestamp has to be aggregated with MAX(end_ts), + 0, // in case of counter track use MIN(value) + track->max_lane //in case of counter track use MAX(value) + ); + if (result != kProfilerHubStatusSuccess) + { + return result; + } + } + + future->show_progress(m_trace_path.c_str(), 3, "Add track histograms", kProfilerHubAsyncBusy); + for (auto& track : tracks) + { + auto & track_histogram = missing_t::get_track_histogram(track, m_histogram_bucket_count); + for (auto& bucket : track_histogram) + { + result = client::interface::AddTrackHistogramBucket(m_client_trace, track->id.value, bucket.get_bucket_number(), bucket.get_events_count(), bucket.get_bucket_value()); + if (result != kProfilerHubStatusSuccess) + { + return result; + } + } + } + return result; + } + + profiler_hub_result_t + profiler_hub_trace_t::get_time_slice( + future_t * future, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end) + { + if (!m_client_trace) + { + throw missing_error_t("fatal error: client trace cannot be null!"); + } + profiler_hub_result_t result = kProfilerHubStatusNotLoaded; + reader_types::event_filter_t filter = { {timestamp_start, timestamp_end} }; + auto& tracks = m_reader->get_tracks(); + auto it = std::find_if(tracks.begin(), tracks.end(), [track_id](reader_types::track_info_ptr_t track_info) {return track_info->id.value == track_id; }); + if (it != tracks.end()) + { + // toto: consider pushing data to client rigth away + auto& events = m_reader->get_events_for_track(*it, filter); + for (auto& event : events) + { + result = client::interface::AddEventRecord( + m_client_trace, + missing_t::integer("it->get()->instance_info->id"), + slice_container, + event.unique_identifier.type, + event.unique_identifier.id, + event.start_timestamp, + event.end_timestamp-event.start_timestamp, + missing_t::integer("event.category_id"), + missing_t::integer("event.symbol_id"), + missing_t::integer("event.level")); + if (result != kProfilerHubStatusSuccess) + { + return result; + } + } + } + return result; + } + + + profiler_hub_result_t + profiler_hub_trace_t::get_pmc_time_slice( + future_t * future, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end, + bool left_neighbor, + bool right_neighbor) + { + if (!m_client_trace) + { + throw missing_error_t("fatal error: client trace cannot be null!"); + } + profiler_hub_result_t result = kProfilerHubStatusNotLoaded; + reader_types::event_filter_t filter = { {timestamp_start, timestamp_end} }; + auto& tracks = m_reader->get_tracks(); + auto it = std::find_if(tracks.begin(), tracks.end(), [track_id](reader_types::track_info_ptr_t track_info) {return track_info->id.value == track_id; }); + if (it != tracks.end()) + { + auto& events = missing_t::get_counter_events_for_track(*it, filter, left_neighbor, right_neighbor); + for (auto& event : events) + { + result = client::interface::AddPmcRecord( + m_client_trace, + missing_t::integer("it->get()->instance_info->id"), + slice_container, + event.timestamp, + event.value); + if (result != kProfilerHubStatusSuccess) + { + return result; + } + } + } + return result; + } + + profiler_hub_result_t profiler_hub_trace_t::add_table_to_client(missing_types::table_ptr_t table, profiler_hub_table_handle_t client_table_handle) + { + profiler_hub_result_t result = kProfilerHubStatusNotLoaded; + for (int ri=0; ri < table->rows.size(); ri++) + { + if (table->rows[ri].cells.size() != table->columns.size()) + return kProfilerHubStatusUnknownError; + profiler_hub_table_row_handle_t row_handle = client::interface::AddTableRowContainer(m_client_trace, client_table_handle, table->columns.size()); + for (int ci = 0; ci < table->rows[ri].cells.size(); ci++) + { + size_t value_int; + double value_double; + std::string value_str; + const void* value_ptr = nullptr; + if (std::holds_alternative(table->rows[ri].cells[ci].value)) + { + value_str = std::get(table->rows[ri].cells[ci].value); + value_ptr = value_str.c_str(); + } else + if (std::holds_alternative(table->rows[ri].cells[ci].value)) + { + value_int = std::get(table->rows[ri].cells[ci].value); + value_ptr = &value_int; + } else + if (std::holds_alternative(table->rows[ri].cells[ci].value)) + { + value_double = std::get(table->rows[ri].cells[ci].value); + value_ptr = &value_double; + } + + result = client::interface::AddTableCell( + m_client_trace, + row_handle, + ci, + table->columns[ci].name.c_str(), + table->columns[ci].type, + (profiler_hub_value_handle_t)value_ptr); + if (result != kProfilerHubStatusSuccess) + { + return result; + } + } + } + return result; + } + + profiler_hub_result_t + profiler_hub_trace_t::get_table_time_slice( + future_t * future, + profiler_hub_table_handle_t client_table_handle, + profiler_hub_track_id_t track_id, + uint64_t timestamp_start, + uint64_t timestamp_end) + { + auto& tracks = m_reader->get_tracks(); + auto it = std::find_if(tracks.begin(), tracks.end(), [track_id](reader_types::track_info_ptr_t track_info) {return track_info->id.value == track_id; }); + if (it != tracks.end()) + { + missing_types::event_filter_t filter = { {timestamp_start, timestamp_end}, {}, {}, {}, {}, (profiler_hub_instance_id_t)missing_t::integer("it->get()->instance_info->id") }; + // todo : instead of reading the whole table, then adding it to client, add every row of the table as soon as you get it + // change add_table_to_client to add add_table_columns_to_client and add_table_row_to_client + auto& table = missing_t::get_event_table_for_track(*it, filter, timestamp_start, timestamp_end); + + return add_table_to_client(table, client_table_handle); + } + return kProfilerHubStatusNotLoaded; + } + + profiler_hub_result_t + profiler_hub_trace_t::get_search_time_slice( + future_t * future, + profiler_hub_instance_id_t instance, + profiler_hub_table_handle_t client_table_handle, + profiler_hub_event_operation_t operation, + uint64_t timestamp_start, + uint64_t timestamp_end, + size_t num_search_strings, + profiler_hub_search_strings_t string_filters) + { + std::vector search_strings; + for (int i = 0; i < num_search_strings; i++) + { + search_strings.push_back(string_filters[i]); + } + missing_types::event_filter_t filter = { {timestamp_start, timestamp_end}, {}, {}, operation, search_strings, instance }; + + auto& table = missing_t::get_event_table(filter, timestamp_start, timestamp_end); + return add_table_to_client(table, client_table_handle); + } + + profiler_hub_result_t profiler_hub_trace_t::get_data_flow_for_event( + future_t* future, + profiler_hub_instance_id_t instance, + profiler_hub_flowtrace_handle_t flow_container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id) { + missing_t::check_missing_client(m_client_trace); + profiler_hub_result_t result = kProfilerHubStatusNotLoaded; + auto & flow_endpoints = m_reader->get_flows_for_event(reader_types::detail::event_id_access::make(operation, event_id)); + for (auto& endpoint : flow_endpoints) + { + size_t dest_id = reader_types::detail::event_id_access::row_id(endpoint.dest); + profiler_hub_event_operation_t dest_type = reader_types::detail::event_id_access::type(endpoint.dest); + size_t src_id = reader_types::detail::event_id_access::row_id(endpoint.source); + profiler_hub_event_operation_t src_type = reader_types::detail::event_id_access::type(endpoint.source); + if (event_id == src_id) + { + result = client::interface::AddEventDataFlowEndPoint( + m_client_trace, + flow_container, + dest_type, + dest_id, + missing_t::integer("reader_types::detail::event_id_access::track_id(endpoint.dest)"), + kProfilerHubDirectionOutgoing, + missing_t::integer("reader_types::detail::event_id_access::timestamp(endpoint.dest)"), + missing_t::integer("reader_types::detail::event_id_access::duration(endpoint.dest)"), + missing_t::integer("reader_types::detail::event_id_access::category_id(endpoint.dest)"), + missing_t::integer("reader_types::detail::event_id_access::symbol_id(endpoint.dest)"), + missing_t::integer("reader_types::detail::event_id_access::level(endpoint.dest)")); + if (kProfilerHubStatusSuccess != result) + return result; + } + else if (event_id == dest_id) + { + result = client::interface::AddEventDataFlowEndPoint( + m_client_trace, + flow_container, + src_type, + src_id, + missing_t::integer("reader_types::detail::event_id_access::track_id(endpoint.dest)"), + kProfilerHubDirectionIncoming, + missing_t::integer("reader_types::detail::event_id_access::timestamp(endpoint.src)"), + missing_t::integer("reader_types::detail::event_id_access::duration(endpoint.src)"), + missing_t::integer("reader_types::detail::event_id_access::category_id(endpoint.src)"), + missing_t::integer("reader_types::detail::event_id_access::symbol_id(endpoint.src)"), + missing_t::integer("reader_types::detail::event_id_access::level(endpoint.src)")); + if (kProfilerHubStatusSuccess != result) + return result; + } + else + { + return kProfilerHubStatusUnknownError; + } + } + return result; + } + + profiler_hub_result_t profiler_hub_trace_t::get_event_details( + future_t* future, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t event_info_container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id) { + missing_t::check_missing_client(m_client_trace); + profiler_hub_result_t result = kProfilerHubStatusNotLoaded; + auto & event_info = m_reader->get_event_info(reader_types::detail::event_id_access::make(operation, event_id)); + if (event_info.has_value()) + { + for (auto& prop : event_info->properties) + { + if (std::holds_alternative(prop.value)) + { + result = client::interface::AddEventExtendedInfo(m_client_trace, instance, event_info_container, "profiler-hub", prop.key.c_str(), kPprofilerHubDataTypeString, std::get(prop.value).c_str()); + } else + if (std::holds_alternative(prop.value)) + { + result = client::interface::AddEventExtendedInfo(m_client_trace, instance, event_info_container, "profiler-hub", prop.key.c_str(), kPprofilerHubDataTypeInt, std::to_string(std::get(prop.value)).c_str()); + } else + if (std::holds_alternative(prop.value)) + { + result = client::interface::AddEventExtendedInfo(m_client_trace, instance, event_info_container, "profiler-hub", prop.key.c_str(), kPprofilerHubDataTypeInt, std::to_string(std::get(prop.value)).c_str()); + } + else + { + result = kProfilerHubStatusUnknownError; + } + + if (kProfilerHubStatusSuccess != result) + return result; + } + result = client::interface::AddEventEssentialInfo(m_client_trace, instance, event_info_container, + missing_t::integer("event_info->track->id.value"), + missing_t::integer("event_info->stream_track->id.value"), + missing_t::integer("event_info->level"), + missing_t::integer("event_info->stream_track_level") + ); + if (kProfilerHubStatusSuccess != result) + return result; + auto event_arguments = m_reader->get_arguments(reader_types::detail::event_id_access::make(operation, event_id)); + for (auto& arg : event_arguments) + { + result = client::interface::AddEventArgumentsInfo(m_client_trace, instance, event_info_container, arg->position, arg->name.c_str(), arg->type.c_str(), arg->value.c_str()); + if (kProfilerHubStatusSuccess != result) + return result; + } + + } + else + { + return kProfilerHubStatusNotLoaded; + } + return result; + } + + profiler_hub_result_t profiler_hub_trace_t::get_event_stack_trace( + future_t* future, + profiler_hub_instance_id_t instance, + profiler_hub_call_stack_handle_t call_stack_container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id) { + missing_t::check_missing_client(m_client_trace); + profiler_hub_result_t result = kProfilerHubStatusNotLoaded; + auto & call_stack = m_reader->get_call_stack(reader_types::detail::event_id_access::make(operation, event_id)); + size_t depth = 0; + for (auto& stack_frame : call_stack) + { + if (stack_frame.program_counter.has_value()) + { + result = client::interface::AddEventCallStackFrame( + m_client_trace, + instance, + call_stack_container, + stack_frame.program_counter->function.c_str(), + stack_frame.program_counter->filename.c_str(), + stack_frame.program_counter->line_number.has_value() ? std::to_string(stack_frame.program_counter->line_number.value()).c_str() : "", + stack_frame.address_range.has_value() ? std::to_string(stack_frame.address_range->address_base).c_str() : "", + depth++); + if (kProfilerHubStatusSuccess != result) + return result; + } + } + return result; + } + + + profiler_hub_result_t profiler_hub_trace_t::trim_trace_database( + future_t* future, + uint64_t timestamp_start, + uint64_t timestamp_end, + profiler_hub_string_t new_path) + { + return missing_t::trim_trace_database(timestamp_start, timestamp_end, new_path); + } + +} diff --git a/thirdparty/profiler-hub-client/optiq/profiler_hub_trace.h b/thirdparty/profiler-hub-client/optiq/profiler_hub_trace.h new file mode 100644 index 000000000..caa0c5fe6 --- /dev/null +++ b/thirdparty/profiler-hub-client/optiq/profiler_hub_trace.h @@ -0,0 +1,133 @@ +#include "profiler_hub_interface.h" +#include "profiler_hub_client_interface.h" +#include "profiler-hub/reader.hpp" +#include "profiler-hub/storage.hpp" +#include "profiler_hub_missing.hpp" +#include "profiler_hub_future.hpp" + +namespace profiler_hub::interface +{ + class profiler_hub_trace_t + { + public: + profiler_hub_trace_t(std::string trace_path) : + m_client_trace(nullptr), m_trace_path(trace_path), m_config_path(nullptr), m_histogram_bucket_count(300) { + } + + void set_trace_properties(client_trace_handle_t client_trace, std::string config_path, size_t histogram_bucket_count) { + m_client_trace = client_trace; m_config_path = config_path; m_histogram_bucket_count = histogram_bucket_count; + } + + profiler_hub_result_t + open_trace( + future_t * future); + + profiler_hub_result_t + get_time_slice( + future_t * future, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end); + + profiler_hub_result_t + get_pmc_time_slice( + future_t * future, + profiler_hub_track_id_t track_id, + profiler_hub_timeslice_handle_t slice_container, + uint64_t timestamp_start, + uint64_t timestamp_end, + bool left_neighbor, + bool right_neighbor); + + profiler_hub_result_t + get_table_time_slice( + future_t * future, + profiler_hub_table_handle_t table_handle, + profiler_hub_track_id_t track_id, + uint64_t timestamp_start, + uint64_t timestamp_end); + + profiler_hub_result_t + get_search_time_slice( + future_t * future, + profiler_hub_instance_id_t instance, + profiler_hub_table_handle_t table_handle, + profiler_hub_event_operation_t operation, + uint64_t timestamp_start, + uint64_t timestamp_end, + size_t num_search_strings, + profiler_hub_search_strings_t string_filters); + + profiler_hub_result_t + get_data_flow_for_event( + future_t* future, + profiler_hub_instance_id_t instance, + profiler_hub_flowtrace_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id); + + profiler_hub_result_t + get_event_details( + future_t* future, + profiler_hub_instance_id_t instance, + profiler_hub_ext_data_handle_t ontainer, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id); + + profiler_hub_result_t + get_event_stack_trace( + future_t* future, + profiler_hub_instance_id_t instance, + profiler_hub_call_stack_handle_t container, + profiler_hub_event_operation_t operation, + profiler_hub_event_id_t event_id); + + + static profiler_hub_db_type_t + detect_trace(std::string trace_path); + + profiler_hub_result_t trim_trace_database( + future_t* future, + uint64_t timestamp_start, + uint64_t timestamp_end, + profiler_hub_string_t new_path); + + profiler_hub::storage_t* get_storage() { return m_storage.get(); } + profiler_hub::reader_t* get_reader() { return m_reader.get(); } + + profiler_hub_result_t report_node_info(profiler_hub_instance_id_t instance_id); + profiler_hub_result_t report_process_info(profiler_hub_instance_id_t instance_id); + profiler_hub_result_t report_thread_info(profiler_hub_instance_id_t instance_id); + profiler_hub_result_t report_agent_info(profiler_hub_instance_id_t instance_id); + profiler_hub_result_t report_queue_info(profiler_hub_instance_id_t instance_id); + profiler_hub_result_t report_stream_info(profiler_hub_instance_id_t instance_id); + profiler_hub_result_t report_pmc_info(profiler_hub_instance_id_t instance_id); + + private: + // Type conversion can be avoided if types match completely + // Currently there is no corresponded track type for kProfilerHubCategoryRegionSampled on profiler hub side + // And there is no "dma" type of track on rocOptiq side + profiler_hub_track_category_t get_track_category(profiler_hub::reader_types::track_type_t type, profiler_hub::reader_types::region_track_kind_t region_kind); + profiler_hub_agent_type_t get_agent_type(std::string& type); + profiler_hub_result_t add_table_to_client( + missing_types::table_ptr_t table, + profiler_hub_table_handle_t client_table_handle); + + template + profiler_hub_result_t report_info_properties( + ListT& collection, + const char* category, + profiler_hub_instance_id_t instance_id, + const std::unordered_map>& properties_dispatcher); + + private: + client_trace_handle_t m_client_trace; + std::string m_trace_path; + std::string m_config_path; + size_t m_histogram_bucket_count; + std::unique_ptr m_storage; + std::shared_ptr m_reader; + }; + +}