Merge pull request #403 from physycom/db

Move output from `CSV` files to a `SQL` db

Author: Grufoony

.github/workflows/binding.yml

@@ -127,13 +127,14 @@ jobs:
     - name: Verify package metadata
       run: |
         python -c "
-        import pkg_resources
+        import importlib.metadata
         try:
-            dist = pkg_resources.get_distribution('dsf')
-            print(f'Package name: {dist.project_name}')
+            dist = importlib.metadata.distribution('dsf')
+            print(f'Package name: {dist.name}')
             print(f'Package version: {dist.version}')
-            print(f'Package location: {dist.location}')
-        except pkg_resources.DistributionNotFound:
+            location = str(dist.locate_file(''))
+            print(f'Package location: {location}')
+        except importlib.metadata.PackageNotFoundError:
             print('Warning: Package metadata not found')
         "
 

.gitignore

@@ -48,3 +48,6 @@ test/data/*dsf
 
 webapp/data/*
 *egg-info*
+
+# Jupyter Notebook (temporary checks)
+*.ipynb

CMakeLists.txt

@@ -138,6 +138,21 @@ if(NOT simdjson_POPULATED)
 endif()
 # Check if the user has TBB installed
 find_package(TBB REQUIRED CONFIG)
+# Get SQLiteCpp
+# Disable cppcheck on Windows to avoid issues with missing cppcheck installation
+if(WIN32)
+  set(SQLITECPP_RUN_CPPCHECK
+      OFF
+      CACHE BOOL "Run cppcheck on SQLiteCpp" FORCE)
+endif()
+FetchContent_Declare(
+  SQLiteCpp
+  GIT_REPOSITORY https://github.com/SRombauts/SQLiteCpp
+  GIT_TAG 3.3.3)
+FetchContent_GetProperties(SQLiteCpp)
+if(NOT SQLiteCpp_POPULATED)
+  FetchContent_MakeAvailable(SQLiteCpp)
+endif()
 
 add_library(dsf STATIC ${SOURCES})
 target_compile_definitions(dsf PRIVATE SPDLOG_USE_STD_FORMAT)
@@ -151,7 +166,7 @@ target_include_directories(
 target_include_directories(dsf PRIVATE ${rapidcsv_SOURCE_DIR}/src)
 
 # Link other libraries - no csv dependency needed now
-target_link_libraries(dsf PRIVATE TBB::tbb simdjson::simdjson spdlog::spdlog)
+target_link_libraries(dsf PUBLIC TBB::tbb SQLiteCpp PRIVATE simdjson::simdjson spdlog::spdlog)
 
 # Install dsf library
 install(

README.md

@@ -6,6 +6,7 @@
 [![SPDLOG](https://img.shields.io/badge/spdlog-1.17.0-blue.svg)](https://github.com/gabime/spdlog)
 [![CSV](https://img.shields.io/badge/rapidcsv-8.89-blue.svg)](https://github.com/d99kris/rapidcsv)
 [![JSON](https://img.shields.io/badge/simdjson-4.2.1-blue.svg)](https://github.com/simdjson/simdjson)
+[![SQLite](https://img.shields.io/badge/SQLiteCpp-3.3.3-blue.svg)](https://github.com/SRombauts/SQLiteCpp)
 [![codecov](https://codecov.io/gh/physycom/DynamicalSystemFramework/graph/badge.svg?token=JV53J6IUJ3)](https://codecov.io/gh/physycom/DynamicalSystemFramework)
 
 The aim of this project is to rework the original [Traffic Flow Dynamics Model](https://github.com/Grufoony/TrafficFlowDynamicsModel).
@@ -35,7 +36,7 @@ print(dsf.__version__)
 ## Installation (from source)
 
 ### Requirements
-The project requires `C++20` or greater, `cmake`, `tbb` `simdjson`, `spdlog` and `rapidcsv`.
+The project requires `C++20` or greater, `cmake`, `tbb` `simdjson`, `spdlog`, `rapidcsv` and `SQLiteCpp`.
 To install requirements on Ubuntu:
 ```shell
 sudo apt install cmake libtbb-dev

examples/slow_charge_rb.cpp

@@ -27,8 +27,6 @@ std::atomic<bool> bExitFlag{false};
 
 // uncomment these lines to print densities, flows and speeds
 #define PRINT_DENSITIES
-// #define PRINT_FLOWS
-// #define PRINT_SPEEDS
 
 using RoadNetwork = dsf::mobility::RoadNetwork;
 using Dynamics = dsf::mobility::FirstOrderDynamics;
@@ -68,14 +66,13 @@ int main(int argc, char** argv) {
                                            std::to_string(SEED))};  // output folder
   constexpr auto MAX_TIME{static_cast<unsigned int>(5e5)};  // maximum time of simulation
 
-  // Clear output folder or create it if it doesn't exist
+  // Create output folder if it doesn't exist (preserve existing database)
   if (!fs::exists(BASE_OUT_FOLDER)) {
     fs::create_directory(BASE_OUT_FOLDER);
   }
-  if (fs::exists(OUT_FOLDER)) {
-    fs::remove_all(OUT_FOLDER);
+  if (!fs::exists(OUT_FOLDER)) {
+    fs::create_directory(OUT_FOLDER);
   }
-  fs::create_directory(OUT_FOLDER);
   // Starting
   std::cout << "Using dsf version: " << dsf::version() << '\n';
   RoadNetwork graph{};
@@ -119,24 +116,18 @@ int main(int argc, char** argv) {
   // dynamics.setForcePriorities(true);
   dynamics.setSpeedFluctuationSTD(0.1);
 
+  // Connect database for saving data
+  dynamics.connectDataBase(OUT_FOLDER + "simulation_data.db");
+
+  // Configure data saving: interval=10, saveAverageStats=true, saveStreetData=true
+#ifdef PRINT_DENSITIES
+  dynamics.saveData(300, true, true, false);
+#else
+  dynamics.saveData(300, true, false, false);
+#endif
+
   std::cout << "Done." << std::endl;
   std::cout << "Running simulation...\n";
-#ifdef PRINT_FLOWS
-  std::ofstream streetFlow(OUT_FOLDER + "flows.csv");
-  streetFlow << "time";
-  for (const auto& [id, street] : dynamics.graph().edges()) {
-    streetFlow << ';' << id;
-  }
-  streetFlow << '\n';
-#endif
-#ifdef PRINT_SPEEDS
-  std::ofstream streetSpeed(OUT_FOLDER + "speeds.csv");
-  streetSpeed << "time;";
-  for (const auto& [id, street] : dynamics.graph().edges()) {
-    streetSpeed << ';' << id;
-  }
-  streetSpeed << '\n';
-#endif
 
   int deltaAgents{std::numeric_limits<int>::max()};
   int previousAgents{0};
@@ -176,47 +167,12 @@ int main(int argc, char** argv) {
     }
 
     if (dynamics.time_step() % 300 == 0) {
-      dynamics.saveCoilCounts(std::format("{}coil_counts.csv", OUT_FOLDER));
+      // Data is now saved automatically by saveData() configuration
       printLoadingBar(dynamics.time_step(), MAX_TIME);
-      dynamics.saveMacroscopicObservables(std::format("{}data.csv", OUT_FOLDER));
-    }
-    if (dynamics.time_step() % 10 == 0) {
-#ifdef PRINT_DENSITIES
-      dynamics.saveStreetDensities(OUT_FOLDER + "densities.csv", true);
-#endif
-#ifdef PRINT_FLOWS
-      streetFlow << dynamics.time_step();
-      for (const auto& [id, street] : dynamics.graph().edges()) {
-        const auto& meanSpeed = dynamics.streetMeanSpeed(id);
-        if (meanSpeed.has_value()) {
-          streetFlow << ';' << meanSpeed.value() * street->density();
-        } else {
-          streetFlow << ';';
-        }
-      }
-      streetFlow << std::endl;
-#endif
-#ifdef PRINT_SPEEDS
-      streetSpeed << dynamics.time_step();
-      for (const auto& [id, street] : dynamics.graph().edges()) {
-        const auto& meanSpeed = dynamics.streetMeanSpeed(id);
-        if (meanSpeed.has_value()) {
-          streetSpeed << ';' << meanSpeed.value();
-        } else {
-          streetSpeed << ';';
-        }
-      }
-      streetSpeed << std::endl;
-#endif
     }
+    // Street densities are now saved automatically by saveData() configuration
     ++progress;
   }
-#ifdef PRINT_FLOWS
-  streetFlow.close();
-#endif
-#ifdef PRINT_SPEEDS
-  streetSpeed.close();
-#endif
   // std::cout << std::endl;
   // std::map<uint8_t, std::string> turnNames{
   //     {0, "left"}, {1, "straight"}, {2, "right"}, {3, "u-turn"}};

examples/slow_charge_tl.cpp

@@ -28,8 +28,6 @@ std::atomic<bool> bExitFlag{false};
 
 // uncomment these lines to print densities, flows and speeds
 #define PRINT_DENSITIES
-// #define PRINT_FLOWS
-// #define PRINT_SPEEDS
 // #define PRINT_TP
 
 using RoadNetwork = dsf::mobility::RoadNetwork;
@@ -74,14 +72,13 @@ int main(int argc, char** argv) {
                                            ERROR_PROBABILITY,
                                            std::to_string(SEED))};  // output folder
   constexpr auto MAX_TIME{static_cast<unsigned int>(5e5)};  // maximum time of simulation
-  // Clear output folder or create it if it doesn't exist
+  // Create output folder if it doesn't exist (preserve existing database)
   if (!fs::exists(BASE_OUT_FOLDER)) {
     fs::create_directory(BASE_OUT_FOLDER);
   }
-  if (fs::exists(OUT_FOLDER)) {
-    fs::remove_all(OUT_FOLDER);
+  if (!fs::exists(OUT_FOLDER)) {
+    fs::create_directory(OUT_FOLDER);
   }
-  fs::create_directory(OUT_FOLDER);
   // Starting
   std::cout << "Using dsf version: " << dsf::version() << '\n';
   RoadNetwork graph{};
@@ -175,26 +172,20 @@ int main(int argc, char** argv) {
   if (OPTIMIZE)
     dynamics.setDataUpdatePeriod(30);  // Store data every 30 time steps
 
+  // Connect database for saving data
+  dynamics.connectDataBase(OUT_FOLDER + "simulation_data.db");
+
+  // Configure data saving: interval=10, saveAverageStats=true, saveStreetData=true
+#ifdef PRINT_DENSITIES
+  dynamics.saveData(300, true, true, false);
+#else
+  dynamics.saveData(300, true, false, false);
+#endif
+
   const auto TM = dynamics.turnMapping();
 
   std::cout << "Done." << std::endl;
   std::cout << "Running simulation...\n";
-#ifdef PRINT_FLOWS
-  std::ofstream streetFlow(OUT_FOLDER + "flows.csv");
-  streetFlow << "time";
-  for (const auto& [id, street] : dynamics.graph().edges()) {
-    streetFlow << ';' << id;
-  }
-  streetFlow << '\n';
-#endif
-#ifdef PRINT_SPEEDS
-  std::ofstream streetSpeed(OUT_FOLDER + "speeds.csv");
-  streetSpeed << "time";
-  for (const auto& [id, street] : dynamics.graph().edges()) {
-    streetSpeed << ';' << id;
-  }
-  streetSpeed << '\n';
-#endif
 #ifdef PRINT_TP
   std::ofstream outTP(OUT_FOLDER + "turn_probabilities.csv");
   outTP << "time";
@@ -256,8 +247,7 @@ int main(int argc, char** argv) {
     if (dynamics.time_step() % 300 == 0) {
       // printLoadingBar(dynamics.time_step(), MAX_TIME);
       // deltaAgents = std::labs(dynamics.agents().size() - previousAgents);
-      dynamics.saveCoilCounts(std::format("{}coil_counts.csv", OUT_FOLDER));
-      dynamics.saveMacroscopicObservables(std::format("{}data.csv", OUT_FOLDER));
+      // Data is now saved automatically by saveData() configuration
       // deltas.push_back(deltaAgents);
       // previousAgents = dynamics.agents().size();
 #ifdef PRINT_TP
@@ -292,43 +282,9 @@ int main(int argc, char** argv) {
       outTP << std::endl;
 #endif
     }
-    if (dynamics.time_step() % 10 == 0) {
-#ifdef PRINT_DENSITIES
-      dynamics.saveStreetDensities(OUT_FOLDER + "densities.csv", true);
-#endif
-#ifdef PRINT_FLOWS
-      streetFlow << ';' << dynamics.time_step();
-      for (const auto& [id, street] : dynamics.graph().edges()) {
-        const auto& meanSpeed = dynamics.streetMeanSpeed(id);
-        if (meanSpeed.has_value()) {
-          streetFlow << ';' << meanSpeed.value() * street->density();
-        } else {
-          streetFlow << ';';
-        }
-      }
-      streetFlow << std::endl;
-#endif
-#ifdef PRINT_SPEEDS
-      streetSpeed << dynamics.time_step();
-      for (const auto& [id, street] : dynamics.graph().edges()) {
-        const auto& meanSpeed = dynamics.streetMeanSpeed(id);
-        if (meanSpeed.has_value()) {
-          streetSpeed << ';' << meanSpeed.value();
-        } else {
-          streetSpeed << ';';
-        }
-      }
-      streetSpeed << std::endl;
-#endif
-    }
+    // Street densities are now saved automatically by saveData() configuration
     ++progress;
   }
-#ifdef PRINT_FLOWS
-  streetFlow.close();
-#endif
-#ifdef PRINT_SPEEDS
-  streetSpeed.close();
-#endif
   // std::cout << std::endl;
   // std::map<uint8_t, std::string> turnNames{
   //     {0, "left"}, {1, "straight"}, {2, "right"}, {3, "u-turn"}};