feat(perf): performance improvements for parallel reading, indexing, and aggregation

.github/workflows/ci.yml

@@ -2,9 +2,10 @@ name: CI
 
 on:
   push:
-    branches: [ main, develop, initialize, 'feat/**', 'fix/**', 'chore/**' ]
+    branches:
+      [main, develop, initialize, "feat/**", "fix/**", "chore/**", "perf/**"]
   pull_request:
-    branches: [ main, develop ]
+    branches: [main, develop]
   workflow_dispatch:
 
 jobs:
@@ -13,22 +14,22 @@ jobs:
     outputs:
       code: ${{ steps.filter.outputs.code }}
     steps:
-    - uses: actions/checkout@v6
-    - uses: dorny/paths-filter@v3.0.2
-      id: filter
-      with:
-        filters: |
-          code:
-            - 'include/**'
-            - 'src/**'
-            - 'tests/**'
-            - 'python/**'
-            - 'cmake/**'
-            - 'CMakeLists.txt'
-            - 'CMakePresets.json'
-            - 'pyproject.toml'
-            - 'Makefile'
-            - '.github/workflows/ci.yml'
+      - uses: actions/checkout@v6
+      - uses: dorny/paths-filter@v3.0.2
+        id: filter
+        with:
+          filters: |
+            code:
+              - 'include/**'
+              - 'src/**'
+              - 'tests/**'
+              - 'python/**'
+              - 'cmake/**'
+              - 'CMakeLists.txt'
+              - 'CMakePresets.json'
+              - 'pyproject.toml'
+              - 'Makefile'
+              - '.github/workflows/ci.yml'
 
   test:
     needs: changes
@@ -42,82 +43,82 @@ jobs:
       matrix:
         os: [ubuntu-22.04, ubuntu-24.04, ubuntu-latest, macos-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
-    
+
     steps:
-    - uses: actions/checkout@v6
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v6.1.0
-      with:
-        python-version: ${{ matrix.python-version }}
+      - uses: actions/checkout@v6
 
-    - name: Cache ccache
-      uses: actions/cache@v5
-      with:
-        path: ~/.ccache
-        key: ccache-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('CMakeLists.txt', 'pyproject.toml', '.github/workflows/ci.yml') }}
-        restore-keys: |
-          ccache-${{ runner.os }}-${{ matrix.python-version }}-
-          ccache-${{ runner.os }}-
-
-    - name: Install dependencies (Ubuntu)
-      if: runner.os == 'Linux'
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y build-essential cmake ccache lcov zlib1g-dev libsqlite3-dev pkg-config ninja-build
-
-    - name: Install dependencies (macOS)
-      if: runner.os == 'macOS'
-      run: |
-        brew update
-        for f in cmake ccache lcov zlib sqlite pkg-config ninja; do
-          if brew list --versions "$f" >/dev/null; then
-            echo "$f already installed"
-          else
-            brew install "$f"
-          fi
-        done
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v6.1.0
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache ccache
+        uses: actions/cache@v5
+        with:
+          path: ~/.ccache
+          key: ccache-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('CMakeLists.txt', 'pyproject.toml', '.github/workflows/ci.yml') }}
+          restore-keys: |
+            ccache-${{ runner.os }}-${{ matrix.python-version }}-
+            ccache-${{ runner.os }}-
+
+      - name: Install dependencies (Ubuntu)
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential cmake ccache lcov zlib1g-dev libsqlite3-dev pkg-config ninja-build
 
-    - name: Run coverage
-      if: (matrix.os == 'ubuntu-22.04' || matrix.os == 'macos-latest') && matrix.python-version == '3.12'
-      run: |
-        make coverage
+      - name: Install dependencies (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew update
+          for f in cmake ccache lcov zlib sqlite pkg-config ninja; do
+            if brew list --versions "$f" >/dev/null; then
+              echo "$f already installed"
+            else
+              brew install "$f"
+            fi
+          done
 
-    - name: Run test (Unix)
-      if: "!((matrix.os == 'ubuntu-22.04' || matrix.os == 'macos-latest') && matrix.python-version == '3.12')"
-      run: |
-        make test
-
-    - name: Run Python tests (with venv)
-      if: "!((matrix.os == 'ubuntu-22.04' || matrix.os == 'macos-latest') && matrix.python-version == '3.12')"
-      run: |
-        if [ "${{ runner.os }}" = "Linux" ] && [ "${{ matrix.python-version }}" = "3.12" ]; then
-          make test-py RUN_TY=1
-        else
-          make test-py
-        fi
+      - name: Run coverage
+        if: (matrix.os == 'ubuntu-22.04' || matrix.os == 'macos-latest') && matrix.python-version == '3.12'
+        run: |
+          make coverage
+
+      - name: Run test (Unix)
+        if: "!((matrix.os == 'ubuntu-22.04' || matrix.os == 'macos-latest') && matrix.python-version == '3.12')"
+        run: |
+          make test
+
+      - name: Run Python tests (with venv)
+        if: "!((matrix.os == 'ubuntu-22.04' || matrix.os == 'macos-latest') && matrix.python-version == '3.12')"
+        run: |
+          if [ "${{ runner.os }}" = "Linux" ] && [ "${{ matrix.python-version }}" = "3.12" ]; then
+            make test-py RUN_TY=1
+          else
+            make test-py
+          fi
 
-    - name: Upload coverage reports to Coveralls
-      if: (matrix.os == 'ubuntu-22.04' || matrix.os == 'macos-latest') && matrix.python-version == '3.12'
-      uses: coverallsapp/github-action@v2.3.6
-      continue-on-error: true
-      with:
-        file: coverage/coverage_filtered.info
-        format: lcov
-        flag-name: ${{ matrix.os }}
-        parallel: true
-      env:
-        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+      - name: Upload coverage reports to Coveralls
+        if: (matrix.os == 'ubuntu-22.04' || matrix.os == 'macos-latest') && matrix.python-version == '3.12'
+        uses: coverallsapp/github-action@v2.3.6
+        continue-on-error: true
+        with:
+          file: coverage/coverage_filtered.info
+          format: lcov
+          flag-name: ${{ matrix.os }}
+          parallel: true
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
 
   coverage-finish:
     needs: test
     if: always()
     runs-on: ubuntu-latest
     steps:
-    - name: Coveralls finished
-      uses: coverallsapp/github-action@v2.3.6
-      continue-on-error: true
-      with:
-        parallel-finished: true
-      env:
-        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
+      - name: Coveralls finished
+        uses: coverallsapp/github-action@v2.3.6
+        continue-on-error: true
+        with:
+          parallel-finished: true
+        env:
+          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}

.gitignore

@@ -94,3 +94,4 @@ PLANS.md
 docs/plans
 
 profiling-results*/
+dfanalyzer/

CMakeLists.txt

@@ -65,8 +65,11 @@ option(DFTRACER_UTILS_ENABLE_ASAN "Enable AddressSanitizer" OFF)
 option(DFTRACER_UTILS_ENABLE_UBSAN "Enable UndefinedBehaviorSanitizer" OFF)
 option(DFTRACER_UTILS_ENABLE_TSAN "Enable ThreadSanitizer" OFF)
 option(DFTRACER_UTILS_ENABLE_MPI "Enable MPI support for call tree" OFF)
+option(DFTRACER_USE_ZLIB_NG "Use zlib-ng (compat) instead of madler/zlib; falls back to madler on failure" ON)
 option(DFTRACER_UTILS_ENABLE_ARROW "Enable Arrow C Data Interface via nanoarrow" ON)
 option(DFTRACER_UTILS_ENABLE_ARROW_IPC "Enable Arrow IPC file read/write via nanoarrow" ON)
+option(DFTRACER_UTILS_ENABLE_ZSTD "Enable ZSTD compression for RocksDB" ON)
+option(DFTRACER_UTILS_ENABLE_LZ4 "Enable LZ4 compression for RocksDB" OFF)
 
 if(DFTRACER_UTILS_TESTS)
   message(STATUS "Building tests")
@@ -136,6 +139,18 @@ else()
   message(STATUS "kqueue support: disabled (sys/event.h not found)")
 endif()
 
+# lustreapi: optional for stripe_count / stripe_size queries. When absent, the
+# parallel writer treats Lustre like an opaque POSIX filesystem.
+check_include_file("lustre/lustreapi.h" HAVE_LUSTRE_LUSTREAPI_H)
+find_library(LUSTREAPI_LIBRARY NAMES lustreapi)
+if(HAVE_LUSTRE_LUSTREAPI_H AND LUSTREAPI_LIBRARY)
+  set(DFTRACER_UTILS_HAVE_LUSTREAPI ON)
+  message(STATUS "lustreapi support: enabled (${LUSTREAPI_LIBRARY})")
+else()
+  set(DFTRACER_UTILS_HAVE_LUSTREAPI OFF)
+  message(STATUS "lustreapi support: disabled (header or library not found)")
+endif()
+
 # Set C++ standard
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)

CMakePresets.json

@@ -77,7 +77,8 @@
       "inherits": "dev",
       "cacheVariables": {
         "DFTRACER_UTILS_TESTS": "ON",
-        "DFTRACER_UTILS_DEBUG": "ON"
+        "DFTRACER_UTILS_DEBUG": "ON",
+        "DFTRACER_UTILS_BUILD_PYTHON": "ON"
       }
     },
     {

cmake/configure_files/dftracer_utils_config.dbg.h.in

@@ -31,6 +31,13 @@
 /* Macro flags */
 #cmakedefine DFTRACER_UTILS_HAS_STD_FILESYSTEM 1
 
+/* Feature flags */
+#cmakedefine DFTRACER_UTILS_ENABLE_ARROW 1
+#cmakedefine DFTRACER_UTILS_ENABLE_ARROW_IPC 1
+#cmakedefine DFTRACER_UTILS_ENABLE_LZ4 1
+#cmakedefine DFTRACER_UTILS_ENABLE_ZSTD 1
+#cmakedefine DFTRACER_UTILS_HAVE_LUSTREAPI 1
+
 #define DFTRACER_UTILS_LOGGER_CPP_LOGGER 1
 #define DFTRACER_UTILS_LOGGER_LEVEL_TRACE 1
 #define DFTRACER_UTILS_LOGGER_LEVEL_DEBUG 1

cmake/configure_files/dftracer_utils_config.h.in

@@ -31,6 +31,13 @@
 /* Macro flags */
 #cmakedefine DFTRACER_UTILS_HAS_STD_FILESYSTEM 1
 
+/* Feature flags */
+#cmakedefine DFTRACER_UTILS_ENABLE_ARROW 1
+#cmakedefine DFTRACER_UTILS_ENABLE_ARROW_IPC 1
+#cmakedefine DFTRACER_UTILS_ENABLE_LZ4 1
+#cmakedefine DFTRACER_UTILS_ENABLE_ZSTD 1
+#cmakedefine DFTRACER_UTILS_HAVE_LUSTREAPI 1
+
 #define DFTRACER_UTILS_LOGGER_CPP_LOGGER 1
 #define DFTRACER_UTILS_LOGGER_LEVEL_TRACE 0
 #define DFTRACER_UTILS_LOGGER_LEVEL_DEBUG 0