ci: automated Python-vs-Rust benchmarks

.github/workflows/bench.yml

@@ -3,30 +3,151 @@ name: Benchmark
 on:
   push:
     branches: [main]
+    paths:
+      - 'src/**'
+      - 'Cargo.toml'
+      - '.github/workflows/bench.yml'
+  schedule:
+    - cron: '0 7 * * 1'
   workflow_dispatch:
 
 permissions:
-  contents: read
+  contents: write
 
 concurrency:
   group: bench-${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: false
 
+env:
+  FIXTURE_URL: https://github.com/Arkptz/mitm2openapi/releases/download/bench-fixtures-v1/big.flow
+  FIXTURE_SHA256_URL: https://github.com/Arkptz/mitm2openapi/releases/download/bench-fixtures-v1/big.flow.sha256
+  FIXTURE_PREFIX: http://localhost:18000
+
 jobs:
   bench:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - uses: dtolnay/rust-toolchain@stable
-      - run: cargo install hyperfine --locked
-      - name: Build release
+
+      - name: Cache cargo
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: bench-cargo-${{ hashFiles('Cargo.lock') }}
+
+      - name: Install hyperfine
+        run: |
+          wget -q https://github.com/sharkdp/hyperfine/releases/download/v1.18.0/hyperfine_1.18.0_amd64.deb
+          sudo dpkg -i hyperfine_1.18.0_amd64.deb
+          hyperfine --version
+
+      - name: Install Python mitmproxy2swagger
+        run: |
+          pip install --user "mitmproxy<12" "ruamel.yaml<0.19" "mitmproxy2swagger==0.14.0"
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+          python -m pip show mitmproxy2swagger | head -3
+
+      - name: Build Rust release
         run: cargo build --release
-      - name: Run benchmark
+
+      - name: Download benchmark fixture
         run: |
-          hyperfine --warmup 2 --runs 5 \
-            --export-json bench.json \
-            'target/release/mitm2openapi discover -i testdata/flows/multiple.flow -o /tmp/tpl.yaml -p http://petstore:8080'
-      - uses: actions/upload-artifact@v4
+          mkdir -p /tmp/bench
+          wget -q "$FIXTURE_URL" -O /tmp/bench/big.flow
+          wget -q "$FIXTURE_SHA256_URL" -O /tmp/bench/big.flow.sha256
+          echo "$(cat /tmp/bench/big.flow.sha256)  /tmp/bench/big.flow" | sha256sum -c -
+          ls -lh /tmp/bench/big.flow
+
+      - name: Run benchmarks
+        id: bench
+        run: |
+          RUST=$(pwd)/target/release/mitm2openapi
+          OUT=/tmp/bench/out
+          mkdir -p $OUT
+
+          run_py() {
+            local flow=$1 prefix=$2
+            cp "$flow" $OUT/in.flow
+            mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$prefix" -f flow > /dev/null 2>&1
+            sed -i 's/^- ignore:/- /' $OUT/py.yaml
+            mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$prefix" -f flow > /dev/null 2>&1
+          }
+          run_rs() {
+            local flow=$1 prefix=$2
+            rm -f $OUT/tpl.yaml
+            "$RUST" discover -i "$flow" -o $OUT/tpl.yaml -p "$prefix" > /dev/null 2>&1
+            sed -i 's|^- ignore:|- |' $OUT/tpl.yaml
+            "$RUST" generate -i "$flow" -t $OUT/tpl.yaml -o $OUT/rs.yaml -p "$prefix" > /dev/null 2>&1
+          }
+          export -f run_py run_rs
+          export RUST OUT
+
+          hyperfine --warmup 2 --runs 5 --shell=bash \
+            --export-json $OUT/synthetic.json \
+            --export-markdown $OUT/synthetic.md \
+            --command-name "Python mitmproxy2swagger" "run_py '/tmp/bench/big.flow' '$FIXTURE_PREFIX'" \
+            --command-name "Rust mitm2openapi"         "run_rs '/tmp/bench/big.flow' '$FIXTURE_PREFIX'"
+
+          echo "## Peak RSS" > $OUT/rss.md
+          echo "| Tool | RSS |" >> $OUT/rss.md
+          echo "|------|----:|" >> $OUT/rss.md
+          cp /tmp/bench/big.flow $OUT/in.flow
+          mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$FIXTURE_PREFIX" -f flow > /dev/null 2>&1
+          py_rss=$(/usr/bin/time -f '%M' mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$FIXTURE_PREFIX" -f flow 2>&1 > /dev/null | tail -1)
+          rm -f $OUT/tpl.yaml
+          rs_discover=$(/usr/bin/time -f '%M' "$RUST" discover -i /tmp/bench/big.flow -o $OUT/tpl.yaml -p "$FIXTURE_PREFIX" 2>&1 > /dev/null | tail -1)
+          sed -i 's|^- ignore:|- |' $OUT/tpl.yaml
+          rs_generate=$(/usr/bin/time -f '%M' "$RUST" generate -i /tmp/bench/big.flow -t $OUT/tpl.yaml -o $OUT/rs.yaml -p "$FIXTURE_PREFIX" 2>&1 > /dev/null | tail -1)
+          rs_peak=$(( rs_discover > rs_generate ? rs_discover : rs_generate ))
+          printf "| Python mitmproxy2swagger | %s KB |\n" "$py_rss" >> $OUT/rss.md
+          printf "| Rust mitm2openapi        | %s KB |\n" "$rs_peak" >> $OUT/rss.md
+
+          {
+            echo "# Benchmark results"
+            echo
+            echo "_Run: $(date -u '+%Y-%m-%d %H:%M UTC'), commit \`${GITHUB_SHA:0:8}\`, runner: $(uname -sr)_"
+            echo
+            echo "Fixture: 89 MB, 40k requests across 8 endpoint shapes (\`bench-fixtures-v1\`)."
+            echo
+            echo "## Timing"
+            echo
+            cat $OUT/synthetic.md
+            echo
+            cat $OUT/rss.md
+          } > $OUT/summary.md
+
+          cat $OUT/summary.md
+          cp $OUT/summary.md bench-results.md
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
         with:
-          name: bench-results
-          path: bench.json
+          name: bench-results-${{ github.sha }}
+          path: |
+            /tmp/bench/out/*.json
+            /tmp/bench/out/*.md
+            bench-results.md
+          retention-days: 90
+
+      - name: Update docs/benchmarks.md
+        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+        run: |
+          mkdir -p docs
+          {
+            echo "# Benchmarks"
+            echo
+            echo "Generated by the [benchmark workflow](.github/workflows/bench.yml)."
+            echo
+            cat bench-results.md
+          } > docs/benchmarks.md
+          git diff --quiet docs/benchmarks.md && { echo "no change"; exit 0; }
+          git config user.name arkptz
+          git config user.email arkptz@gmail.com
+          git add docs/benchmarks.md
+          git commit -m "docs(bench): refresh benchmark results"
+          git push

README.md

@@ -256,10 +256,14 @@ Format is auto-detected from file content. Use `--format` to override.
 
 ## Benchmarks
 
-A [GitHub Actions workflow](.github/workflows/bench.yml) runs `hyperfine`
-against the release binary on every push to `main`. Results are uploaded as
-build artifacts for manual inspection. No automated regression gate is enforced
-yet — the artifacts provide a historical record for eyeballing trends.
+Automated CI benchmark runs weekly against the Python original
+([`mitmproxy2swagger`](https://github.com/alufers/mitmproxy2swagger)). See
+[docs/benchmarks.md](docs/benchmarks.md) for the latest timing and memory
+comparison on a ~80 MB synthetic capture, or
+trigger a fresh run via
+[Actions → Benchmark](../../actions/workflows/bench.yml).
+
+Reproduce locally with the commands documented in the workflow file.
 
 ## Contributing
 

docs/benchmarks.md

@@ -0,0 +1,22 @@
+# Benchmarks
+
+Generated by the [benchmark workflow](.github/workflows/bench.yml).
+
+# Benchmark results
+
+_Run: 2026-04-22 22:31 UTC, commit `22ef2faa`, runner: Linux 6.17.0-1011-azure_
+
+Fixture: 89 MB, 40k requests across 8 endpoint shapes (`bench-fixtures-v1`).
+
+## Timing
+
+| Command | Mean [s] | Min [s] | Max [s] | Relative |
+|:---|---:|---:|---:|---:|
+| `Python mitmproxy2swagger` | 44.757 ± 0.219 | 44.384 | 44.965 | 16.80 ± 0.26 |
+| `Rust mitm2openapi` | 2.663 ± 0.039 | 2.618 | 2.712 | 1.00 |
+
+## Peak RSS
+| Tool | RSS |
+|------|----:|
+| Python mitmproxy2swagger | 46104 KB |
+| Rust mitm2openapi        | 6168 KB |