diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 90bc705..e971d02 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -3,30 +3,151 @@ name: Benchmark on: push: branches: [main] + paths: + - 'src/**' + - 'Cargo.toml' + - '.github/workflows/bench.yml' + schedule: + - cron: '0 7 * * 1' workflow_dispatch: permissions: - contents: read + contents: write concurrency: group: bench-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false +env: + FIXTURE_URL: https://github.com/Arkptz/mitm2openapi/releases/download/bench-fixtures-v1/big.flow + FIXTURE_SHA256_URL: https://github.com/Arkptz/mitm2openapi/releases/download/bench-fixtures-v1/big.flow.sha256 + FIXTURE_PREFIX: http://localhost:18000 + jobs: bench: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable - - run: cargo install hyperfine --locked - - name: Build release + + - name: Cache cargo + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: bench-cargo-${{ hashFiles('Cargo.lock') }} + + - name: Install hyperfine + run: | + wget -q https://github.com/sharkdp/hyperfine/releases/download/v1.18.0/hyperfine_1.18.0_amd64.deb + sudo dpkg -i hyperfine_1.18.0_amd64.deb + hyperfine --version + + - name: Install Python mitmproxy2swagger + run: | + pip install --user "mitmproxy<12" "ruamel.yaml<0.19" "mitmproxy2swagger==0.14.0" + echo "$HOME/.local/bin" >> $GITHUB_PATH + python -m pip show mitmproxy2swagger | head -3 + + - name: Build Rust release run: cargo build --release - - name: Run benchmark + + - name: Download benchmark fixture run: | - hyperfine --warmup 2 --runs 5 \ - --export-json bench.json \ - 'target/release/mitm2openapi discover -i testdata/flows/multiple.flow -o /tmp/tpl.yaml -p http://petstore:8080' - - uses: actions/upload-artifact@v4 + mkdir -p /tmp/bench + wget -q "$FIXTURE_URL" -O /tmp/bench/big.flow + wget -q "$FIXTURE_SHA256_URL" -O /tmp/bench/big.flow.sha256 + echo "$(cat /tmp/bench/big.flow.sha256) /tmp/bench/big.flow" | sha256sum -c - + ls -lh /tmp/bench/big.flow + + - name: Run benchmarks + id: bench + run: | + RUST=$(pwd)/target/release/mitm2openapi + OUT=/tmp/bench/out + mkdir -p $OUT + + run_py() { + local flow=$1 prefix=$2 + cp "$flow" $OUT/in.flow + mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$prefix" -f flow > /dev/null 2>&1 + sed -i 's/^- ignore:/- /' $OUT/py.yaml + mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$prefix" -f flow > /dev/null 2>&1 + } + run_rs() { + local flow=$1 prefix=$2 + rm -f $OUT/tpl.yaml + "$RUST" discover -i "$flow" -o $OUT/tpl.yaml -p "$prefix" > /dev/null 2>&1 + sed -i 's|^- ignore:|- |' $OUT/tpl.yaml + "$RUST" generate -i "$flow" -t $OUT/tpl.yaml -o $OUT/rs.yaml -p "$prefix" > /dev/null 2>&1 + } + export -f run_py run_rs + export RUST OUT + + hyperfine --warmup 2 --runs 5 --shell=bash \ + --export-json $OUT/synthetic.json \ + --export-markdown $OUT/synthetic.md \ + --command-name "Python mitmproxy2swagger" "run_py '/tmp/bench/big.flow' '$FIXTURE_PREFIX'" \ + --command-name "Rust mitm2openapi" "run_rs '/tmp/bench/big.flow' '$FIXTURE_PREFIX'" + + echo "## Peak RSS" > $OUT/rss.md + echo "| Tool | RSS |" >> $OUT/rss.md + echo "|------|----:|" >> $OUT/rss.md + cp /tmp/bench/big.flow $OUT/in.flow + mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$FIXTURE_PREFIX" -f flow > /dev/null 2>&1 + py_rss=$(/usr/bin/time -f '%M' mitmproxy2swagger -i $OUT/in.flow -o $OUT/py.yaml -p "$FIXTURE_PREFIX" -f flow 2>&1 > /dev/null | tail -1) + rm -f $OUT/tpl.yaml + rs_discover=$(/usr/bin/time -f '%M' "$RUST" discover -i /tmp/bench/big.flow -o $OUT/tpl.yaml -p "$FIXTURE_PREFIX" 2>&1 > /dev/null | tail -1) + sed -i 's|^- ignore:|- |' $OUT/tpl.yaml + rs_generate=$(/usr/bin/time -f '%M' "$RUST" generate -i /tmp/bench/big.flow -t $OUT/tpl.yaml -o $OUT/rs.yaml -p "$FIXTURE_PREFIX" 2>&1 > /dev/null | tail -1) + rs_peak=$(( rs_discover > rs_generate ? rs_discover : rs_generate )) + printf "| Python mitmproxy2swagger | %s KB |\n" "$py_rss" >> $OUT/rss.md + printf "| Rust mitm2openapi | %s KB |\n" "$rs_peak" >> $OUT/rss.md + + { + echo "# Benchmark results" + echo + echo "_Run: $(date -u '+%Y-%m-%d %H:%M UTC'), commit \`${GITHUB_SHA:0:8}\`, runner: $(uname -sr)_" + echo + echo "Fixture: 89 MB, 40k requests across 8 endpoint shapes (\`bench-fixtures-v1\`)." + echo + echo "## Timing" + echo + cat $OUT/synthetic.md + echo + cat $OUT/rss.md + } > $OUT/summary.md + + cat $OUT/summary.md + cp $OUT/summary.md bench-results.md + + - name: Upload artifacts + uses: actions/upload-artifact@v4 with: - name: bench-results - path: bench.json + name: bench-results-${{ github.sha }} + path: | + /tmp/bench/out/*.json + /tmp/bench/out/*.md + bench-results.md + retention-days: 90 + + - name: Update docs/benchmarks.md + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + run: | + mkdir -p docs + { + echo "# Benchmarks" + echo + echo "Generated by the [benchmark workflow](.github/workflows/bench.yml)." + echo + cat bench-results.md + } > docs/benchmarks.md + git diff --quiet docs/benchmarks.md && { echo "no change"; exit 0; } + git config user.name arkptz + git config user.email arkptz@gmail.com + git add docs/benchmarks.md + git commit -m "docs(bench): refresh benchmark results" + git push diff --git a/README.md b/README.md index d5c0830..ab7bffe 100644 --- a/README.md +++ b/README.md @@ -256,10 +256,14 @@ Format is auto-detected from file content. Use `--format` to override. ## Benchmarks -A [GitHub Actions workflow](.github/workflows/bench.yml) runs `hyperfine` -against the release binary on every push to `main`. Results are uploaded as -build artifacts for manual inspection. No automated regression gate is enforced -yet — the artifacts provide a historical record for eyeballing trends. +Automated CI benchmark runs weekly against the Python original +([`mitmproxy2swagger`](https://github.com/alufers/mitmproxy2swagger)). See +[docs/benchmarks.md](docs/benchmarks.md) for the latest timing and memory +comparison on a ~80 MB synthetic capture, or +trigger a fresh run via +[Actions → Benchmark](../../actions/workflows/bench.yml). + +Reproduce locally with the commands documented in the workflow file. ## Contributing diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..1a53a08 --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,22 @@ +# Benchmarks + +Generated by the [benchmark workflow](.github/workflows/bench.yml). + +# Benchmark results + +_Run: 2026-04-22 22:31 UTC, commit `22ef2faa`, runner: Linux 6.17.0-1011-azure_ + +Fixture: 89 MB, 40k requests across 8 endpoint shapes (`bench-fixtures-v1`). + +## Timing + +| Command | Mean [s] | Min [s] | Max [s] | Relative | +|:---|---:|---:|---:|---:| +| `Python mitmproxy2swagger` | 44.757 ± 0.219 | 44.384 | 44.965 | 16.80 ± 0.26 | +| `Rust mitm2openapi` | 2.663 ± 0.039 | 2.618 | 2.712 | 1.00 | + +## Peak RSS +| Tool | RSS | +|------|----:| +| Python mitmproxy2swagger | 46104 KB | +| Rust mitm2openapi | 6168 KB |