diff --git a/.github/workflows/blink_parity.yaml b/.github/workflows/blink_parity.yaml index 4d6e10a3bd..3f3bad16e8 100644 --- a/.github/workflows/blink_parity.yaml +++ b/.github/workflows/blink_parity.yaml @@ -1,9 +1,11 @@ name: Blink Parity -# Runs blink and the ECS in this repository on the same real codebase (full -# rector-src) and fails if blink's output differs from ECS (psr12 + common) on -# any file. This is the 1:1 parity gate - it stays red until blink reproduces -# ECS exactly. +# Runs blink (--blink) and the PHP ECS engine over the same real-world corpora +# and fails when blink's output diverges beyond each corpus's ratchet threshold. +# One matrix entry per corpus; thresholds and sets live in blink/parity/corpora/. +# The company goal is 0% divergence on every corpus, so the thresholds only ever +# go down. The job summary lists every differing file grouped by the ECS checker +# that caused it, so the remaining gap stays visible. on: pull_request: @@ -16,8 +18,17 @@ on: workflow_dispatch: null jobs: - blink-parity: + parity: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + corpus: + - rector-src + - mautic + - laravel + - rector + name: "parity: ${{ matrix.corpus }}" steps: - uses: actions/checkout@v4 @@ -25,138 +36,12 @@ jobs: with: go-version: "1.26.6" - - name: Build blink - working-directory: blink - run: go build -o /tmp/blink . - - uses: shivammathur/setup-php@v2 with: php-version: "8.4" coverage: none - # installs this repository's ECS so parity runs against the code being shipped - uses: "ramsey/composer-install@v4" - - name: Fetch a real codebase sample - run: | - git clone --depth 1 https://github.com/rectorphp/rector-src.git /tmp/rector - # full rector-src tree; .php.inc fixtures are skipped by the *.php find below - mkdir -p /tmp/sample - cp -r /tmp/rector/src /tmp/rector/rules /tmp/rector/tests /tmp/sample/ - cp -r /tmp/sample /tmp/ours - cp -r /tmp/sample /tmp/ecs - - - name: Run blink - run: /tmp/blink --fix /tmp/ours || true - - - name: Build the local ECS config - run: | - cat > /tmp/ecs.php <<'PHP' - withPreparedSets(psr12: true, common: true); - PHP - - - name: Run the local ECS - run: bin/ecs check /tmp/ecs --fix --no-progress-bar --config /tmp/ecs.php || true - - - name: Rank missing fixers (data-driven backlog) - run: | - # what ECS would apply on the pristine sample, as JSON - bin/ecs check /tmp/sample --output-format=json --config /tmp/ecs.php > /tmp/ecs.json 2>/dev/null || true - # fixers we already implement - /tmp/blink list-checkers | sed 's/^[[:space:]]*-[[:space:]]*//; /Registered/d' | sort -u > /tmp/impl.txt - # count applied checkers across files, most frequent first - jq -r '[.files[]?.diffs[]?.applied_checkers[]?] | group_by(.) | map("\(length) \(.[0])") | .[]' /tmp/ecs.json 2>/dev/null | sort -rn > /tmp/counts.txt || true - { - echo "## Backlog: ECS fixers not yet in blink, ranked by frequency on the sample" - echo "" - echo "| fixer | files |" - echo "|---|---|" - } >> "$GITHUB_STEP_SUMMARY" - n=0 - while read -r count chk; do - [ "$n" -ge 25 ] && break - if ! grep -qxF "$chk" /tmp/impl.txt; then - echo "| \`$chk\` | $count |" >> "$GITHUB_STEP_SUMMARY" - n=$((n + 1)) - fi - done < /tmp/counts.txt - if [ "$n" -eq 0 ]; then - echo "| (no data - check the ECS JSON shape) | |" >> "$GITHUB_STEP_SUMMARY" - fi - # stdout diagnostics (the summary table is not fetchable via CLI) - echo "=== implemented: $(wc -l < /tmp/impl.txt) | ecs.json bytes: $(wc -c < /tmp/ecs.json) ===" - echo "=== top applied checkers, raw (before excluding implemented) ===" - head -20 /tmp/counts.txt || true - - - name: Compare outputs - run: | - total=0 - differ=0 - for f in $(cd /tmp/ours && find . -name '*.php'); do - total=$((total + 1)) - if ! diff -q "/tmp/ours/$f" "/tmp/ecs/$f" >/dev/null 2>&1; then - differ=$((differ + 1)) - fi - done - same=$((total - differ)) - { - echo "## blink vs ECS (psr12 + common) on full rector-src" - echo "" - echo "| metric | value |" - echo "|---|---|" - echo "| files | $total |" - echo "| identical output | $same |" - echo "| differing output | $differ |" - echo "" - echo "This is the 1:1 parity gate: it fails while any file differs from ECS." - } >> "$GITHUB_STEP_SUMMARY" - echo "identical=$same differing=$differ of $total" - - # Per differing file: which rules ECS applied vs blink, and the - # snippet they differ on. ECS applied checkers come from the JSON - # captured on the pristine sample; blink's from a dry-run check. - echo "### Divergences ($differ files): applied rules + diff" >> "$GITHUB_STEP_SUMMARY" - echo "" - echo "=== Divergences ($differ files) ===" - export LC_ALL=C - for f in $(cd /tmp/ours && find . -name '*.php' | sort); do - diff -q "/tmp/ours/$f" "/tmp/ecs/$f" >/dev/null 2>&1 && continue - rel=${f#./} - abs="/tmp/sample/$rel" - ecs_rules=$(jq -r --arg k "$abs" '.files[$k].diffs[]?.applied_checkers[]?' /tmp/ecs.json 2>/dev/null | sed -E 's#.*\\##' | sort -u) - go_rules=$(/tmp/blink "$abs" 2>/dev/null | sed -n 's/^[[:space:]]*\*[[:space:]]//p' | sed -E 's#.*\\##' | sort -u) - ecs_only=$(comm -23 <(printf '%s\n' "$ecs_rules") <(printf '%s\n' "$go_rules") | grep -v '^$' | paste -sd, - || true) - go_only=$(comm -13 <(printf '%s\n' "$ecs_rules") <(printf '%s\n' "$go_rules") | grep -v '^$' | paste -sd, - || true) - # stdout (CI log): file + which rules differ + the diff - echo "" - echo "--- $rel" - echo " ECS applied, blink did not: ${ecs_only:-(none)}" - echo " blink applied, ECS did not: ${go_only:-(none)}" - diff "/tmp/ours/$f" "/tmp/ecs/$f" | sed 's/^/ /' | head -40 || true - # job summary (rich, collapsible) - { - echo "" - echo "
$rel" - echo "" - echo "- **ECS applied, blink did not:** ${ecs_only:-(none)}" - echo "- **blink applied, ECS did not:** ${go_only:-(none)}" - echo "" - echo '```diff' - diff "/tmp/ours/$f" "/tmp/ecs/$f" | head -40 || true - echo '```' - echo "
" - } >> "$GITHUB_STEP_SUMMARY" - done - - # fail when blink output does not match ECS - if [ "$differ" -gt 0 ]; then - echo "::error::blink output differs from ECS on $differ of $total files" - exit 1 - fi + - name: "Parity ${{ matrix.corpus }}" + run: blink/parity/run.sh "blink/parity/corpora/${{ matrix.corpus }}.json" diff --git a/.github/workflows/blink_parity_laravel.yaml b/.github/workflows/blink_parity_laravel.yaml deleted file mode 100644 index b106251079..0000000000 --- a/.github/workflows/blink_parity_laravel.yaml +++ /dev/null @@ -1,151 +0,0 @@ -name: Blink Parity (laravel) - -# Runs the same ECS config both ways on a real laravel/framework checkout: -# bin/ecs check --fix (PHP engine) -# bin/ecs check --fix --blink (ecs-go binary, via --ecs-config) -# on two identical copies, then diffs the resulting trees. This applies every -# prepared set (psr12 + perCs + common + standaloneLine + cleanCode) to measure -# how far blink parity reaches on laravel under the full rule surface. -# -# Ratchet gate: it reports the parity table and per-file divergences to the job -# summary, and fails once the differing share crosses MAX_DIFF_PERCENT. Lower the -# threshold as blink parity improves. - -on: - pull_request: - paths: - - 'blink/**' - - 'src/**' - - 'packages/**' - - 'config/**' - - '.github/workflows/blink_parity_laravel.yaml' - workflow_dispatch: null - -jobs: - blink-parity-laravel: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-go@v5 - with: - go-version: "1.26.6" - - - name: Build blink - working-directory: blink - run: go build -o /tmp/blink . - - - uses: shivammathur/setup-php@v2 - with: - php-version: "8.4" - coverage: none - - # installs this repository's ECS so parity runs against the code being shipped - - uses: "ramsey/composer-install@v4" - - - name: Fetch laravel/framework - run: git clone --depth 1 https://github.com/laravel/framework.git /tmp/laravel - - # the ECS config lives inside the checkout so __DIR__ resolves against it; - # applies every prepared set, no per-fixer skips, to exercise the full surface - - name: Write the shared ECS config - run: | - cat > /tmp/laravel/ecs.php <<'PHP' - withPaths([ - __DIR__ . '/src', - __DIR__ . '/tests', - ]) - ->withSkip([ - '*/node_modules/*', - '*/Fixture/*', - ]) - ->withPreparedSets( - psr12: true, - perCs: true, - common: true, - standaloneLine: true, - cleanCode: true, - ) - // single process: parallel workers overflow the NDJSON buffer on - // large files and abort the PHP run (blink ignores this) - ->withoutParallel(); - PHP - cp /tmp/laravel/ecs.php /tmp/laravel/ecs-copy.php - - - name: Two identical copies of the source - run: | - cp -r /tmp/laravel /tmp/ours - cp -r /tmp/laravel /tmp/ecs - - - name: Fix with blink (--blink) - env: - ECS_BLINK_BIN: /tmp/blink - run: bin/ecs check --fix --no-progress-bar --config /tmp/ours/ecs.php --blink || true - - - name: Fix with the PHP engine - run: bin/ecs check --fix --no-progress-bar --config /tmp/ecs/ecs.php || true - - - name: Compare the two trees - run: | - # ratchet gate: fail once the differing share crosses this; lower it as parity improves - MAX_DIFF_PERCENT=2.4 - total=0 - differ=0 - differing_files="" - for f in $(cd /tmp/ours && find src tests -name '*.php' | sort); do - total=$((total + 1)) - if ! diff -q "/tmp/ours/$f" "/tmp/ecs/$f" >/dev/null 2>&1; then - differ=$((differ + 1)) - differing_files="$differing_files $f" - fi - done - same=$((total - differ)) - pct=0 - diff_pct=0 - if [ "$total" -gt 0 ]; then - pct=$(awk "BEGIN{printf \"%.2f\", $same * 100 / $total}") - diff_pct=$(awk "BEGIN{printf \"%.2f\", $differ * 100 / $total}") - fi - { - echo "## blink (--blink) vs PHP ECS on laravel/framework (all sets)" - echo "" - echo "Sets applied: psr12 + perCs + common + standaloneLine + cleanCode." - echo "" - echo "| metric | value |" - echo "|---|---|" - echo "| files | $total |" - echo "| identical output | $same |" - echo "| differing output | $differ |" - echo "| parity | ${pct}% |" - echo "| diff | ${diff_pct}% (max ${MAX_DIFF_PERCENT}%) |" - echo "" - echo "Ratchet gate: fails once the differing share exceeds ${MAX_DIFF_PERCENT}%." - } >> "$GITHUB_STEP_SUMMARY" - echo "identical=$same differing=$differ of $total (${pct}% parity, ${diff_pct}% diff, max ${MAX_DIFF_PERCENT}%)" - - for f in $differing_files; do - { - echo "" - echo "
$f" - echo "" - echo '```diff' - diff "/tmp/ecs/$f" "/tmp/ours/$f" | head -60 || true - echo '```' - echo "
" - } >> "$GITHUB_STEP_SUMMARY" - echo "--- $f (< PHP, > blink)" - diff "/tmp/ecs/$f" "/tmp/ours/$f" | sed 's/^/ /' | head -60 || true - done - - if awk "BEGIN{exit !($differ * 100 / $total > $MAX_DIFF_PERCENT)}"; then - echo "::error::--blink differs from the PHP engine on ${diff_pct}% of files ($differ of $total), over the ${MAX_DIFF_PERCENT}% limit" - exit 1 - fi - echo "::notice::--blink differs on ${diff_pct}% of files ($differ of $total), within the ${MAX_DIFF_PERCENT}% limit" diff --git a/.github/workflows/blink_parity_mautic.yaml b/.github/workflows/blink_parity_mautic.yaml deleted file mode 100644 index fa63c01bf1..0000000000 --- a/.github/workflows/blink_parity_mautic.yaml +++ /dev/null @@ -1,151 +0,0 @@ -name: Blink Parity (mautic) - -# Runs the same ECS config both ways on a real mautic/mautic checkout: -# bin/ecs check --fix (PHP engine) -# bin/ecs check --fix --blink (ecs-go binary, via --ecs-config) -# on two identical copies, then diffs the resulting trees. This applies every -# prepared set (psr12 + perCs + common + standaloneLine + cleanCode) to measure -# how far blink parity reaches on mautic under the full rule surface. -# -# Ratchet gate: it reports the parity table and per-file divergences to the job -# summary, and fails once the differing share crosses MAX_DIFF_PERCENT. Lower the -# threshold as blink parity improves. - -on: - pull_request: - paths: - - 'blink/**' - - 'src/**' - - 'packages/**' - - 'config/**' - - '.github/workflows/blink_parity_mautic.yaml' - workflow_dispatch: null - -jobs: - blink-parity-mautic: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-go@v5 - with: - go-version: "1.26.6" - - - name: Build blink - working-directory: blink - run: go build -o /tmp/blink . - - - uses: shivammathur/setup-php@v2 - with: - php-version: "8.4" - coverage: none - - # installs this repository's ECS so parity runs against the code being shipped - - uses: "ramsey/composer-install@v4" - - - name: Fetch mautic/mautic - run: git clone --depth 1 https://github.com/mautic/mautic.git /tmp/mautic - - # the ECS config lives inside the checkout so __DIR__ resolves against it; - # applies every prepared set, no per-fixer skips, to exercise the full surface - - name: Write the shared ECS config - run: | - cat > /tmp/mautic/ecs.php <<'PHP' - withPaths([ - __DIR__ . '/app', - __DIR__ . '/plugins', - ]) - ->withSkip([ - '*/node_modules/*', - '*/Fixture/*', - ]) - ->withPreparedSets( - psr12: true, - perCs: true, - common: true, - standaloneLine: true, - cleanCode: true, - ) - // single process: parallel workers overflow the NDJSON buffer on - // large mautic files and abort the PHP run (blink ignores this) - ->withoutParallel(); - PHP - cp /tmp/mautic/ecs.php /tmp/mautic/ecs-copy.php - - - name: Two identical copies of the source - run: | - cp -r /tmp/mautic /tmp/ours - cp -r /tmp/mautic /tmp/ecs - - - name: Fix with blink (--blink) - env: - ECS_BLINK_BIN: /tmp/blink - run: bin/ecs check --fix --no-progress-bar --config /tmp/ours/ecs.php --blink || true - - - name: Fix with the PHP engine - run: bin/ecs check --fix --no-progress-bar --config /tmp/ecs/ecs.php || true - - - name: Compare the two trees - run: | - # ratchet gate: fail once the differing share crosses this; lower it as parity improves - MAX_DIFF_PERCENT=0.07 - total=0 - differ=0 - differing_files="" - for f in $(cd /tmp/ours && find app plugins -name '*.php' | sort); do - total=$((total + 1)) - if ! diff -q "/tmp/ours/$f" "/tmp/ecs/$f" >/dev/null 2>&1; then - differ=$((differ + 1)) - differing_files="$differing_files $f" - fi - done - same=$((total - differ)) - pct=0 - diff_pct=0 - if [ "$total" -gt 0 ]; then - pct=$(awk "BEGIN{printf \"%.2f\", $same * 100 / $total}") - diff_pct=$(awk "BEGIN{printf \"%.2f\", $differ * 100 / $total}") - fi - { - echo "## blink (--blink) vs PHP ECS on mautic/mautic (all sets)" - echo "" - echo "Sets applied: psr12 + perCs + common + standaloneLine + cleanCode." - echo "" - echo "| metric | value |" - echo "|---|---|" - echo "| files | $total |" - echo "| identical output | $same |" - echo "| differing output | $differ |" - echo "| parity | ${pct}% |" - echo "| diff | ${diff_pct}% (max ${MAX_DIFF_PERCENT}%) |" - echo "" - echo "Ratchet gate: fails once the differing share exceeds ${MAX_DIFF_PERCENT}%." - } >> "$GITHUB_STEP_SUMMARY" - echo "identical=$same differing=$differ of $total (${pct}% parity, ${diff_pct}% diff, max ${MAX_DIFF_PERCENT}%)" - - for f in $differing_files; do - { - echo "" - echo "
$f" - echo "" - echo '```diff' - diff "/tmp/ecs/$f" "/tmp/ours/$f" | head -60 || true - echo '```' - echo "
" - } >> "$GITHUB_STEP_SUMMARY" - echo "--- $f (< PHP, > blink)" - diff "/tmp/ecs/$f" "/tmp/ours/$f" | sed 's/^/ /' | head -60 || true - done - - if awk "BEGIN{exit !($differ * 100 / $total > $MAX_DIFF_PERCENT)}"; then - echo "::error::--blink differs from the PHP engine on ${diff_pct}% of files ($differ of $total), over the ${MAX_DIFF_PERCENT}% limit" - exit 1 - fi - echo "::notice::--blink differs on ${diff_pct}% of files ($differ of $total), within the ${MAX_DIFF_PERCENT}% limit" diff --git a/blink/parity/README.md b/blink/parity/README.md new file mode 100644 index 0000000000..8cbc6c0e51 --- /dev/null +++ b/blink/parity/README.md @@ -0,0 +1,35 @@ +# Blink parity harness + +Measures how far blink's `--blink` output diverges from the PHP ECS engine on +real-world codebases. The goal is **0% divergence on every corpus**, so blink can +become the default engine without changing any formatting. + +## Run one corpus + +```bash +blink/parity/run.sh blink/parity/corpora/laravel.json +``` + +It builds blink from the current checkout, clones the corpus, fixes two copies +(one with `--blink`, one with the PHP engine), diffs them, and prints every +differing file grouped by the ECS checker that caused it. Exit status is non-zero +when the differing share exceeds the corpus's `max_diff_percent`. + +Flags: `--workdir DIR` reuses a directory, `--keep` leaves it in place for +inspection (the diffed trees are `ours/` vs `ecs/`, the report is `report.txt`). + +## Corpora + +Each `corpora/*.json` describes one codebase: `repo`, optional `ref`, the `paths` +to check, ECS `skip` patterns, the prepared `sets`, and the ratchet +`max_diff_percent`. Lower a threshold whenever parity improves; never raise it. + +CI runs every corpus on each pull request (`.github/workflows/blink_parity.yaml`) +and prints the categorized report to the job summary. + +## Closing the gap + +Pick the checker with the highest count in the report, reproduce one of its files +locally with `--keep`, fix the matching blink rule, and re-run. Verify the change +against **every** corpus before lowering a threshold — a fix that helps one +codebase can regress another. diff --git a/blink/parity/corpora/laravel.json b/blink/parity/corpora/laravel.json new file mode 100644 index 0000000000..66ff8af37d --- /dev/null +++ b/blink/parity/corpora/laravel.json @@ -0,0 +1,8 @@ +{ + "name": "laravel", + "repo": "https://github.com/laravel/framework.git", + "paths": ["src", "tests"], + "skip": ["*/node_modules/*", "*/Fixture/*"], + "sets": ["psr12", "perCs", "common", "standaloneLine", "cleanCode"], + "max_diff_percent": 2.0 +} diff --git a/blink/parity/corpora/mautic.json b/blink/parity/corpora/mautic.json new file mode 100644 index 0000000000..14e4e24769 --- /dev/null +++ b/blink/parity/corpora/mautic.json @@ -0,0 +1,8 @@ +{ + "name": "mautic", + "repo": "https://github.com/mautic/mautic.git", + "paths": ["app", "plugins"], + "skip": ["*/node_modules/*", "*/Fixture/*"], + "sets": ["psr12", "perCs", "common", "standaloneLine", "cleanCode"], + "max_diff_percent": 0.07 +} diff --git a/blink/parity/corpora/rector-src.json b/blink/parity/corpora/rector-src.json new file mode 100644 index 0000000000..e3063da1e8 --- /dev/null +++ b/blink/parity/corpora/rector-src.json @@ -0,0 +1,8 @@ +{ + "name": "rector-src", + "repo": "https://github.com/rectorphp/rector-src.git", + "paths": ["src", "rules", "tests"], + "skip": ["*/node_modules/*", "*/Fixture/*"], + "sets": ["psr12", "common"], + "max_diff_percent": 0.0 +} diff --git a/blink/parity/corpora/rector.json b/blink/parity/corpora/rector.json new file mode 100644 index 0000000000..c73f02b732 --- /dev/null +++ b/blink/parity/corpora/rector.json @@ -0,0 +1,8 @@ +{ + "name": "rector", + "repo": "https://github.com/rectorphp/rector.git", + "paths": ["src", "rules", "tests", "packages"], + "skip": ["*/node_modules/*", "*/Fixture/*"], + "sets": ["psr12", "perCs", "common", "standaloneLine", "cleanCode"], + "max_diff_percent": 1.0 +} diff --git a/blink/parity/run.sh b/blink/parity/run.sh new file mode 100755 index 0000000000..2baae18c30 --- /dev/null +++ b/blink/parity/run.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# +# Blink parity harness: run blink (--blink) and the PHP ECS engine over the same +# real-world corpus with the same prepared sets, then report every file whose +# blink output differs from ECS, grouped by the ECS checker that caused it. +# +# Usage: +# blink/parity/run.sh [--workdir DIR] [--keep] +# +# A corpus.json describes one codebase: +# { +# "name": "laravel", +# "repo": "https://github.com/laravel/framework.git", +# "ref": "HEAD", // optional, default HEAD +# "paths": ["src", "tests"], // dirs to check, relative to the clone +# "skip": ["*/node_modules/*"], // optional ECS skip patterns +# "sets": ["psr12","perCs","common","standaloneLine","cleanCode"], +# "max_diff_percent": 2.0 // gate: non-zero exit when exceeded +# } +# +# Exit status: 0 when the differing share is within max_diff_percent, 1 when it +# is exceeded, 2 on a setup error. A $GITHUB_STEP_SUMMARY, when set, receives the +# same categorized report as a collapsible markdown table. +set -euo pipefail + +fail() { echo "parity: $*" >&2; exit 2; } + +CORPUS="" +WORKDIR="" +KEEP=0 +while [ $# -gt 0 ]; do + case "$1" in + --workdir) WORKDIR="$2"; shift 2 ;; + --keep) KEEP=1; shift ;; + -*) fail "unknown flag $1" ;; + *) CORPUS="$1"; shift ;; + esac +done +[ -n "$CORPUS" ] && [ -f "$CORPUS" ] || fail "corpus json not found: $CORPUS" +command -v jq >/dev/null || fail "jq is required" +command -v go >/dev/null || fail "go is required" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +[ -x "$REPO_ROOT/bin/ecs" ] || fail "bin/ecs not found at $REPO_ROOT" + +name=$(jq -r '.name' "$CORPUS") +repo=$(jq -r '.repo' "$CORPUS") +ref=$(jq -r '.ref // "HEAD"' "$CORPUS") +threshold=$(jq -r '.max_diff_percent' "$CORPUS") +mapfile -t paths < <(jq -r '.paths[]' "$CORPUS") +[ "${#paths[@]}" -gt 0 ] || fail "corpus $name has no paths" + +if [ -z "$WORKDIR" ]; then + WORKDIR="$(mktemp -d)" + [ "$KEEP" -eq 1 ] || trap 'rm -rf "$WORKDIR"' EXIT +fi +mkdir -p "$WORKDIR" + +# 1. build blink from this checkout +blink_bin="$WORKDIR/blink" +( cd "$REPO_ROOT/blink" && go build -o "$blink_bin" . ) || fail "blink build failed" + +# 2. the shared ECS config: the corpus paths, skips and prepared sets +ecs_php="$WORKDIR/ecs.php" +{ + echo 'withPreparedSets(${sets_args})" + skip_args=$(jq -r '(.skip // []) | map("'"'"'\(.)'"'"'") | join(", ")' "$CORPUS") + [ -n "$skip_args" ] && echo " ->withSkip([${skip_args}])" + echo ' ->withoutParallel();' +} > "$ecs_php" + +# 3. fetch the corpus once, then two identical copies +src="$WORKDIR/corpus" +rm -rf "$src" +git clone --depth 1 --branch "$ref" "$repo" "$src" >/dev/null 2>&1 \ + || git clone --depth 1 "$repo" "$src" >/dev/null 2>&1 \ + || fail "clone failed: $repo" +rev=$(git -C "$src" rev-parse --short HEAD 2>/dev/null || echo "?") +ours="$WORKDIR/ours"; ecs="$WORKDIR/ecs" +rm -rf "$ours" "$ecs"; cp -r "$src" "$ours"; cp -r "$src" "$ecs" + +abs_paths_ours=(); abs_paths_ecs=(); abs_paths_src=() +for p in "${paths[@]}"; do + [ -d "$ours/$p" ] && abs_paths_ours+=("$ours/$p") + [ -d "$ecs/$p" ] && abs_paths_ecs+=("$ecs/$p") + [ -d "$src/$p" ] && abs_paths_src+=("$src/$p") +done + +# 4. fix one tree with blink, the other with the PHP engine +ECS_BLINK_BIN="$blink_bin" "$REPO_ROOT/bin/ecs" check --fix --no-progress-bar \ + --config "$ecs_php" --blink "${abs_paths_ours[@]}" >/dev/null 2>&1 || true +"$REPO_ROOT/bin/ecs" check --fix --no-progress-bar \ + --config "$ecs_php" "${abs_paths_ecs[@]}" >/dev/null 2>&1 || true + +# 5. the checkers ECS applied on the pristine tree, as JSON, for attribution +ecs_json="$WORKDIR/ecs.json" +"$REPO_ROOT/bin/ecs" check --no-progress-bar --output-format=json \ + --config "$ecs_php" "${abs_paths_src[@]}" > "$ecs_json" 2>/dev/null || echo '{}' > "$ecs_json" + +# 6. diff the two trees, attribute each differing file to the ECS checkers that +# touched it on the pristine sample +report="$WORKDIR/report.txt" +: > "$report" +total=0; differ=0 +declare -A by_checker +while IFS= read -r f; do + total=$((total + 1)) + diff -q "$ours/$f" "$ecs/$f" >/dev/null 2>&1 && continue + differ=$((differ + 1)) + abs="$src/$f" + checkers=$(jq -r --arg k "$abs" '.files[$k].diffs[]?.applied_checkers[]?' "$ecs_json" 2>/dev/null \ + | sed -E 's#.*\\##' | sort -u | paste -sd, - || true) + echo "--- $f${checkers:+ [${checkers}]}" >> "$report" + for c in $(echo "$checkers" | tr ',' ' '); do by_checker[$c]=$(( ${by_checker[$c]:-0} + 1 )); done +done < <(cd "$ours" && find "${paths[@]}" -name '*.php' 2>/dev/null | sort) + +pct=$(awk "BEGIN{ if ($total>0) printf \"%.2f\", $differ*100/$total; else print \"0.00\" }") + +emit() { + echo "## blink vs ECS parity: $name @ $rev" + echo + echo "| metric | value |" + echo "|---|---|" + echo "| files | $total |" + echo "| differing | $differ |" + echo "| diff | ${pct}% (max ${threshold}%) |" + echo + echo "### differing files by ECS checker" + echo + echo "| count | checker |" + echo "|---|---|" + for c in "${!by_checker[@]}"; do echo "${by_checker[$c]} $c"; done | sort -rn \ + | while read -r n c; do echo "| $n | \`$c\` |"; done +} +emit +[ -n "${GITHUB_STEP_SUMMARY:-}" ] && emit >> "$GITHUB_STEP_SUMMARY" +echo +echo "report: $report" +echo "identical=$((total - differ)) differing=$differ of $total (${pct}% diff, max ${threshold}%)" + +awk "BEGIN{ exit !($differ*100/$total > $threshold) }" && { + echo "::error::blink differs from ECS on ${pct}% of $name files ($differ of $total), over ${threshold}%" + exit 1 +} +exit 0