diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index f7c7f017..d2583657 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -100,6 +100,10 @@ jobs: has_v: ${{ steps.scan.outputs.has_v }} has_haskell: ${{ steps.scan.outputs.has_haskell }} has_deno: ${{ steps.scan.outputs.has_deno }} + # Ecosystems this pipeline can DETECT but has no job to CHECK. Reported + # separately from `total` on purpose — see the scan step's comment. + n_unsupported: ${{ steps.scan.outputs.n_unsupported }} + unsupported: ${{ steps.scan.outputs.unsupported }} # The Deno gate REFUSES and names what it found, so it needs the # count itself, not merely the boolean. n_deno: ${{ steps.scan.outputs.n_deno }} @@ -121,6 +125,9 @@ jobs: git ls-files -z -- "$@" | tr -cd '\0' | wc -c } + # ── SUPPORTED ecosystems ──────────────────────────────────────── + # Every probe here has a CONSUMING job later in this file. That is + # the entry requirement for this list, not a coincidence. N_RUST=$(count 'Cargo.toml' '*/Cargo.toml') N_NICKEL=$(count '*.ncl') N_RESCRIPT=$(count 'rescript.json' '*/rescript.json' 'bsconfig.json' '*/bsconfig.json' '*.res') @@ -129,10 +136,71 @@ jobs: # a V toolchain to lint a Coq development. N_V=$(count 'v.mod' '*/v.mod') N_HASKELL=$(count '*.cabal' 'stack.yaml' '*/stack.yaml' '*.hs') + # ⚠ DO NOT REMOVE THE DENO PROBE. It does not exist to lint Deno — + # it is the trigger for the REFUSAL job `deno` below, i.e. the teeth + # of the estate's standing no-Deno ruling, with slugs still live in + # `.machine_readable/deno-allow.txt`. Deleting the probe would not + # "drop support"; it would silently un-enforce a ban, because the + # refusal simply stops firing and the repo goes green. N_DENO=$(count 'deno.json' '*/deno.json' 'deno.jsonc' '*/deno.jsonc') + # TOTAL counts ONLY ecosystems some job in this file actually + # examines. Deno counts: a refusal IS a check. TOTAL=$(( N_RUST + N_NICKEL + N_RESCRIPT + N_V + N_HASKELL + N_DENO )) + # ── DETECTED BUT UNSUPPORTED ──────────────────────────────────── + # These have NO lint or format job in this file. They are counted + # SEPARATELY and deliberately DO NOT enter TOTAL. + # + # ⚠ THAT SEPARATION IS THE ENTIRE POINT OF THIS SECTION. Folding + # them into TOTAL would lift it above zero and SILENCE the + # zero-denominator refusal below while nothing whatsoever had been + # checked. That is a vacuous gate, and a vacuous gate is worse than + # an honest blind spot: the blind spot reports nothing, the vacuous + # gate reports success. A probe moves from this list to the one + # above ONLY in the same commit that adds its consuming job. + # + # Keyed on MANIFESTS, not extensions, for the reason `v.mod` already + # is: an extension is shared between languages, a manifest is not. + UNSUPPORTED="" + N_UNSUPPORTED=0 + probe() { + local name="$1"; shift + local n + n=$(count "$@") + if [ "$n" -gt 0 ]; then + UNSUPPORTED="${UNSUPPORTED:+${UNSUPPORTED}, }${name} (${n})" + N_UNSUPPORTED=$(( N_UNSUPPORTED + n )) + fi + } + + probe 'Julia' 'Project.toml' '*/Project.toml' + probe 'Zig' 'build.zig' '*/build.zig' 'build.zig.zon' '*/build.zig.zon' + probe 'Idris2' '*.ipkg' + probe 'Elixir' 'mix.exs' '*/mix.exs' + probe 'Lean' 'lakefile.toml' '*/lakefile.toml' 'lakefile.lean' '*/lakefile.lean' 'lean-toolchain' '*/lean-toolchain' + probe 'Agda' '*.agda-lib' + probe 'Ada' 'alire.toml' '*/alire.toml' '*.gpr' + probe 'Ruby' 'Gemfile' '*/Gemfile' '*.gemspec' + probe 'PHP' 'composer.json' '*/composer.json' + probe 'Gleam' 'gleam.toml' '*/gleam.toml' + # ⚠ `ROOT` is an unusually generic filename and this probe is the + # most likely of the set to produce a FALSE POSITIVE. It is tolerable + # only because a false positive here cannot bless anything: it can + # only move a repo from "nothing detected" to "detected, unchecked", + # and both refuse. It must NOT be promoted to the supported list on + # this evidence alone. + probe 'Isabelle' 'ROOT' '*/ROOT' + # Bun. `bun.lock*` and `bunfig.toml` are positive identification. + # ⚠ `package.json` ALONE is an ASSUMPTION, stated here rather than + # hidden: the estate's standing ruling is bun-only, with Node and + # Deno banned, so a bare package.json in this estate is presumed to + # be a Bun manifest. That assumption is safe while Bun sits in THIS + # list — it yields a refusal, never a pass. Re-examine it on the day + # a Bun lint job is added, because then it would start selecting + # which repositories get linted. + probe 'Bun' 'bun.lock' '*/bun.lock' 'bun.lockb' '*/bun.lockb' 'bunfig.toml' '*/bunfig.toml' 'package.json' '*/package.json' + bool() { [ "$1" -gt 0 ] && echo true || echo false; } { @@ -144,6 +212,8 @@ jobs: echo "has_deno=$(bool "$N_DENO")" echo "n_deno=$N_DENO" echo "total=$TOTAL" + echo "n_unsupported=$N_UNSUPPORTED" + echo "unsupported=$UNSUPPORTED" } >> "$GITHUB_OUTPUT" # Print the denominator. A gate that never shows what it measured @@ -159,17 +229,36 @@ jobs: echo "| V | \`v.mod\` only | ${N_V} |" echo "| Haskell | \`*.cabal\`, \`stack.yaml\`, \`*.hs\` | ${N_HASKELL} |" echo "| Deno | \`deno.json(c)\` | ${N_DENO} |" - echo "| **Total** | | **${TOTAL}** |" + echo "| **Total checkable** | | **${TOTAL}** |" echo "" + if [ "$N_UNSUPPORTED" -gt 0 ]; then + echo "#### Detected, but this pipeline has no gate for it" + echo "" + echo "${UNSUPPORTED}" + echo "" + echo "_These files were found and **not checked**. They are excluded from the" + echo "total above on purpose: counting them would hide the fact that nothing" + echo "examined them. Tracked as hyperpolymath/standards#967._" + echo "" + fi } >> "$GITHUB_STEP_SUMMARY" if [ "$TOTAL" -eq 0 ]; then + # Same switch, two different truths. Telling them apart is what + # makes the adoption ledger measure real debt instead of detector + # blindness: "we have no gate for Julia" and "we could not + # recognise this repository at all" need different cures. + if [ "$N_UNSUPPORTED" -gt 0 ]; then + MSG="Detected ${UNSUPPORTED}, but this pipeline has no lint or format job for any of them, so NOTHING was checked. This is measured debt, not an unrecognised repository — see hyperpolymath/standards#967." + else + MSG="No known ecosystem detected (denominator = 0). Every lint and format gate would be vacuous, so this pipeline refuses to report success. Add the ecosystem's marker file, or call this workflow with fail_on_no_ecosystem: false and say why." + fi if [ "${{ inputs.fail_on_no_ecosystem }}" = "true" ]; then - echo "::error::No known ecosystem detected (denominator = 0). Every lint and format gate would be vacuous, so this pipeline refuses to report success. Add the ecosystem's marker file, or call this workflow with fail_on_no_ecosystem: false and say why." - echo "**REFUSED: denominator is zero — no ecosystem matched, so nothing could be checked.**" >> "$GITHUB_STEP_SUMMARY" + echo "::error::${MSG}" + echo "**REFUSED: nothing could be checked.** ${MSG}" >> "$GITHUB_STEP_SUMMARY" exit 1 fi - echo "::warning::No known ecosystem detected; fail_on_no_ecosystem is false, so continuing with nothing to check." + echo "::warning::${MSG} fail_on_no_ecosystem is false, so continuing with nothing checked." fi # ─────────────────────────────────────────────────────────────────────── @@ -717,6 +806,8 @@ jobs: H_RESCRIPT: ${{ needs.detect.outputs.has_rescript }} H_V: ${{ needs.detect.outputs.has_v }} H_HASKELL: ${{ needs.detect.outputs.has_haskell }} + N_UNSUP: ${{ needs.detect.outputs.n_unsupported }} + UNSUP: ${{ needs.detect.outputs.unsupported }} run: | set -euo pipefail @@ -750,6 +841,18 @@ jobs: echo "" echo "_A skipped gate examined nothing. It is not a pass._" echo "" + # ⚠ NOT a row(). These ecosystems have no job, so they have no + # `needs.*.result` to report — and row()'s catch-all branch prints + # the word "ran" for an unset result, which would assert the exact + # opposite of the truth here. Say it in prose instead. + if [ "${N_UNSUP:-0}" -gt 0 ] 2>/dev/null; then + echo "> **Detected with no gate at all:** ${UNSUP}" + echo ">" + echo "> No job in this pipeline examines these. They are absent from the table" + echo "> above because there is no result to report, not because they passed." + echo "> Tracked as hyperpolymath/standards#967." + fi + echo "" } >> "$GITHUB_STEP_SUMMARY" # ── The pipeline adoption ratchet ──────────────────────────────── diff --git a/scripts/tests/detect-ecosystem-test.sh b/scripts/tests/detect-ecosystem-test.sh new file mode 100755 index 00000000..9d61d0c9 --- /dev/null +++ b/scripts/tests/detect-ecosystem-test.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# +# Regression suite for ci-pipeline.yml's `detect` scan step. +# +# ⚠ THIS SUITE EXECUTES THE SHIPPED YAML, NOT A COPY OF IT. The step body is +# extracted from .github/workflows/ci-pipeline.yml at run time. A test that +# re-implements the logic it is testing passes forever after the real thing +# breaks, which is the failure mode this file exists to avoid. +# +# The property under test is NOT "detection works". It is: +# +# an ecosystem the pipeline cannot CHECK must never lift TOTAL above zero +# +# because TOTAL == 0 is what triggers the refusal. A probe added to the wrong +# accumulator silences the refusal while examining nothing — a vacuous gate, +# which reports success. Fixture 3 and the mutant below are the controls for +# exactly that. +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +WORKFLOW="$ROOT/.github/workflows/ci-pipeline.yml" +PASS=0; FAIL=0 +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# Extract the `run:` body of the step whose id is `scan`. +# +# No Python: LANGUAGE-POLICY bans it, so the extraction is awk. The block is +# the run-on scalar after `run: |`, i.e. every following line indented deeper +# than the `run:` key itself, plus blank lines. 10 leading spaces are stripped. +extract_scan() { + awk ' + /^ id: scan$/ { seen=1 } + seen && /^ run: \|$/ { grab=1; next } + grab { + if ($0 ~ /^[[:space:]]*$/) { print ""; next } + if ($0 !~ /^ /) { exit } + sub(/^ /, ""); print + } + ' "$WORKFLOW" +} + +SCAN="$WORK/scan.sh" +extract_scan > "$SCAN" +[ -s "$SCAN" ] || { echo "FATAL: extracted an EMPTY scan body — the awk anchors no longer match $WORKFLOW"; exit 2; } +grep -q 'N_UNSUPPORTED=0' "$SCAN" || { echo "FATAL: extraction produced a body with no N_UNSUPPORTED accumulator"; exit 2; } + +# Run the extracted body inside a throwaway git repo holding $* as tracked +# files. Sets: T (total), U (n_unsupported), RC, and OUT (stderr+stdout). +run_fixture() { + local fail_on="$1"; shift + local mutate="$1"; shift + local repo; repo="$(mktemp -d -p "$WORK")" + ( + cd "$repo" || exit 9 + git init -q . && git config user.email t@t && git config user.name t + for f in "$@"; do mkdir -p "$(dirname "$f")"; : > "$f"; done + [ $# -gt 0 ] && git add -A && git -c commit.gpgsign=false commit -qm x >/dev/null 2>&1 + cp "$SCAN" ./s.sh + # The workflow expression is not shell; substitute it as Actions would. + sed -i "s|\${{ inputs.fail_on_no_ecosystem }}|$fail_on|g" ./s.sh + [ -n "$mutate" ] && sed -i "$mutate" ./s.sh + GITHUB_OUTPUT=./out GITHUB_STEP_SUMMARY=./sum bash ./s.sh 2>&1 + echo "__RC__=$?" + cat ./out 2>/dev/null + ) > "$repo/.captured" 2>&1 + OUT="$(cat "$repo/.captured")" + RC="$(sed -n 's/^__RC__=//p' <<< "$OUT" | tail -1)" + T="$(sed -n 's/^total=//p' <<< "$OUT" | tail -1)" + U="$(sed -n 's/^n_unsupported=//p' <<< "$OUT" | tail -1)" +} + +check() { + local what="$1" got="$2" want="$3" + if [ "$got" = "$want" ]; then PASS=$((PASS+1)); printf ' ok %-46s = %s\n' "$what" "$got" + else FAIL=$((FAIL+1)); printf ' FAIL %-46s = %s (wanted %s)\n' "$what" "$got" "$want"; fi +} + +echo "== Fixture 1: empty repo, fail_on_no_ecosystem=true ==" +run_fixture true '' +check "total" "$T" "0" +check "n_unsupported" "$U" "0" +check "exit code" "$RC" "1" +check "names the UNRECOGNISED case" "$(grep -qF 'No known ecosystem detected' <<< "$OUT" && echo y || echo n)" "y" +check "does NOT claim unsupported" "$(grep -qF 'no lint or format job for any of them' <<< "$OUT" && echo y || echo n)" "n" + +echo "== Fixture 2: Cargo.toml only (supported) ==" +run_fixture true '' Cargo.toml +check "total" "$T" "1" +check "n_unsupported" "$U" "0" +check "exit code (green)" "$RC" "0" +check "no refusal text" "$(grep -qF 'REFUSED' <<< "$OUT" && echo y || echo n)" "n" + +echo "== Fixture 3: Project.toml only (DETECTED, UNSUPPORTED) ==" +echo " the load-bearing case: Julia must NOT lift total above zero" +run_fixture true '' Project.toml +check "total STAYS zero" "$T" "0" +check "n_unsupported" "$U" "1" +check "still REFUSES" "$RC" "1" +check "names Julia" "$(grep -qF 'Julia (1)' <<< "$OUT" && echo y || echo n)" "y" +check "uses the DEBT message" "$(grep -qF 'no lint or format job for any of them' <<< "$OUT" && echo y || echo n)" "y" +check "cites #967" "$(grep -qF 'standards#967' <<< "$OUT" && echo y || echo n)" "y" +check "NOT the unrecognised msg" "$(grep -qF 'No known ecosystem detected' <<< "$OUT" && echo y || echo n)" "n" + +echo "== Fixture 3b: same, fail_on_no_ecosystem=false ==" +run_fixture false '' Project.toml +check "warns, does not fail" "$RC" "0" +check "warning names Julia" "$(grep -qF '::warning::Detected Julia (1)' <<< "$OUT" && echo y || echo n)" "y" + +echo "== Fixture 4: both Cargo.toml and Project.toml ==" +run_fixture true '' Cargo.toml Project.toml +check "total counts ONLY rust" "$T" "1" +check "unsupported counts julia" "$U" "1" +check "green (something checked)" "$RC" "0" + +echo "== Fixture 5: Bun markers ==" +run_fixture true '' package.json bunfig.toml +check "total" "$T" "0" +check "n_unsupported" "$U" "2" +check "names Bun" "$(grep -qF 'Bun (2)' <<< "$OUT" && echo y || echo n)" "y" +check "refuses" "$RC" "1" + +echo "== Fixture 6: deno.json still trips the BAN probe ==" +run_fixture true '' deno.json +check "has_deno true" "$(grep -qF 'has_deno=true' <<< "$OUT" && echo y || echo n)" "y" +check "n_deno" "$(sed -n 's/^n_deno=//p' <<< "$OUT" | tail -1)" "1" +check "counts toward total" "$T" "1" + +echo "== MUTANT: delete the Julia probe. Fixture 3 must change its answer. ==" +echo " A suite that only ever goes green proves nothing." +run_fixture true "/probe 'Julia'/d" Project.toml +check "mutant: unsupported now 0" "$U" "0" +check "mutant: falls back to the UNRECOGNISED message" \ + "$(grep -qF 'No known ecosystem detected' <<< "$OUT" && echo y || echo n)" "y" +check "mutant: no longer names Julia" \ + "$(grep -qF 'Julia' <<< "$OUT" && echo y || echo n)" "n" + +echo "== MUTANT 2: fold unsupported into TOTAL. The refusal must go SILENT. ==" +echo " This is the vacuous gate this design exists to prevent; if the" +echo " mutant still refuses, the separation is not what makes it refuse." +# Folded AFTER the probes run, which is the only place the fold could +# plausibly be written by someone "tidying up" the two accumulators into one. +run_fixture true 's|^bool() {|TOTAL=$(( TOTAL + N_UNSUPPORTED ))\nbool() {|' Project.toml +check "mutant2: total wrongly non-zero" "$T" "1" +check "mutant2: refusal SILENCED (rc=0)" "$RC" "0" + +echo "" +echo "detect-ecosystem-test: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ]