diff --git a/.gitattributes b/.gitattributes index f8f83cc..67a056f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -102,6 +102,20 @@ Containerfile text eol=lf *.rlib binary *.beam binary +# --- Sequencing data ------------------------------------------------------- +# History carried 269 MiB of these. They are binary, they must never be +# line-ending-normalised (a CRLF rewrite corrupts a gzip member), and they are +# not source. The recurrence guard with teeth is scripts/check-blob-hygiene.sh, +# run by .githooks/pre-commit and by CI; these lines are hygiene, not the gate. +*.fastq binary -diff linguist-generated=true +*.fastq.gz binary -diff linguist-generated=true +*.fq binary -diff linguist-generated=true +*.fq.gz binary -diff linguist-generated=true +*.fasta binary -diff linguist-generated=true +*.fa binary -diff linguist-generated=true +*.sam binary -diff linguist-generated=true +*.bam binary -diff linguist-generated=true + # --- Generated lockfiles: no diff noise, not counted as source ------------- Cargo.lock text eol=lf -diff linguist-generated=true mix.lock text eol=lf -diff linguist-generated=true diff --git a/.githooks/commit-msg b/.githooks/commit-msg old mode 100644 new mode 100755 diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..2ff0225 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +# +# pre-commit hook -- refuses a commit that would reintroduce the sequencing +# blobs stripped from history. Enable once per clone with `just hooks` +# (or `git config core.hooksPath .githooks`). +# +# The rule itself lives in scripts/check-blob-hygiene.sh and is shared verbatim +# with the CI blob-hygiene step, so the local gate and the remote gate cannot +# disagree about what is allowed. +exec "$(git rev-parse --show-toplevel)/scripts/check-blob-hygiene.sh" --staged diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 34700eb..4c35ce6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,17 @@ jobs: continue-on-error: ${{ github.repository != 'hyperpolymath/MetaManifold-WebUI' }} run: scripts/check-lint.sh + # Blob-hygiene mirror of .githooks/pre-commit. Both callers exec the SAME + # script, so the local gate and this one cannot disagree about what is + # allowed -- a CI check that re-implements a hook drifts from it, and the + # drift is invisible because both keep passing. + # + # --tree, not a commit range: the checkout above is depth 2, and a whole- + # tree check also catches anything that landed before the guard existed. + - name: Blob hygiene check + continue-on-error: ${{ github.repository != 'hyperpolymath/MetaManifold-WebUI' }} + run: scripts/check-blob-hygiene.sh --tree HEAD + # Commit convention mirror of .githooks/commit-msg (available locally via # the hooksPath setting documented in CONTRIBUTING.md). Advisory only when # this runs under upstream, whose commit subjects need not match. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9fd11f6..9f91906 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,10 +49,19 @@ Allowed types: `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, checklist is in `.gitmessage`: ```bash -git config commit.template .gitmessage # one-time, loads it -git config core.hooksPath .githooks # one-time, enables commit-msg gate +just hooks # one-time, enables the commit-msg + # and pre-commit gates +git config commit.template .gitmessage # one-time, loads the template ``` +`just hooks` is already a dependency of `just bootstrap`, so a clone that was +bootstrapped has both gates live. It only sets `core.hooksPath`, which is local +config and therefore cannot be committed — that is why it has to be a command +rather than a file. The hooks are `commit-msg` (conventional-commit subject) and +`pre-commit` (blob hygiene: no uncompressed sequencing data, nothing over 4 MiB). +CI re-checks both, so forgetting to run this costs a red build, not a bad commit +on `main`. + --- diff --git a/Justfile b/Justfile index c95d34d..7ad675d 100644 --- a/Justfile +++ b/Justfile @@ -112,9 +112,18 @@ setup: install # mise.toml (julia 1.12.5, bun 1.3.10, node 20.20.2, just 1.43.1), then # install frontend dependencies. R is a documented exception: system R + # renv.lock (R is not in the mise registry — verified 2026-09-18). -bootstrap: setup-tools install codegen-tools +bootstrap: setup-tools install codegen-tools hooks @echo "bootstrap: toolchain + deps + machine tool map ready — next: just ci" +# Point git at .githooks so the commit-msg gate actually runs. core.hooksPath is +# per-clone local config -- it cannot be committed -- so documenting it in +# CONTRIBUTING.md left it unset in every clone that did not read that line. +# Wiring it here makes the enablement a consequence of bootstrapping rather than +# of remembering. Idempotent; safe to re-run. +hooks: + @git config core.hooksPath .githooks + @echo "hooks: core.hooksPath -> .githooks (commit-msg gate live)" + # Provision the pinned toolchain via mise (fail-loud with the installer # one-liner when mise is absent; the Guix lane in guix.scm is the # alternative, see docs/reproducibility.md). diff --git a/docs/compliance/standards-alignment.md b/docs/compliance/standards-alignment.md index a07c0d3..b124fbf 100644 --- a/docs/compliance/standards-alignment.md +++ b/docs/compliance/standards-alignment.md @@ -46,9 +46,17 @@ Reference: `hyperpolymath/standards@main` (in particular | Expectation | Here | Status | |---|---|---| | Documented | `CONTRIBUTING.md` + `.gitmessage` template (`git config commit.template .gitmessage`) | ✅ | -| Enforced | `.githooks/commit-msg` (`git config core.hooksPath .githooks`) **and** CI repo-hygiene subject check | ✅ | +| Enforced (the gate) | CI repo-hygiene `Commit convention check` — binding on this repo, advisory only under upstream (`continue-on-error: github.repository != 'hyperpolymath/MetaManifold-WebUI'`) | ✅ | +| Enforced (local pre-flight) | `.githooks/commit-msg`, enabled by `just hooks` (a `just bootstrap` dependency) | ⚠ opt-in per clone — `core.hooksPath` is local config and cannot be committed | | Type list | `feat fix docs style refactor perf test build ci chore revert` | ✅ canonical list | +## History hygiene + +| Expectation | Here | Status | +|---|---|---| +| Large/dead blobs kept out | `scripts/check-blob-hygiene.sh` — one implementation, two callers: `.githooks/pre-commit` (local) and the CI repo-hygiene `Blob hygiene check` (binding on this repo). Primary rule is a 4 MiB size ceiling, not a path list; the six `data/MiSeq_SOP/run_[AB]/*.fastq.gz` fixtures are allowlisted | ✅ verified by mutant — five reintroduction attempts refused, two legitimate files admitted | +| Diff/linguist markings | `.gitattributes` marks `*.fastq{,.gz}`, `*.fq{,.gz}`, `*.fasta`, `*.fa`, `*.sam`, `*.bam` binary `-diff linguist-generated=true` | ✅ hygiene only, not the gate | + ## Branch conventions Documented in `CONTRIBUTING.md`: `feat/… fix/… chore/… test/… docs/…`, diff --git a/scripts/check-blob-hygiene.sh b/scripts/check-blob-hygiene.sh new file mode 100755 index 0000000..898b4b1 --- /dev/null +++ b/scripts/check-blob-hygiene.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +# +# Blob-hygiene guard: stop the 269 MiB of sequencing data from coming back. +# +# History carried 269.35 MiB of packed objects, ~96.6% of it dead weight: +# 163.09 MiB of uncompressed *.fastq, 143.17 MiB under data/VESPA_pool/, +# 71.56 MiB under data/Multiplex_pool/ (which ALSO lived at inputs/fastq/), +# 20.13 MiB under a long-gone web/, and 18.84 MiB across 1,322 node_modules +# blobs. Nothing in the repo prevented any of that, and -- until this file -- +# nothing prevented it returning after the rewrite. +# +# ONE implementation, TWO callers: .githooks/pre-commit and the CI +# blob-hygiene step both exec this script. That is deliberate. A hook and a CI +# check that re-implement the same rule drift apart, and the drift is invisible +# because both keep reporting success. +# +# THE PRIMARY RULE IS SIZE, NOT PATH. A path rule can only forbid the paths +# somebody already thought of; the Multiplex pool survived a history rewrite +# precisely because it was reachable under a second path nobody had enumerated. +# Size catches the class. The path rules below are secondary -- they exist to +# produce a better error message, not to be the gate. +# +# Usage: +# check-blob-hygiene.sh --staged # pre-commit: what is about to land +# check-blob-hygiene.sh --tree [ref] # CI: every tracked file at a ref +set -euo pipefail + +# Ceiling: the largest legitimate tracked file is 2,723,348 B (a MiSeq_SOP +# fixture). 4 MiB leaves headroom for a comparable fixture without admitting +# anything of the order that caused the problem -- the dead pools ran to tens +# of MiB per blob. +MAX_BLOB_BYTES=$((4 * 1024 * 1024)) + +# The six deliberately-whitelisted fixtures (.gitignore:294-304). Matched as a +# glob, so a further fixture in run_A/run_B is admitted while +# data//*.fastq.gz is not. +ALLOW_GLOB='data/MiSeq_SOP/run_[AB]/*.fastq.gz' + +violations=0 + +fail() { + printf 'BLOB HYGIENE: %s\n' "$1" >&2 + violations=$((violations + 1)) +} + +# Every rule, applied to one (path, size) pair. Both modes call exactly this +# function, so neither mode can end up with a filter the other lacks. +check_one() { + local path="$1" size="$2" + + # shellcheck disable=SC2254 + case "$path" in + $ALLOW_GLOB) return 0 ;; + esac + + # 1. Uncompressed sequencing data: 48 blobs, 163.09 MiB, none of which ever + # existed in a working tree. Anchored so .gz never matches -- the live + # fixtures are compressed and must survive. + case "$path" in + *.fastq|*.fq|*.fasta|*.fa|*.sam) + fail "$path -- uncompressed sequencing data must never be committed (gzip it, or keep it out of the repo)" + ;; + esac + + # 2. Paths that have already done this once. + case "$path" in + node_modules/*|*/node_modules/*) + fail "$path -- node_modules was committed once before (1,322 blobs, 18.84 MiB)" + ;; + inputs/*|*/inputs/*) + fail "$path -- inputs/ held a second copy of the Multiplex pool and survived a history rewrite by being reachable under two paths" + ;; + logs_*.zip|*/logs_*.zip) + fail "$path -- committed CI log artefact" + ;; + esac + + # 3. The rule that catches what the rules above did not think of. + if [ "$size" -gt "$MAX_BLOB_BYTES" ]; then + fail "$path -- $size bytes exceeds the $MAX_BLOB_BYTES byte ceiling" + fi +} + +mode="${1:---staged}" + +case "$mode" in + --staged) + # Added or modified index entries, NUL-separated so a path containing a + # space or newline cannot split into two. + while IFS= read -r -d '' path; do + blob="$(git ls-files -s -- "$path" | awk '{print $2}')" + [ -n "$blob" ] || continue + size="$(git cat-file -s "$blob")" + check_one "$path" "$size" + done < <(git diff --cached --name-only --diff-filter=AM -z) + ;; + --tree) + # Every tracked file at a ref, rather than a commit range. A range needs + # history CI does not fetch (the hygiene job checks out at depth 2), and + # the whole tree also catches anything that landed before this guard + # existed -- which a range, by construction, cannot. + # + # Deliberately NOT `git rev-list --objects`: that emits each object once + # paired with only ONE of the paths it is reachable under, which is + # exactly how the duplicated pool went unnoticed. + ref="${2:-HEAD}" + while IFS= read -r -d '' path; do + blob="$(git rev-parse --quiet --verify "$ref:$path" 2>/dev/null || true)" + [ -n "$blob" ] || continue + size="$(git cat-file -s "$blob")" + check_one "$path" "$size" + done < <(git ls-tree -r -z --name-only "$ref") + ;; + *) + echo "usage: $0 --staged | --tree [ref]" >&2 + exit 2 + ;; +esac + +if [ "$violations" -gt 0 ]; then + echo "" >&2 + echo "$violations violation(s). If one is a deliberate fixture, add it to" >&2 + echo "ALLOW_GLOB in scripts/check-blob-hygiene.sh and say why in the commit." >&2 + exit 1 +fi + +echo "blob hygiene: ok"