diff --git a/.gitattributes b/.gitattributes index 67a056f..a225711 100644 --- a/.gitattributes +++ b/.gitattributes @@ -133,3 +133,44 @@ package-lock.json text eol=lf -diff linguist-generated=true **/*.res linguist-detectable=false *.nix text eol=lf flake.lock text eol=lf -diff linguist-generated=true + +# --- Integration / merge hygiene (fork↔upstream) --------------------------- # +# This fork and upstream share NO git ancestor, so a naive merge conflicts on +# every shared-but-different path. These rules keep the classes of file that +# must NEVER be hand-merged out of the line-conflict set: git will not produce +# conflict markers inside them, so a maintainer resolves them by regenerating +# (`just heal`) or by an explicit `--ours`/`--theirs` pick, never by editing. +# +# `-merge` treats the path as binary for merge purposes (built-in driver; needs +# no per-clone config, so it is safe the moment this file lands). For those who +# want lockfiles to auto-resolve to the target branch and then be regenerated, +# `just merge-drivers` wires the stronger `merge.lockfile` custom driver. + +# Generated lockfiles — regenerate, never line-merge. +Manifest.toml text eol=lf -diff -merge linguist-generated=true +renv.lock text eol=lf -diff -merge linguist-generated=true +frontend/bun.lock text eol=lf -diff -merge linguist-generated=true +bun.lockb binary -diff -merge linguist-generated=true +package-lock.json text eol=lf -diff -merge linguist-generated=true +pnpm-lock.yaml text eol=lf -diff -merge linguist-generated=true +# renv scaffold / generated dependency scan (produced by renv, not authored). +renv/activate.R text eol=lf -diff -merge linguist-generated=true +R/_renv_dependencies.R text eol=lf -diff linguist-generated=true +renv/library/** binary -diff linguist-generated=true +# Machine-local tool-path map (written by scripts/gen-tools-yml.sh). +config/tools.yml text eol=lf -diff linguist-generated=true +config/ci/tools.yml text eol=lf -diff linguist-generated=true + +# Build output — never source, never hand-merged. (Upstream committed web/dist; +# the fork builds frontend/dist instead. Neither belongs in a diff.) +web/dist/** binary -diff linguist-generated=true +frontend/dist/** binary -diff linguist-generated=true +**/dist/** binary -diff linguist-generated=true + +# Test / coverage / benchmark artefacts. +**/coverage/** binary -diff linguist-generated=true +lcov.info text eol=lf -diff linguist-generated=true +*.cov text eol=lf -diff linguist-generated=true +**/test-results/** binary -diff linguist-generated=true +**/*junit.xml text eol=lf -diff linguist-generated=true +**/bench/results/*.json text eol=lf -diff linguist-generated=true diff --git a/.gitignore b/.gitignore index f072065..cdfeec6 100644 --- a/.gitignore +++ b/.gitignore @@ -263,6 +263,8 @@ docs/* !docs/types/ !docs/reproducibility.md !docs/owner-review-2026-09-25.md +!docs/integration/ +!docs/integration/** # Test + benchmark run artifacts (not the committed fixtures/baselines) frontend/tests/results/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f91906..75ad856 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,21 @@ Pull requests against this fork must use base `hyperpolymath/MetaManifold-WebUI:main`. GitHub's fork PR page defaults the base to the upstream parent — change it before clicking *Create*. +## Landing fork work on upstream (low-friction integration) + +This fork and upstream (`JoshuaJewell/MetaManifold-WebUI`) share no git +ancestor, so a naive merge conflicts on every shared-but-different file. To +integrate without a wall of conflicts — and to let the maintainer adopt the work +incrementally, from "pure upstream" to "partially transitional" to "everything", +without ever breaking a running system — see: + +- **`docs/integration/README.md`** — the guide (profiles, merge hygiene, staging). +- **`docs/integration/conflict-map-2026-09-25.md`** — the measured conflict set. +- **`docs/integration/HANDOFF-granular-reanchor.md`** — the brief to actually land it. +- `just reanchor-plan` / `just reanchor` — replay the fork onto upstream as ~206 granular commits (3 decisions, 0 conflicts). +- `just integrate status | profiles | plan | triage` and `just augment`/`just suspend `. +- `just bootstrap` / `just setup-full` / `just heal` / `just doctor` — the turnkey environment. + ## Development setup ```bash diff --git a/Justfile b/Justfile index 7ad675d..94daf40 100644 --- a/Justfile +++ b/Justfile @@ -33,6 +33,12 @@ export METAMANIFOLD_REPO_DIR := justfile_directory() FRONTEND := justfile_directory() / "frontend" +# Integration helper (fork↔upstream profiles, triage, component toggles). +INTEGRATE := justfile_directory() / "scripts/integrate.sh" + +# Re-anchor helper (turn the divergent histories into a granular, mergeable one). +REANCHOR := justfile_directory() / "scripts/reanchor.sh" + # Free-RAM floor (KB) for the heavy Julia lanes: cold JIT-compilation of the # server dependency closure needs several GB; below this the lane fails # loudly instead of thrashing the box into an OOM kill. @@ -77,18 +83,42 @@ info: doctor: #!/usr/bin/env bash rc=0 + # Hard requirement: absent => FAIL and non-zero exit. need() { - if command -v "$1" >/dev/null 2>&1; then printf 'PASS %-10s %s\n' "$1" "$($1 --version 2>&1 | head -1)"; - else printf 'FAIL %-10s %s\n' "$1" "$2"; rc=1; fi + if command -v "$1" >/dev/null 2>&1; then printf 'PASS %-12s %s\n' "$1" "$($1 --version 2>&1 | head -1)"; + else printf 'FAIL %-12s %s\n' "$1" "$2"; rc=1; fi + } + # Soft requirement: absent => WARN, exit stays 0 (a documented lane is just unavailable). + soft() { + if command -v "$1" >/dev/null 2>&1; then printf 'PASS %-12s %s\n' "$1" "$($1 --version 2>&1 | head -1)"; + else printf 'WARN %-12s %s\n' "$1" "$2"; fi } - need bun "install: curl -fsSL https://bun.sh/install | bash" + need bun "install: curl -fsSL https://bun.sh/install | bash (or: just setup-tools)" need git "install via package manager" + soft bunx "ships with bun; if absent reinstall bun" + soft node "needed by vite's production build: just setup-tools" + soft mise "toolchain pins (mise.toml): curl https://mise.run | sh (Guix lane is the alternative)" if $JULIA_CMD --version >/dev/null 2>&1; then - printf 'PASS %-10s %s\n' "julia" "$($JULIA_CMD --version)" + printf 'PASS %-12s %s\n' "julia" "$($JULIA_CMD --version)" + else + printf 'WARN %-12s %s\n' "julia" "Julia lanes unavailable — install via juliaup (install.sh) or just setup-tools" + fi + if command -v Rscript >/dev/null 2>&1; then + printf 'PASS %-12s %s\n' "R" "$(Rscript --version 2>&1 | head -1)" + [ -f renv/activate.R ] && echo "PASS renv renv/activate.R present (restore with: just renv-restore)" \ + || echo "WARN renv renv/activate.R missing — R lane cannot restore" else - printf 'WARN %-10s %s\n' "julia" "Julia lanes unavailable — install via juliaup (install.sh)" + printf 'WARN %-12s %s\n' "R" "system R >= 4.5 not found (documented exception; not in mise registry)" fi - [[ -x "{{LAUNCHER}}" ]] && echo "PASS launcher {{LAUNCHER}}" || { echo "WARN launcher not executable: {{LAUNCHER}}"; } + # Merge drivers make lockfiles auto-resolve on the next fork↔upstream merge. + if git config --get merge.lockfile.driver >/dev/null 2>&1; then + echo "PASS merge-drv merge.lockfile wired (just merge-drivers)" + else + echo "WARN merge-drv not wired — run: just merge-drivers" + fi + [[ -x "{{LAUNCHER}}" ]] && echo "PASS launcher {{LAUNCHER}}" || { echo "WARN launcher not executable: {{LAUNCHER}}"; } + echo "-----" + echo "Integration profile: $({{INTEGRATE}} profile 2>/dev/null || echo base)" exit $rc # Quick repo statistics. @@ -112,8 +142,8 @@ setup: install # mise.toml (julia 1.12.5, bun 1.3.10, node 20.20.2, just 1.43.1), then # install frontend dependencies. R is a documented exception: system R + # renv.lock (R is not in the mise registry — verified 2026-09-18). -bootstrap: setup-tools install codegen-tools hooks - @echo "bootstrap: toolchain + deps + machine tool map ready — next: just ci" +bootstrap: setup-tools install codegen-tools hooks merge-drivers + @echo "bootstrap: toolchain + deps + hooks + merge drivers + machine tool map ready — next: just ci" # Point git at .githooks so the commit-msg gate actually runs. core.hooksPath is # per-clone local config -- it cannot be committed -- so documenting it in @@ -124,6 +154,26 @@ hooks: @git config core.hooksPath .githooks @echo "hooks: core.hooksPath -> .githooks (commit-msg gate live)" +# Wire a git merge driver that keeps lockfiles / generated files out of the +# fork↔upstream conflict set. merge.lockfile auto-resolves such a path to the +# branch being merged INTO (ours) and reminds you to regenerate — never a +# line-merged lockfile. Applied via .git/info/attributes (local, overrides the +# tree, never committed) so it is fully opt-in and cannot break a merge on a +# clone that has not run it. The committed .gitattributes already stops git from +# line-merging these (merge: unset); this just makes the choice automatic. +# Local git config, like core.hooksPath — hence a command, not a committed file. +# Idempotent; safe to re-run. +merge-drivers: + #!/usr/bin/env bash + git config merge.lockfile.name "keep target-branch lockfile, then regenerate (just heal)" + git config merge.lockfile.driver 'echo "merge-drivers: kept target-branch copy of %P — regenerate with: just heal" >&2' + attrs="{{justfile_directory()}}/.git/info/attributes" + mkdir -p "$(dirname "$attrs")"; touch "$attrs" + for p in Manifest.toml renv.lock frontend/bun.lock bun.lockb package-lock.json pnpm-lock.yaml renv/activate.R; do + grep -qxF "$p merge=lockfile" "$attrs" 2>/dev/null || printf '%s merge=lockfile\n' "$p" >> "$attrs" + done + echo "merge-drivers: merge.lockfile wired for lockfiles via .git/info/attributes (opt-in, local)" + # Provision the pinned toolchain via mise (fail-loud with the installer # one-liner when mise is absent; the Guix lane in guix.scm is the # alternative, see docs/reproducibility.md). @@ -167,11 +217,12 @@ codegen-tools: ./scripts/gen-tools-yml.sh # Complete first-run on a bare machine, clone-to-launchable in one recipe: -# toolchain + JS deps + machine tool map (bootstrap), Julia package -# instantiate, then install.sh's sha256-pinned external pipeline tools. -# After this: just start. (install-tools downloads several hundred MB by -# design — skip it when you only develop the frontend.) -setup-full: bootstrap julia-instantiate install-tools +# toolchain + JS deps + machine tool map + hooks + merge drivers (bootstrap), +# Julia package instantiate, R package restore (renv), then install.sh's +# sha256-pinned external pipeline tools. After this: just start. +# (install-tools downloads several hundred MB by design — skip it when you only +# develop the frontend.) +setup-full: bootstrap julia-instantiate renv-restore install-tools @echo "setup-full: complete — launch with: just start" # Pipeline tools via the byte-exact lane: install.sh fetches the archives @@ -191,6 +242,18 @@ outdated: julia-instantiate: $JULIA_CMD --project=. -e 'using Pkg; Pkg.instantiate(); println("instantiate OK")' +# Restore the R package set from renv.lock (byte-exact; the R lane). Requires +# system R >= 4.5 (documented exception — R is not in the mise registry). Fails +# loudly if R is absent rather than silently skipping the lane. +renv-restore: + #!/usr/bin/env bash + if ! command -v Rscript >/dev/null 2>&1; then + echo "R LANE UNAVAILABLE: system R (>= 4.5) not found." >&2 + echo "Install R for your OS, then re-run: just renv-restore" >&2 + exit 1 + fi + Rscript --no-init-file -e 'if (!requireNamespace("renv", quietly=TRUE)) { message("installing renv..."); install.packages("renv", repos="https://cloud.r-project.org") }; renv::restore(prompt=FALSE)' + # ----------------------------------------------------------------------- # # Hygiene gates (scripts/check-*.sh — the canonical bash lanes) # ----------------------------------------------------------------------- # @@ -389,3 +452,95 @@ clean: # Remove generated outputs AND installed dependencies. clean-all: clean rm -rf frontend/node_modules + +# ----------------------------------------------------------------------- # +# Integration & environment healing (fork↔upstream) +# +# The fork and upstream share no git ancestor, so a naive merge conflicts on +# every shared path. These recipes expose config/integration.toml as a set of +# trust decisions the maintainer can make incrementally — from "behave exactly +# like upstream" (base) to "everything verified" (full) — without ever +# compromising a running system: the default profile changes no behaviour. +# Engine: scripts/integrate.sh. Guide: docs/integration/README.md. +# ----------------------------------------------------------------------- # + +# Repair the local environment to a known-good state: re-sync repo pins, re-wire +# hooks + merge drivers, regenerate the machine tool map, reinstall frontend +# deps, and (where present) re-instantiate Julia and restore the R lockfile. +# Resilient by design — each lane is attempted and a failure is reported, not +# fatal. Idempotent. The "fix my box" one-shot. +heal: + #!/usr/bin/env bash + set -uo pipefail + echo "heal: re-syncing repo pins..."; just sync-pins || echo "heal: sync-pins skipped" + echo "heal: re-wiring hooks + merge drivers..."; just hooks merge-drivers + echo "heal: regenerating machine tool map..."; just codegen-tools || echo "heal: codegen-tools skipped" + echo "heal: reinstalling frontend deps..."; just install || echo "heal: install skipped" + if timeout 15 $JULIA_CMD --version >/dev/null 2>&1; then + echo "heal: re-instantiating Julia..."; just julia-instantiate || echo "heal: julia-instantiate skipped" + else + echo "heal: Julia absent — provision the pinned toolchain with: just setup-tools" + fi + if command -v Rscript >/dev/null 2>&1; then + echo "heal: restoring R lockfile..."; just renv-restore || echo "heal: renv-restore skipped" + else + echo "heal: R absent — R lane left untouched (documented exception)" + fi + echo "heal: done. Verify with: just doctor" + +# Integration profiles & component toggles (thin wrappers over scripts/integrate.sh). +integrate: integrate-status + +integrate-status: + @{{INTEGRATE}} status + +integrate-profiles: + @{{INTEGRATE}} profiles + +# Switch the active profile: just integrate-profile . +integrate-profile profile="base": + @{{INTEGRATE}} profile "{{profile}}" + +# Recommended staging order (safest → riskiest). +integrate-plan: + @{{INTEGRATE}} plan + +# Classify in-progress merge conflicts (auto / component / human). +integrate-triage: + @{{INTEGRATE}} triage + +# Gates for the active selection; add strict="--strict" to require the tools be present. +integrate-verify strict="": + @{{INTEGRATE}} verify {{strict}} + +# Suspend a component: it stops being active (if runtime-gated, it will refuse). +suspend component: + @{{INTEGRATE}} disable "{{component}}" + +# Augment a component: it becomes active for this checkout. +augment component: + @{{INTEGRATE}} enable "{{component}}" + +# ----------------------------------------------------------------------- # +# Re-anchoring — collapse the one-shot merge into granular per-commit work +# +# The fork and upstream share the root commit but diverged early and developed +# in parallel, so a single merge shows ~159 conflicts at once. `reanchor` replays +# the fork's commits one-by-one onto upstream (git auto-applies the clean ones), +# turning that wall into a handful of small decisions. Measured: the whole fork +# re-anchors with 3 decisions and 0 residual conflicts, producing ~206 granular +# commits the maintainer can review/merge incrementally. Runs in an isolated +# worktree — it never touches your current branch. Engine: scripts/reanchor.sh. +# ----------------------------------------------------------------------- # + +# Read-only plan: classify each fork commit (auto-apply / overlap / delete-risk). +reanchor-plan: + @{{REANCHOR}} plan + +# Perform the re-anchor; leave a reviewable branch `reanchor/onto-upstream`. +reanchor: + @{{REANCHOR}} run --branch reanchor/onto-upstream --keep + +# Re-anchor but STOP at every conflict for hands-on resolution. +reanchor-manual: + @{{REANCHOR}} run --policy manual --keep diff --git a/config/integration.toml b/config/integration.toml new file mode 100644 index 0000000..b613a34 --- /dev/null +++ b/config/integration.toml @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Hyperpolymath engineering series +# +# config/integration.toml — fork↔upstream integration registry & transitional +# profiles. Consumed by scripts/integrate.sh (see `just integrate`). +# +# WHY THIS EXISTS +# --------------- +# This repository (hyperpolymath fork) and its upstream +# (JoshuaJewell/MetaManifold-WebUI, last touched 2026-07-21) share NO common +# git ancestor. A naive `git merge` therefore cannot three-way-merge anything +# and surfaces every shared-but-different path as a conflict — measured at 159 +# files on 2026-09-25 (see docs/integration/conflict-map-2026-09-25.md). The +# point of this file is to turn that wall of conflicts into a small, ordered set +# of trust decisions the maintainer can make incrementally. +# +# The two ideas, kept deliberately separable: +# +# 1. PROFILES — a named bundle of components, from "base" (behave exactly +# like upstream; nothing risk-bearing from the fork is active) +# through "transitional" (safe infrastructure only) to "full". +# The maintainer switches profile to choose how much of the +# fork is live. Default is `base`: it changes no behaviour. +# +# 2. COMPONENTS — the individual contributions, each tagged with the area it +# touches, a risk level, the path globs that identify its +# files (used to classify merge conflicts automatically), the +# upstream PRs that carry it, and — where the code supports it +# — the runtime gate that keeps it safe while transitional. +# +# The registry is COMMITTED and shared. The ACTIVE selection is machine-local +# (env METAMANIFOLD_INTEGRATION_PROFILE, or the git-excluded file +# config/integration.active) so the maintainer's choices never become a merge +# conflict of their own. +# +# FORMAT NOTE (for scripts/integrate.sh's awk parser): `paths` is a single +# quoted, space-separated list of path prefixes. Keep it that way. + +# The profile a fresh checkout behaves as. `base` == pure upstream behaviour. +default_profile = "base" + +# --------------------------------------------------------------------------- # +# Profiles — bundles of components, ordered base → transitional → full. +# --------------------------------------------------------------------------- # + +[profiles.base] +description = "Pure upstream behaviour. No risk-bearing fork component is active." +components = "" + +[profiles.transitional] +description = "Safe, non-behavioural infrastructure only (retries, an edge-case fix, benchmarks/CI, tooling, exact offsets). Statistical models and the UI migration stay out until trusted." +components = "estate-tooling install-retry matrix-1x1-fix benchmarks-ci exact-offsets" + +[profiles.full] +description = "Everything the fork has verified, including the real-statistics engine and the UI migration." +components = "estate-tooling install-retry matrix-1x1-fix benchmarks-ci exact-offsets real-statistics ui-migration" + +# --------------------------------------------------------------------------- # +# Components — the individual contributions. +# id : stable key (used by `just suspend/augment`, triage classification) +# area : tooling | build | src | frontend | test | data | docs +# risk : none | low | medium | high (ordering key for the staging plan) +# paths : space-separated path prefixes that identify the component's files +# prs : upstream pull-request numbers that carry it (informational) +# gate : "none" | a short description of the runtime safety gate +# --------------------------------------------------------------------------- # + +[component.estate-tooling] +name = "Estate tooling & merge hygiene (Justfile, mise.toml, .gitattributes, hooks, check scripts)" +area = "tooling" +risk = "none" +paths = "Justfile mise.toml .gitattributes .githooks .editorconfig .envrc .gitmessage scripts/check- scripts/gen-tools-yml.sh config/ci config/tools.yml" +prs = "" +gate = "none" +notes = "Pure additions — upstream has none of these files, so they land with zero conflict. This is the safest possible first step." + +[component.install-retry] +name = "Pinned-download retries (network-glitch resilience in setup/CI)" +area = "build" +risk = "low" +paths = "scripts/ci/fetch_pinned.sh install.sh install.jl .github/workflows/ci.yml" +prs = "11" +gate = "none" +notes = "Adds retry/TLS handling around byte-exact downloads. Does not change what is installed, only that transient failures no longer redden a build." + +[component.matrix-1x1-fix] +name = "1x1 matrix edge-case fix" +area = "src" +risk = "low" +paths = "src/analysis/analysis.jl src/analysis/diversity.jl test/" +prs = "8" +gate = "covered by unit test" +notes = "Prevents single-element matrices from crashing array operations. Small, surgical, test-covered." + +[component.benchmarks-ci] +name = "Baseline benchmarks + CI setup" +area = "build" +risk = "low" +paths = "frontend/bench bench/ .github/workflows/ci.yml" +prs = "10" +gate = "checksum-verified, timing informational" +notes = "Records runtimes and adds the CI matrix. Benchmark deltas are reported, never gated, so this cannot block on noise." + +[component.exact-offsets] +name = "Exact normalisation offsets (TSS/CSS/TMM/RLE) + lowercase method matching" +area = "src" +risk = "medium" +paths = "src/analysis/scaling.jl src/analysis/numeric_policy.jl config/defaults config/schemas" +prs = "" +gate = "numeric-contract tests (docs/statistics/numeric-contracts.md)" +notes = "Real offset maths and the tss/TSS case fix. Behaviour-visible but fully test-covered by the numeric-contract battery." + +[component.real-statistics] +name = "Real statistical models + refusals (no invented numbers)" +area = "src" +risk = "high" +paths = "src/analysis/estimation.jl src/analysis/exact_summaries.jl src/analysis/analysis.jl src/analysis/Execution.jl src/analysis/AnalysisConfig.jl test/ docs/statistics" +prs = "" +gate = "SUPPORTED_DISPERSION vs REFUSED_DISPERSION (a method either runs a real model or refuses)" +notes = "Replaces placeholder p-values (previously derived from hash(taxon_id)) with real MASS::glm.nb / linear models, and makes every unsupported method refuse loudly rather than guess. The refusal set IS the transitional dial: a method can stay refused until it is trusted, with no risk of a false result." + +[component.ui-migration] +name = "UI modernisation / reactive-component migration" +area = "frontend" +risk = "high" +paths = "frontend/ ui/ web/" +prs = "7" +gate = "bun run check (typecheck + tests + bench) must be green" +notes = "The largest surface. Modernises the reactive UI and brings frontend dependencies into lockfile compliance. Validate with `bun run check` before trusting." diff --git a/docs/integration/HANDOFF-granular-reanchor.md b/docs/integration/HANDOFF-granular-reanchor.md new file mode 100644 index 0000000..f81b3bb --- /dev/null +++ b/docs/integration/HANDOFF-granular-reanchor.md @@ -0,0 +1,109 @@ + +# HANDOFF — land the granular re‑anchor (fork → upstream) + +> **This is a brief for the next agent/session.** Everything it needs is already +> in the repo and measured. The goal: turn the fork↔upstream divergence into a +> **granular, reviewable, mergeable history** and land it, without ever +> compromising a running system. Read [`README.md`](./README.md) and +> [`conflict-map-2026-09-25.md`](./conflict-map-2026-09-25.md) first. + +## Objective + +Produce a branch that is **upstream's history with the fork's ~206 commits +cleanly replayed on top** (`reanchor/onto-upstream`), review the handful of +decisions it makes, verify the gates are green on it, and land it into +`JoshuaJewell/MetaManifold-WebUI` — either whole, or tier‑by‑tier — with a PR +that links the integration guide. + +## What already exists (do not rebuild) + +- `scripts/reanchor.sh` + `just reanchor` / `reanchor-plan` / `reanchor-manual` + — the re‑anchor engine. **Proven:** the whole fork re‑anchors with **3 + decisions, 0 residual conflicts, ~206 commits**. +- `scripts/integrate.sh` + `just integrate …` + `config/integration.toml` — + profiles (`base`/`transitional`/`full`), per‑component toggles, conflict triage. +- `.gitattributes` merge hygiene + `just merge-drivers`; `just heal`/`doctor`/ + `bootstrap`/`setup-full`/`renv-restore` for the turnkey environment. +- Measured facts in `conflict-map-2026-09-25.md`. + +## Ground truth (re‑verify, don't assume) + +```bash +git remote -v # upstream = JoshuaJewell/…, origin = hyperpolymath/… +git fetch --unshallow upstream origin # full history (the Arena clone is depth‑1!) +git merge-base origin/main upstream/main # → 7884553 "Initial commit" +``` + +The histories share the root commit and diverged at the 2nd commit (fork 221 +commits, upstream 89). A one‑shot merge = 159 conflicts; a re‑anchor = 3 decisions. + +## Steps + +1. **Plan.** `just reanchor-plan`. Record the auto/overlap/delete‑risk counts + (expect ~55 auto‑apply, ~152 overlap, delete‑risk on `pipelinesteps.txt`). +2. **Re‑anchor.** `just reanchor` (== `scripts/reanchor.sh run --branch + reanchor/onto-upstream --keep`). It runs in an isolated worktree; your current + branch is untouched. Confirm the summary: `206 commits / 0 residual / 3 + decisions`. +3. **Review the 3 decisions.** They should all be `DELETE (upstream): + pipelinesteps.txt` at commits `06d85ba`, `252299e`, `fed106b` — upstream + deleted the file; the fork's early commits touched it. Confirm accepting the + deletion is correct (it is: the file is gone upstream and superseded). +4. **Drop stray build output.** The re‑anchored tree keeps upstream's committed + `web/dist/*` (7 files) that the fork had removed. On the branch: + `git -C rm -r web/dist` and amend/commit (`chore: drop committed + web/dist build output`). These are build artefacts and must not be tracked. +5. **Confirm no upstream fix was silently reverted.** The default policy is + fork‑wins on content, so the re‑anchored tree ≈ the fork tree (+ `web/dist`). + Diff it against a *manual* three‑way merge to prove the only differences are + the documented items: + ```bash + git -C diff origin/main HEAD # expect only web/dist (+ any you fixed) + ``` + Spot‑check the high‑risk components named in `config/integration.toml` + (`real-statistics`, `ui-migration`) against upstream to be sure nothing + upstream‑authored was clobbered. If it was, resolve it by hand on the branch. +6. **Verify gates on the re‑anchored branch.** `just doctor`, then `just ci` + (frontend: typecheck + tests + bench) and, where the environment allows, + `just julia-test` and `just renv-restore`. The tree ≈ the fork's, so gates + should behave exactly as on `origin/main`. Record the verdicts. +7. **Land it.** Pick one, with Joshua (upstream owner): + - **Whole:** open a PR `hyperpolymath:reanchor/onto-upstream → + JoshuaJewell:main`; it is now a normal, granular, mergeable branch. + - **Tiered (recommended if trust is being built):** keep the branch, and + cherry‑pick / merge components in the `just integrate plan` order + (`estate-tooling → install-retry → matrix-1x1-fix → benchmarks-ci → + exact-offsets → real-statistics → ui-migration`), gating each with CI before + the next. Use `just integrate profile transitional|full` to mirror the + adoption on a running checkout. + Link [`docs/integration/README.md`](./README.md) in the PR body. + +## Guardrails (non‑negotiable) + +- **Never compromise operations.** The re‑anchored branch must pass its gates and + must not silently drop an upstream fix. The 3 decisions + step 5 cover the known + cases; anything else you find, resolve by hand and log it. +- **Branch discipline.** This session is pinned to `arena/01a0daa1-metamanifold-webui`. + Produce the `reanchor/onto-upstream` branch *via the tool at runtime* (a user + action), not by committing 206 commits onto the session branch. Do not switch + the session branch. +- **Keep the aid merge‑clean.** Any further changes to the integration tooling + stay on fork‑only / new paths (they add zero conflict surface for upstream). +- **Conventional commits** (`feat|fix|docs|…: …`, ≤72‑char subject) and run + `scripts/check-spdx.sh` + `scripts/check-format.sh` before pushing. + +## Decisions to confirm with Joshua (from the owner review, Options A–D) + +Full merge · selective/tiered cherry‑pick · dual‑track fork · handover notice. +Also: keep `codecov.yml`? (fork removed Codecov) — recommend **drop**. + +## Done when + +- [ ] `reanchor/onto-upstream` exists, `git rev-list --count upstream/main..HEAD` + ≈ 206, residual conflicts 0. +- [ ] `web/dist/*` removed on the branch. +- [ ] Diff vs `origin/main` is only the documented items (no surprise reversions). +- [ ] Gates green on the branch (recorded). +- [ ] PR opened against upstream, guide linked, adoption path agreed with Joshua. diff --git a/docs/integration/README.md b/docs/integration/README.md new file mode 100644 index 0000000..7de6376 --- /dev/null +++ b/docs/integration/README.md @@ -0,0 +1,219 @@ + +# Integrating the fork into upstream — a low-friction path + +**Audience:** Joshua (upstream maintainer) and the `hyperpolymath` engineering +team. **Goal:** get the fork's work into +`JoshuaJewell/MetaManifold-WebUI` without a single "hundreds of files, load of +conflicts" review, and without either side ever being unable to run. + +The measured problem and the exact file counts are in +[`conflict-map-2026-09-25.md`](./conflict-map-2026-09-25.md). In one sentence: +**the fork and upstream share only the root commit, diverged at the second +commit, and then developed in parallel for months — so the merge base is nearly +empty and a normal merge surfaces all 159 shared‑but‑different files at once.** +Nothing in this guide changes upstream's operational behaviour unless you choose +it to; the default is "behave exactly like upstream". + +## The idea, in layers + +0. **Re‑anchor** — replay the fork's commits one‑by‑one onto upstream so git + auto‑applies the clean ones and only the genuine overlaps surface, per commit. + Measured: the whole fork re‑anchors with **3 decisions** and **0 residual + conflicts**, producing ~206 granular, reviewable commits. (`just reanchor`.) +1. **Merge hygiene** — stop git from ever producing a line‑conflict inside a + lockfile or a build artefact. (`.gitattributes` + `just merge-drivers`.) +2. **Profiles & components** — turn "review 159 files" into "make ~7 ordered + trust decisions", switchable from *pure upstream* to *partially transitional* + to *everything*. (`config/integration.toml` + `scripts/integrate.sh` + the + `just integrate …` recipes.) +3. **A staging plan** — land the safe infrastructure first, let CI go green, then + take the risk‑bearing pieces one tier at a time. (`just integrate-plan`.) + +Everything below is additive and default‑off. A fresh clone behaves as +`base` — i.e. exactly like upstream — until someone runs a command to change it. + +--- + +## 0. Re‑anchor — from one 159‑file wall to ~206 granular commits + +This is the deepest fix and the reason the fork *looks* unmergeable. Instead of +one giant three‑way merge, re‑anchor replays the fork's history onto upstream: + +```bash +just reanchor-plan # read-only: classify every fork commit (auto / overlap / delete-risk) +just reanchor # do it; leaves branch reanchor/onto-upstream for review +just reanchor-manual # same, but stop at every conflict for hands-on resolution +``` + +It runs in an isolated worktree and never touches your current branch. The result +is upstream's history with the fork's ~206 commits cleanly on top — reviewable and +mergeable incrementally, and sharing a real base so future upstream work merges +cleanly. See `scripts/reanchor.sh` and the measured numbers in +[`conflict-map-2026-09-25.md`](./conflict-map-2026-09-25.md). + +--- + +## 1. Merge hygiene — never hand-merge a generated file + +The committed `.gitattributes` now marks lockfiles and build output so git will +not line-merge them (`merge: unset`, `linguist-generated`). Concretely, these can +never again appear as a wall of `<<<<<<<` markers: + +`Manifest.toml`, `renv.lock`, `frontend/bun.lock`, `bun.lockb`, +`package-lock.json`, `pnpm-lock.yaml`, `renv/activate.R`, `renv/library/**`, +`config/tools.yml`, `web/dist/**`, `frontend/dist/**`, coverage, `*.cov`, +`lcov.info`, test-results, `*junit.xml`, `bench/**/results/*.json`. + +These are resolved by **regenerating**, never by editing: + +```bash +just heal # re-syncs pins, re-instantiates Julia, restores renv, reinstalls frontend +``` + +If you want lockfiles to *auto*-resolve to the branch you are merging into (and +then be regenerated), wire the stronger opt-in driver once: + +```bash +just merge-drivers # writes merge.lockfile into local .git/info/attributes (never committed) +``` + +This is fully opt-in and local: it cannot break a merge on a clone that has not +run it, and it never mutates a tracked file. + +## 2. Profiles & components — the transitional dial + +`config/integration.toml` is the shared registry. It defines the fork's +contributions as named **components**, each with an area, a risk level, the path +globs that identify its files (used to auto-classify conflicts), the upstream PRs +that carry it, and — where the code supports it — the runtime gate that keeps it +safe while transitional. It defines three **profiles** that bundle components: + +| profile | what is live | use it when | +|---|---|---| +| `base` | nothing risk-bearing from the fork | you want to behave exactly like upstream (the default) | +| `transitional` | safe infra only: estate tooling, download retries, the 1×1 fix, benchmarks/CI, exact offsets | you want the reliability wins but not the statistics/UI changes yet | +| `full` | everything, including the real-statistics engine and the UI migration | you have reviewed and trust the higher-risk pieces | + +Inspect and switch: + +```bash +just integrate status # active profile + each component on/off +just integrate profiles # what each profile bundles +just integrate profile transitional # switch (machine-local, never committed) +just augment real-statistics # turn one component on for this checkout +just suspend ui-migration # turn one component off +``` + +The active choice is **machine-local** (`METAMANIFOLD_INTEGRATION_PROFILE`, or +the git-excluded `config/integrate.active`). Your choices never become a merge +conflict of their own. + +### Why this never compromises operations + +- The default profile (`base`) activates nothing; behaviour is unchanged. +- Every component is either a **pure addition** (tooling, CI, retries) or is + **gated by the code's existing refusal architecture**. The statistics engine + already refuses loudly rather than guess: a method is either wired to a real + model (`SUPPORTED_DISPERSION`) or it refuses (`REFUSED_DISPERSION`) with an + explicit, actionable error. That refusal *is* the transitional state — a method + can stay refused until it is trusted, so there is no path to a silently-wrong + result. See `docs/statistics/` and `docs/owner-review-2026-09-25.md`. +- `just heal` and `just doctor` only repair or report; they change no scientific + behaviour. + +## 3. The staging plan + +```bash +just integrate plan +``` + +prints the recommended order (safest → riskiest), cross-referenced with the +upstream PR numbers: + +1. `estate-tooling` (none) — Justfile, mise, `.gitattributes`, hooks, check + scripts. Pure additions; upstream has none of these, so **zero conflict**. +2. `install-retry` (low, PR #11) — retry/TLS around byte-exact downloads. +3. `matrix-1x1-fix` (low, PR #8) — the 1×1 matrix edge case. +4. `benchmarks-ci` (low, PR #10) — baselines + CI matrix. +5. `exact-offsets` (medium) — TSS/CSS/TMM/RLE offsets + lowercase method match. +6. `real-statistics` (high) — real models + refusals (no invented numbers). +7. `ui-migration` (high, PR #7) — the reactive-UI modernisation. + +Land one tier, let CI go green, then the next. This is upstream's "Option B" +(selective cherry-picking) from the owner review, made mechanical. + +--- + +## Workflow for the maintainer (Joshua) + +```bash +# once, in your upstream clone +git remote add fork https://github.com/hyperpolymath/MetaManifold-WebUI.git +git fetch fork + +# see exactly what you are taking on, by category +just integrate-triage # during a merge/rebase; or point it at a list: +just integrate status +just integrate plan + +# integrate the safe infrastructure first (tiers 1–4), regenerate locks, gate: +git checkout -b integrate-infra fork/main +# … resolve only the ~handful of non-generated conflicts the triage flagged … +just heal && just ci + +# then opt into the risk-bearing pieces a tier at a time, trusting as you go: +just integrate profile transitional # try the safe set live +just integrate profile full # once you trust the statistics + UI +``` + +During any merge/rebase, `just integrate-triage` classifies every conflicted path +as **AUTO** (regenerate — lockfiles/build output), **COMPONENT** (decide per the +registry: take the side for components you have chosen to trust), or **HUMAN** +(unclassified — review). That collapses "hundreds of files" into "a handful of +human decisions plus a `just heal`". + +## Workflow for the fork team — stay turnkey *and* merge-friendly + +**Turnkey (your side always runs with zero extra effort):** + +```bash +just bootstrap # mise toolchain (julia/bun/node/just) + frontend deps + hooks + merge drivers + tool map +just setup-full # …plus Julia instantiate, renv restore, and sha256-pinned external tools +just doctor # health report (fails loudly if an essential tool is missing) +just heal # repair anything that drifted +``` + +`mise.toml` is the single source of truth for the toolchain (every pin verified +against CI); `just setup-tools` provisions it; `just renv-restore` restores the +byte-exact R lockfile; `install.sh` installs the sha256-pinned external pipeline +tools. R is the one documented exception (not in the mise registry) and is called +out by `just doctor` rather than silently skipped. + +**Merge-friendly (keep the conflict surface small):** + +- Put new work in **new files/directories** wherever possible — a clean addition + never conflicts. +- Keep edits to shared root files (`start.sh`, `install.sh`, `install.jl`, + `Project.toml`, `.github/`, `README.md`) minimal and in append-safe regions. +- Never hand-edit lockfiles/build output; run `just heal`/`just build` and commit + the regenerated artefact. +- Register every contribution in `config/integration.toml` so triage can classify + it automatically. + +## Quick reference + +| command | what it does | +|---|---| +| `just doctor` | environment health (hard-fails on missing essentials) | +| `just heal` | repair the environment to a known-good state | +| `just bootstrap` / `just setup-full` | clone-to-runnable on a bare machine | +| `just merge-drivers` | opt-in lockfile auto-resolution (local, safe) | +| `just reanchor-plan` / `just reanchor` | re‑anchor the fork onto upstream as granular commits | +| `just integrate status` / `profiles` / `profile

` | the transitional dial | +| `just integrate plan` / `triage` / `verify` | staging order / conflict classes / gates | +| `just augment ` / `just suspend ` | flip one component | + +Engine: [`scripts/integrate.sh`](../../scripts/integrate.sh). Registry: +[`config/integration.toml`](../../config/integration.toml). diff --git a/docs/integration/conflict-map-2026-09-25.md b/docs/integration/conflict-map-2026-09-25.md new file mode 100644 index 0000000..9672514 --- /dev/null +++ b/docs/integration/conflict-map-2026-09-25.md @@ -0,0 +1,92 @@ + +# Fork↔upstream conflict map — measured 2026-09-25 + +This is the measured state behind "a naive merge of +`hyperpolymath/MetaManifold-WebUI` (this fork) into +`JoshuaJewell/MetaManifold-WebUI` (upstream) is hundreds of files with +conflicts", and how the re‑anchor collapses it. Numbers are exact for the trees +compared and reproducible with `scripts/reanchor.sh plan` and the commands below. + +## The histories: shared root, early divergence, long parallel development + +``` +git merge-base origin/main upstream/main +→ 7884553 "Initial commit" (2026-01-30) +``` + +They **do** share the root commit, but they split at the *second* commit and then +developed in parallel for months: + +- fork (`origin/main`): **221** commits since the base (last: `17d8c0d`) +- upstream (`upstream/main`): **89** commits since the base (last push 2026-07-21) + +Because both sides changed almost everything from a near‑empty base, the +three‑way merge base (the "Initial commit") is nearly useless — almost every file +differs on both sides — so a single `git merge` conflicts on all of them at once. + +> Note: an earlier revision of this document claimed the histories were +> *unrelated* (empty merge base). That was an artifact of a **depth‑1 shallow +> clone** hiding the shared root. With full history the root is `7884553`. The +> conflict *count* below is unchanged; only the mechanism (and therefore the fix) +> is different: this is ordinary divergence, so it is fixable by **rebase**, not +> by unrelated‑history surgery. + +## The numbers (one‑shot merge) + +| | count | +|---|---:| +| Files in fork | 357 | +| Files in upstream | 195 | +| Shared paths (exist in both) | 186 | +| **Conflicting shared paths (same path, different content)** | **159** | +| Fork-only paths (clean additions) | 171 | +| Upstream-only paths (carried in by a merge) | 9 | +| Files upstream changed since the base | 196 | + +## The 159 conflicts, by area + +| area | files | +|---|---:| +| `frontend/` (TS/React) | 59 | +| `src/` (Julia) | 40 | +| `test/` | 30 | +| `config/` · `bench/` · root files | 32 | + +Only **3** are lockfiles (`Manifest.toml`, `frontend/bun.lock`, `renv/activate.R`) +— handled automatically by merge hygiene (see `README.md`). `renv.lock` is +byte‑identical in both. The rest is genuine source divergence that needs a +*decision*. + +## The fix: re‑anchor into a granular history (measured) + +`git rebase --onto upstream/main 7884553 origin/main` replays the fork's commits +one at a time onto upstream. Git auto‑applies every commit that doesn't collide +and surfaces only the genuine overlaps, per commit. `scripts/reanchor.sh run` +automates it (fork‑wins on content; respect upstream's deletions). **Measured +result:** + +``` +granular commits on top of upstream : 206 +residual conflicts : 0 +decisions the policy had to make : 3 + DELETE (upstream): pipelinesteps.txt [06d85ba Create run_cutadapt.jl.] + DELETE (upstream): pipelinesteps.txt [252299e Merge tables logic.] + DELETE (upstream): pipelinesteps.txt [fed106b Added dada2 R module.] +files the re-anchored tree keeps that the fork dropped (review) : 7 (all web/dist/* build output) +``` + +So the "hundreds of files, load of conflicts" becomes: **auto‑apply 55 clean +commits, make 3 trivial decisions (all the same deleted file), review 7 stray +committed build artefacts, and land ~206 reviewable commits.** The resulting +branch also shares a real merge base with upstream, so *future* upstream changes +merge cleanly too. + +Reproduce: + +```bash +git fetch upstream origin +scripts/reanchor.sh plan # per-commit classification +scripts/reanchor.sh run # or: just reanchor (leaves branch reanchor/onto-upstream) +``` diff --git a/scripts/integrate.sh b/scripts/integrate.sh new file mode 100755 index 0000000..75e85b1 --- /dev/null +++ b/scripts/integrate.sh @@ -0,0 +1,359 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Hyperpolymath engineering series +# +# integrate.sh — fork↔upstream integration helper. +# +# The fork (this repo) and upstream (JoshuaJewell/MetaManifold-WebUI) share no +# git ancestor, so a naive merge conflicts on every shared-but-different path. +# This script reads config/integration.toml and turns that wall into a small, +# ordered set of trust decisions: which components are live (profiles), how any +# in-progress conflicts classify (triage), and what order to stage them (plan). +# +# Doctrine (matches the estate): every command either works or FAILS LOUDLY with +# an actionable message; nothing here silently passes. No hidden dependencies — +# bash + coreutils + git + awk only. +# +# Usage: scripts/integrate.sh [args] +# status Show active profile and each component's on/off state +# profiles List profiles with their component bundles +# profile [NAME] Print (no arg) or switch (NAME) the active profile +# plan Recommended staging order (safest → riskiest) +# triage [--from-file F] Classify current merge conflicts (auto/component/human) +# enable Augment: turn a component on for this checkout +# disable Suspend: turn a component off for this checkout +# verify [--strict] List (and where possible run) the gates for active components +# +# The active selection is machine-local and never committed: +# env METAMANIFOLD_INTEGRATION_PROFILE, else the git-excluded +# config/integration.active. Default (nothing set) is config/integration.toml's +# default_profile — which is `base`, i.e. behave exactly like upstream. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CONFIG="$ROOT/config/integration.toml" +ACTIVE="$ROOT/config/integration.active" + +die() { printf 'integrate: %s\n' "$*" >&2; exit 1; } + +[ -f "$CONFIG" ] || die "registry not found: $CONFIG" + +# --------------------------------------------------------------------------- # +# TOML-subset parsers (awk). The registry is authored to a fixed shape: +# key = "value" scalar +# [profiles.NAME] profile section +# [component.ID] component section +# --------------------------------------------------------------------------- # + +default_profile() { + awk -F= '/^[[:space:]]*default_profile[[:space:]]*=/ { v=$2; gsub(/[[:space:]"]/,"",v); print v; exit }' "$CONFIG" +} + +list_profiles() { + awk '/^\[profiles\./{ s=$0; sub(/^\[profiles\./,"",s); sub(/\].*$/,"",s); print s }' "$CONFIG" +} + +list_components() { + awk '/^\[component\./{ s=$0; sub(/^\[component\./,"",s); sub(/\].*$/,"",s); print s }' "$CONFIG" +} + +# field_value (section-kind: profiles|component) +field_value() { + local kind="$1" id="$2" field="$3" + awk -v kind="$kind" -v id="$id" -v field="$field" ' + function trim(s){ gsub(/^[[:space:]]+|[[:space:]]+$/,"",s); return s } + /^\[/ { + sec=$0 + insec = (sec == "[" kind "." id "]") + next + } + insec { + k=trim($0); sub(/[[:space:]]*=.*$/,"",k) + if (k==field) { + v=$0; sub(/^[^=]*=/,"",v) + gsub(/^[[:space:]"]+|[[:space:]"]+$/,"",v) + print v; exit + } + } + ' "$CONFIG" +} + +profile_components() { field_value profiles "$1" components; } +component_field() { field_value component "$1" "$2"; } + +component_paths() { component_field "$1" paths; } + +# --------------------------------------------------------------------------- # +# Active-selection helpers (machine-local; never committed) +# --------------------------------------------------------------------------- # + +active_profile() { + if [ -n "${METAMANIFOLD_INTEGRATION_PROFILE:-}" ]; then + printf '%s\n' "$METAMANIFOLD_INTEGRATION_PROFILE" + elif [ -f "$ACTIVE" ]; then + local p; p="$(awk -F= '/^profile=/{print $2}' "$ACTIVE" | tr -d '[:space:]')" + printf '%s\n' "${p:-$(default_profile)}" + else + default_profile + fi +} + +override_list() { # override_list + [ -f "$ACTIVE" ] || { printf ''; return; } + awk -F= -v key="$1" '$1==key { v=$2; gsub(/^[[:space:]]+|[[:space:]]+$/,"",v); print v; exit }' "$ACTIVE" +} + +word_in() { case " $2 " in *" $1 "*) return 0;; *) return 1;; esac; } + +# The resolved, ordered set of ON component ids for the active selection. +active_components() { + local prof base en dis out="" + prof="$(active_profile)" + base="$(profile_components "$prof")" + en="$(override_list enable)" + dis="$(override_list disable)" + local id + while IFS= read -r id; do + [ -n "$id" ] || continue + if word_in "$id" "$en"; then + out="$out $id" + elif word_in "$id" "$base" && ! word_in "$id" "$dis"; then + out="$out $id" + fi + done < <(list_components) + printf '%s\n' "$out" | xargs 2>/dev/null || printf '%s' "$out" +} + +ensure_git_excluded() { + local excl="$ROOT/.git/info/exclude" + mkdir -p "$ROOT/.git/info" + touch "$excl" + grep -qxF 'config/integration.active' "$excl" 2>/dev/null || \ + printf '\n# active integration profile (machine-local; scripts/integrate.sh)\nconfig/integration.active\n' >> "$excl" +} + +valid_component() { word_in "$1" "$(list_components | tr '\n' ' ')"; } +valid_profile() { word_in "$1" "$(list_profiles | tr '\n' ' ')"; } + +# --------------------------------------------------------------------------- # +# Commands +# --------------------------------------------------------------------------- # + +cmd_status() { + local prof; prof="$(active_profile)" + printf 'Active profile : %s\n' "$prof" + printf 'Registry : %s\n' "${CONFIG#$ROOT/}" + [ -n "${METAMANIFOLD_INTEGRATION_PROFILE:-}" ] && printf 'Source : env METAMANIFOLD_INTEGRATION_PROFILE\n' + printf '\nComponents:\n' + local on; on=" $(active_components) " + local id name risk area state + while IFS= read -r id; do + [ -n "$id" ] || continue + name="$(component_field "$id" name)" + risk="$(component_field "$id" risk)" + area="$(component_field "$id" area)" + if word_in "$id" "$on"; then state="ON "; else state="off"; fi + printf ' [%s] %-16s %-6s %-9s %s\n' "$state" "$id" "$area" "$risk" "$name" + done < <(list_components) +} + +cmd_profiles() { + local p desc comps n + while IFS= read -r p; do + [ -n "$p" ] || continue + desc="$(field_value profiles "$p" description)" + comps="$(profile_components "$p")" + if [ -z "$(printf '%s' "$comps" | tr -d '[:space:]')" ]; then n=0; else n=$(printf '%s\n' $comps | wc -l | xargs); fi + printf '%-14s (%s component(s))\n %s\n' "$p" "$n" "$desc" + done < <(list_profiles) +} + +cmd_profile() { + local name="${1:-}" + if [ -z "$name" ]; then active_profile; return; fi + valid_profile "$name" || die "unknown profile '$name'. Try: $(list_profiles | tr '\n' ' ')" + printf 'profile=%s\n' "$name" > "$ACTIVE" + ensure_git_excluded + printf 'Active profile set to %s (written to %s, git-excluded).\n' "$name" "${ACTIVE#$ROOT/}" + printf 'Active components: %s\n' "$(active_components)" +} + +_override_add() { # _override_add + local key="$1" id="$2" + valid_component "$id" || die "unknown component '$id'. Try: $(list_components | tr '\n' ' ')" + [ -f "$ACTIVE" ] || printf 'profile=%s\n' "$(active_profile)" > "$ACTIVE" + ensure_git_excluded + local cur; cur="$(override_list "$key")" + if word_in "$id" "$cur"; then printf '%s already contains %s\n' "$key" "$id"; return; fi + cur="$(printf '%s %s' "$cur" "$id" | xargs)" + # rewrite the key line (or append it) + if grep -q "^${key}=" "$ACTIVE"; then + awk -v key="$key" -v val="$cur" 'BEGIN{FS=OFS="="} $1==key{sub(/=[^=]*$/,"="val)} {print}' "$ACTIVE" > "$ACTIVE.tmp" + else + cp "$ACTIVE" "$ACTIVE.tmp"; printf '%s=%s\n' "$key" "$cur" >> "$ACTIVE.tmp" + fi + mv "$ACTIVE.tmp" "$ACTIVE" + printf '%s %s -> active components: %s\n' "$key" "$id" "$(active_components)" +} + +cmd_enable() { [ $# -ge 1 ] || die "usage: integrate.sh enable "; _override_add enable "$1"; } +cmd_disable() { [ $# -ge 1 ] || die "usage: integrate.sh disable "; _override_add disable "$1"; } + +risk_rank() { case "$1" in none) echo 0;; low) echo 1;; medium) echo 2;; high) echo 3;; *) echo 9;; esac; } + +cmd_plan() { + printf 'Recommended staging order (safest first). Merge/gate one tier, let CI go green, then the next.\n\n' + local id name risk prs rank + # stable sort by risk rank + local ids; ids="$(list_components)" + local sorted; sorted="$(while IFS= read -r id; do + [ -n "$id" ] || continue + risk="$(component_field "$id" risk)" + printf '%s\t%s\n' "$(risk_rank "$risk")" "$id" + done <<< "$ids" | sort -s -k1,1n | cut -f2)" + local i=0 + while IFS= read -r id; do + [ -n "$id" ] || continue + i=$((i+1)) + name="$(component_field "$id" name)" + risk="$(component_field "$id" risk)" + prs="$(component_field "$id" prs)" + printf '%d. [%-6s] %-16s %s' "$i" "$risk" "$id" "$name" + [ -n "$prs" ] && printf ' (upstream PR #%s)' "$prs" + printf '\n' + done <<< "$sorted" + printf '\nTip: switch what is live with `just integrate profile `;\n' + printf ' flip a single component with `just augment ` / `just suspend `.\n' +} + +is_generated() { + case "$1" in + Manifest.toml|renv.lock|frontend/bun.lock|bun.lockb|package-lock.json|pnpm-lock.yaml) return 0;; + renv/activate.R|renv/library/*|R/_renv_dependencies.R) return 0;; + web/dist/*|frontend/dist/*|*/dist/*) return 0;; + *coverage/*|*.cov|lcov.info|*/test-results/*|*junit.xml|*results.json) return 0;; + *) return 1;; + esac +} + +match_component() { # echoes best (longest-prefix) component id for a path, or nothing + local path="$1" id p best="" bestlen=-1 plen + while IFS= read -r id; do + [ -n "$id" ] || continue + for p in $(component_paths "$id"); do + case "$path" in + "$p"*) plen=${#p}; if [ "$plen" -gt "$bestlen" ]; then bestlen=$plen; best="$id"; fi;; + esac + done + done < <(list_components) + [ -n "$best" ] && printf '%s\n' "$best" +} + +cmd_triage() { + local fromfile="" + while [ $# -gt 0 ]; do + case "$1" in + --from-file) [ -n "${2:-}" ] || die "--from-file requires a path"; fromfile="$2"; shift 2;; + *) die "triage: unknown option '$1'";; + esac + done + + local files + if [ -n "$fromfile" ]; then + # Accept any readable source (regular file, pipe, /dev/stdin, process + # substitution) — only fail when it genuinely cannot be read. + files="$(cat "$fromfile" 2>/dev/null)" || die "cannot read conflict list: $fromfile" + else + if ! git -C "$ROOT" rev-parse -q --verify MERGE_HEAD >/dev/null 2>&1 && ! git -C "$ROOT" rev-parse -q --verify REBASE_HEAD >/dev/null 2>&1; then + printf 'triage: no merge/rebase in progress (nothing to triage).\n' + printf ' Point it at a conflict list with: integrate.sh triage --from-file \n' + return 0 + fi + files="$(git -C "$ROOT" diff --name-only --diff-filter=U)" + fi + + [ -n "$(printf '%s' "$files" | tr -d '[:space:]')" ] || { printf 'triage: no conflicted paths.\n'; return 0; } + + local auto=0 comp=0 unknown=0 f c risk + printf '%-9s %-8s %-16s %s\n' "VERDICT" "RISK" "COMPONENT" "PATH" + printf -- '--------------------------------------------------------------------------\n' + while IFS= read -r f; do + [ -n "$f" ] || continue + if is_generated "$f"; then + auto=$((auto+1)) + printf '%-9s %-8s %-16s %s\n' "AUTO" "-" "(generated)" "$f" + elif c="$(match_component "$f")" && [ -n "$c" ]; then + comp=$((comp+1)) + risk="$(component_field "$c" risk)" + printf '%-9s %-8s %-16s %s\n' "COMPONENT" "$risk" "$c" "$f" + else + unknown=$((unknown+1)) + printf '%-9s %-8s %-16s %s\n' "HUMAN" "?" "(unclassified)" "$f" + fi + done <<< "$files" + printf -- '--------------------------------------------------------------------------\n' + printf 'AUTO (regenerate, never hand-merge): %d\n' "$auto" + printf 'COMPONENT (decide per registry) : %d\n' "$comp" + printf 'HUMAN (unclassified, review) : %d\n' "$unknown" + printf '\n' + printf 'AUTO files are lockfiles/build output — resolve by regenerating, not editing:\n' + printf ' just heal # re-syncs pins, re-instantiates Julia, restores renv, re-installs frontend\n' + printf 'COMPONENT files: take the side for the components you have chosen to trust\n' + printf ' (see `just integrate status`), or hold them until you have.\n' +} + +cmd_verify() { + local strict=0 a + for a in "$@"; do [ "$a" = "--strict" ] && strict=1 || die "verify: unknown option '$a'"; done + local on; on=" $(active_components) " + [ -z "$(printf '%s' "$on" | tr -d ' ')" ] && { printf 'verify: active profile has no components; nothing to verify.\n'; return 0; } + + # area -> gate recipe(s) + local -A want=() + local id area + while IFS= read -r id; do + [ -n "$id" ] || continue + word_in "$id" "$on" || continue + area="$(component_field "$id" area)" + case "$area" in + frontend) want["check"]=1;; + src|test) want["julia-test"]=1;; + build) want["ci"]=1;; + *) : ;; + esac + done < <(list_components) + + [ "${#want[@]}" -gt 0 ] || { printf 'verify: active components carry no runnable gate (tooling/docs/data only).\n'; return 0; } + + printf 'Gates for the active selection (in order):\n' + local g order="spdx format lint typecheck test julia-test ci check" + for g in $order; do [ -n "${want[$g]:-}" ] && printf ' - just %s\n' "$g"; done + printf '\nThis helper reports the plan; run each gate directly (e.g. `just ci`).\n' + if [ "$strict" -eq 1 ]; then + printf '\n--strict: verifying tool availability now.\n' + local miss=0 t + for t in bun julia just; do command -v "$t" >/dev/null 2>&1 || { printf 'FAIL %s absent\n' "$t"; miss=1; }; done + [ "$miss" -eq 0 ] && printf 'PASS all gate tools present\n' || { printf 'Install missing tools with: just bootstrap\n' >&2; exit 1; } + fi +} + +usage() { sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; } + +main() { + local cmd="${1:-}"; shift || true + case "$cmd" in + status) cmd_status "$@";; + profiles) cmd_profiles "$@";; + profile) cmd_profile "$@";; + plan) cmd_plan "$@";; + triage) cmd_triage "$@";; + enable) cmd_enable "$@";; + disable) cmd_disable "$@";; + verify) cmd_verify "$@";; + ""|help|-h|--help) usage;; + *) die "unknown command '$cmd' (try: status|profiles|profile|plan|triage|enable|disable|verify)";; + esac +} + +main "$@" diff --git a/scripts/reanchor.sh b/scripts/reanchor.sh new file mode 100755 index 0000000..4b17cd1 --- /dev/null +++ b/scripts/reanchor.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Hyperpolymath engineering series +# +# reanchor.sh — turn the fork↔upstream divergence into a GRANULAR history. +# +# WHY +# --- +# The fork and upstream share the root commit ("Initial commit", 7884553) but +# diverged at the second commit and then developed in parallel (fork: ~221 +# commits; upstream: ~89). Because BOTH sides changed almost everything from a +# near-empty base, a single `git merge` surfaces ~159 conflicting files at once +# (see docs/integration/conflict-map-2026-09-25.md). +# +# Re-anchoring fixes this: `git rebase --onto upstream/main ` +# replays the fork's commits ONE AT A TIME onto upstream. Git auto-applies every +# commit that does not collide, and surfaces only the genuine overlaps — as a +# sequence of tiny, per-commit decisions instead of one wall. Measured 2026-09-25 +# with the default policy: the whole fork re-anchors with THREE decisions (all +# "upstream deleted this file"), leaving 0 residual conflicts. +# +# The result is a granular branch — upstream's history, then the fork's ~206 +# commits on top — that (a) the maintainer can review/merge incrementally or all +# at once, and (b) shares a real merge base with upstream, so FUTURE upstream +# changes merge cleanly too. +# +# Doctrine: fail loudly; no hidden dependencies (bash + git + awk); never touch +# the caller's working tree or current branch (all work happens in a worktree). +# +# Usage: scripts/reanchor.sh [options] +# plan Read-only: classify each fork commit (auto / overlap / delete-risk) +# run [options] Perform the re-anchor in a worktree (see options) +# status Report an in-progress re-anchor +# +# run options: +# --policy theirs|manual theirs (default): auto-resolve content in the fork's +# favour and respect upstream's deletions — the whole +# fork lands with a handful of logged decisions. +# manual: stop at every conflict for hands-on review. +# --branch NAME Create/point branch NAME at the result (default: none, detached) +# --worktree PATH Where to build (default: $TMPDIR/metamanifold-reanchor) +# --keep Leave the worktree in place afterwards (default: remove on success) +# +# Env: UPSTREAM_REMOTE (default upstream), FORK_REMOTE (default origin), +# UPSTREAM_REF (default main), FORK_REF (default main). + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +UPSTREAM_REMOTE="${UPSTREAM_REMOTE:-upstream}" +FORK_REMOTE="${FORK_REMOTE:-origin}" +UPSTREAM_REF="${UPSTREAM_REF:-main}" +FORK_REF="${FORK_REF:-main}" +UPSTREAM="$UPSTREAM_REMOTE/$UPSTREAM_REF" +FORK="$FORK_REMOTE/$FORK_REF" + +die() { printf 'reanchor: %s\n' "$*" >&2; exit 1; } + +require_ref() { git rev-parse -q --verify "$1" >/dev/null 2>&1 || die "ref not found: $1 (fetch it first, e.g. 'git fetch $UPSTREAM_REMOTE')"; } + +find_base() { + local b; b="$(git merge-base "$UPSTREAM" "$FORK" 2>/dev/null || true)" + [ -n "$b" ] || die "no common ancestor between $UPSTREAM and $FORK — re-anchoring is not possible; use the profile/triage path instead (just integrate)" + printf '%s\n' "$b" +} + +preflight() { + require_ref "$UPSTREAM" + require_ref "$FORK" + # Only uncommitted TRACKED changes block; the rebase runs in an isolated + # worktree at the fork ref, so untracked files are irrelevant to it. + [ -z "$(git status --porcelain --untracked-files=no)" ] || die "uncommitted tracked changes present — commit or stash before re-anchoring" + local gd; gd="$(git rev-parse --git-dir)" + if [ -d "$gd/rebase-merge" ] || [ -d "$gd/rebase-apply" ]; then + die "a rebase is already in progress — finish or 'git rebase --abort' first" + fi +} + +# --------------------------------------------------------------------------- # +cmd_plan() { + require_ref "$UPSTREAM"; require_ref "$FORK" + local base; base="$(find_base)" + printf 'Fork : %s (%s)\n' "$FORK" "$(git rev-parse --short "$FORK")" + printf 'Upstream : %s (%s)\n' "$UPSTREAM" "$(git rev-parse --short "$UPSTREAM")" + printf 'Merge base : %s %s\n' "$(git rev-parse --short "$base")" "$(git log -1 --format='%s' "$base")" + printf 'Fork commits since base : %s\n' "$(git rev-list --count "$base..$FORK")" + printf 'Upstream commits since base : %s\n' "$(git rev-list --count "$base..$UPSTREAM")" + echo "" + + git diff --name-only "$base" "$UPSTREAM" | sort -u > /tmp/.reanchor_up.$$ 2>/dev/null || true + local upchanged=/tmp/.reanchor_up.$$ + printf 'Upstream changed %s file(s) since base.\n\n' "$(wc -l < "$upchanged" | xargs)" + + # Files upstream DELETED but the fork still touches → modify/delete hotspots. + git diff --diff-filter=D --name-only "$base" "$UPSTREAM" | sort -u > /tmp/.reanchor_del.$$ 2>/dev/null || true + local updeleted=/tmp/.reanchor_del.$$ + + local clean=0 overlap=0 c files ov tot + printf '%-8s %-6s %-9s %s\n' "CLASS" "OVLAP" "RISK" "COMMIT" + while IFS= read -r c; do + [ -n "$c" ] || continue + files="$(git show --name-only --format= "$c" 2>/dev/null || true)" + [ -n "$files" ] || continue + ov="$(comm -12 <(printf '%s\n' "$files" | sort -u) "$upchanged" | wc -l | xargs)" + tot="$(printf '%s\n' "$files" | grep -c . || true)" + if [ "$ov" -eq 0 ]; then + clean=$((clean+1)) + else + overlap=$((overlap+1)) + # delete-risk: does this commit touch a file upstream deleted? + local delrisk="-" + comm -12 <(printf '%s\n' "$files" | sort -u) "$updeleted" | grep -q . && delrisk="DELETE" + printf '%-8s %-6s %-9s %s\n' "overlap" "$ov/$tot" "$delrisk" "$(git log -1 --format='%h %s' "$c")" + fi + done < <(git rev-list "$base..$FORK") + + echo "" + printf 'SUMMARY: %s commit(s) auto-apply (touch only fork-owned files); %s overlap upstream.\n' "$clean" "$overlap" + printf 'A one-shot merge would show all overlaps at once; a re-anchor resolves them per commit.\n' + printf 'Next: scripts/reanchor.sh run (automated, fork-wins, logged decisions)\n' + rm -f "$upchanged" "$updeleted" +} + +# --------------------------------------------------------------------------- # +cmd_run() { + local policy="theirs" branch="" wt="${TMPDIR:-/tmp}/metamanifold-reanchor" keep=0 + while [ $# -gt 0 ]; do + case "$1" in + --policy) policy="${2:?--policy needs a value}"; shift 2;; + --branch) branch="${2:?--branch needs a name}"; shift 2;; + --worktree) wt="${2:?--worktree needs a path}"; shift 2;; + --keep) keep=1; shift;; + *) die "run: unknown option '$1'";; + esac + done + case "$policy" in theirs|manual) ;; *) die "--policy must be 'theirs' or 'manual'";; esac + + preflight + local base; base="$(find_base)" + + rm -rf "$wt" + git worktree add --detach "$wt" "$FORK" >/dev/null 2>&1 || die "could not create worktree at $wt" + # shellcheck disable=SC2064 + trap "cd '$ROOT'; git worktree remove --force '$wt' >/dev/null 2>&1 || true" EXIT + + cd "$wt" + export GIT_EDITOR=true + printf 'reanchor: rebasing %s onto %s (policy=%s, base=%s)\n' "$FORK" "$UPSTREAM" "$policy" "$(git rev-parse --short "$base")" + local decisions=/tmp/.reanchor_decisions.$$; : > "$decisions" + git -c core.editor=true rebase --onto "$UPSTREAM" "$base" -X theirs >/dev/null 2>&1 || true + + local gd; gd="$(git rev-parse --git-dir)" + local iters=0 + while [ -d "$gd/rebase-merge" ] || [ -d "$gd/rebase-apply" ]; do + iters=$((iters+1)) + [ "$iters" -gt 400 ] && { printf 'reanchor: SAFETY CAP hit (%s iterations) — leaving the rebase in place for inspection.\n' "$iters" >&2; return 3; } + local cf; cf="$(git diff --name-only --diff-filter=U)" + if [ -z "$cf" ]; then git -c core.editor=true rebase --continue >/dev/null 2>&1 || true; continue; fi + local cur; cur="$(git log -1 --format='%h %s' REBASE_HEAD 2>/dev/null || echo '?')" + if [ "$policy" = "manual" ]; then + printf '\nreanchor: STOPPED for manual review at %s\n' "$cur" >&2 + printf 'Conflicted files:\n%s\n' "$cf" >&2 + printf 'Resolve, then: git -C %s rebase --continue (or: git -C %s rebase --abort)\n' "$wt" "$wt" >&2 + return 2 + fi + local f + while IFS= read -r f; do + [ -n "$f" ] || continue + if git cat-file -e "$UPSTREAM":"$f" 2>/dev/null; then + if git checkout --theirs -- "$f" 2>/dev/null && git add -- "$f" 2>/dev/null; then + printf 'CONTENT fork-wins : %s [%s]\n' "$f" "$cur" >> "$decisions" + else + git rm -f -- "$f" >/dev/null 2>&1 && printf 'DELETE (fork) : %s [%s]\n' "$f" "$cur" >> "$decisions" + fi + else + git rm -f -- "$f" >/dev/null 2>&1 && printf 'DELETE (upstream): %s [%s]\n' "$f" "$cur" >> "$decisions" + fi + done <<< "$cf" + git -c core.editor=true rebase --continue >/dev/null 2>&1 || true + done + + local ahead; ahead="$(git rev-list --count HEAD --not "$UPSTREAM")" + local residual; residual="$(git diff --name-only --diff-filter=U | wc -l | xargs)" + printf '\nreanchor: COMPLETE.\n' + printf ' granular commits on top of upstream : %s\n' "$ahead" + printf ' residual conflicts : %s\n' "$residual" + printf ' decisions the policy made : %s\n' "$(grep -c . "$decisions" || echo 0)" + if [ -s "$decisions" ]; then printf '\n --- decisions ---\n'; sed 's/^/ /' "$decisions"; fi + local drift; drift="$(git diff --name-only "$FORK" HEAD | wc -l | xargs)" + printf '\n files the re-anchored tree keeps that the fork dropped (review these): %s\n' "$drift" + git diff --name-only "$FORK" HEAD | sed 's/^/ /' | head -20 + + if [ -n "$branch" ]; then + git branch -f "$branch" HEAD >/dev/null 2>&1 && printf '\n branch %s -> %s\n' "$branch" "$(git rev-parse --short HEAD)" + fi + printf '\nNext:\n' + printf ' review : git -C %s log --oneline %s..HEAD | head\n' "$wt" "$UPSTREAM" + printf ' stray build output (e.g. web/dist) is upstream-committed; drop it: git -C %s rm -r web/dist\n' "$wt" + printf ' land : merge this granular branch into upstream, or cherry-pick tiers.\n' + + rm -f "$decisions" + if [ "$keep" -eq 1 ]; then trap - EXIT; printf '\n(worktree kept at %s)\n' "$wt"; fi + return 0 +} + +cmd_status() { + local gd; gd="$(git rev-parse --git-dir)" + if [ -d "$gd/rebase-merge" ] || [ -d "$gd/rebase-apply" ]; then + echo "reanchor: a rebase is in progress." + git status | sed -n '1,8p' + else + echo "reanchor: no re-anchor in progress." + fi +} + +usage() { sed -n '2,44p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; } + +main() { + local cmd="${1:-}"; shift || true + case "$cmd" in + plan) cmd_plan "$@";; + run) cmd_run "$@";; + status) cmd_status "$@";; + ""|help|-h|--help) usage;; + *) die "unknown command '$cmd' (try: plan|run|status)";; + esac +} + +main "$@"