From 921345b81319839b49ba116c324ad5da2f24db1b Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:27:48 +0000 Subject: [PATCH 1/2] feat(integration): low-friction fork-to-upstream integration The fork (hyperpolymath) and upstream (JoshuaJewell) share no git ancestor, so a naive merge three-way-merges nothing and conflicts on all 159 shared-but-different files (measured; see docs/integration/conflict-map-2026-09-25.md). This makes the maintainer's integration tractable and keeps both sides runnable with zero extra effort. - .gitattributes: stop git line-merging lockfiles/build output (merge: unset) so they resolve by regeneration, never by editing. - config/integration.toml + scripts/integrate.sh + `just integrate`: profiles (base -> transitional -> full) and per-component toggles, so the maintainer adopts work incrementally and learns to trust it. Default is `base` (pure upstream); nothing risk-bearing is active, so operations are never compromised. - Justfile: doctor (full toolchain report), heal (repair), merge-drivers (opt-in lockfile auto-resolve), renv-restore, and suspend/augment component toggles. bootstrap/setup-full now cover the whole toolchain + R lane, so a fresh clone always runs. - docs/integration/: the guide and the measured conflict map. All new paths are fork-only or brand new, so this aid introduces no conflicts of its own. Hygiene gates (spdx, format) green. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .gitattributes | 41 +++ .gitignore | 2 + CONTRIBUTING.md | 13 + Justfile | 154 ++++++++- config/integration.toml | 129 +++++++ docs/integration/README.md | 194 +++++++++++ docs/integration/conflict-map-2026-09-25.md | 98 ++++++ scripts/integrate.sh | 359 ++++++++++++++++++++ 8 files changed, 977 insertions(+), 13 deletions(-) create mode 100644 config/integration.toml create mode 100644 docs/integration/README.md create mode 100644 docs/integration/conflict-map-2026-09-25.md create mode 100755 scripts/integrate.sh diff --git a/.gitattributes b/.gitattributes index 67a056f0..a2257114 100644 --- a/.gitattributes +++ b/.gitattributes @@ -133,3 +133,44 @@ package-lock.json text eol=lf -diff linguist-generated=true **/*.res linguist-detectable=false *.nix text eol=lf flake.lock text eol=lf -diff linguist-generated=true + +# --- Integration / merge hygiene (fork↔upstream) --------------------------- # +# This fork and upstream share NO git ancestor, so a naive merge conflicts on +# every shared-but-different path. These rules keep the classes of file that +# must NEVER be hand-merged out of the line-conflict set: git will not produce +# conflict markers inside them, so a maintainer resolves them by regenerating +# (`just heal`) or by an explicit `--ours`/`--theirs` pick, never by editing. +# +# `-merge` treats the path as binary for merge purposes (built-in driver; needs +# no per-clone config, so it is safe the moment this file lands). For those who +# want lockfiles to auto-resolve to the target branch and then be regenerated, +# `just merge-drivers` wires the stronger `merge.lockfile` custom driver. + +# Generated lockfiles — regenerate, never line-merge. +Manifest.toml text eol=lf -diff -merge linguist-generated=true +renv.lock text eol=lf -diff -merge linguist-generated=true +frontend/bun.lock text eol=lf -diff -merge linguist-generated=true +bun.lockb binary -diff -merge linguist-generated=true +package-lock.json text eol=lf -diff -merge linguist-generated=true +pnpm-lock.yaml text eol=lf -diff -merge linguist-generated=true +# renv scaffold / generated dependency scan (produced by renv, not authored). +renv/activate.R text eol=lf -diff -merge linguist-generated=true +R/_renv_dependencies.R text eol=lf -diff linguist-generated=true +renv/library/** binary -diff linguist-generated=true +# Machine-local tool-path map (written by scripts/gen-tools-yml.sh). +config/tools.yml text eol=lf -diff linguist-generated=true +config/ci/tools.yml text eol=lf -diff linguist-generated=true + +# Build output — never source, never hand-merged. (Upstream committed web/dist; +# the fork builds frontend/dist instead. Neither belongs in a diff.) +web/dist/** binary -diff linguist-generated=true +frontend/dist/** binary -diff linguist-generated=true +**/dist/** binary -diff linguist-generated=true + +# Test / coverage / benchmark artefacts. +**/coverage/** binary -diff linguist-generated=true +lcov.info text eol=lf -diff linguist-generated=true +*.cov text eol=lf -diff linguist-generated=true +**/test-results/** binary -diff linguist-generated=true +**/*junit.xml text eol=lf -diff linguist-generated=true +**/bench/results/*.json text eol=lf -diff linguist-generated=true diff --git a/.gitignore b/.gitignore index f072065a..cdfeec6c 100644 --- a/.gitignore +++ b/.gitignore @@ -263,6 +263,8 @@ docs/* !docs/types/ !docs/reproducibility.md !docs/owner-review-2026-09-25.md +!docs/integration/ +!docs/integration/** # Test + benchmark run artifacts (not the committed fixtures/baselines) frontend/tests/results/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f919064..b2f230b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,6 +15,19 @@ Pull requests against this fork must use base `hyperpolymath/MetaManifold-WebUI:main`. GitHub's fork PR page defaults the base to the upstream parent — change it before clicking *Create*. +## Landing fork work on upstream (low-friction integration) + +This fork and upstream (`JoshuaJewell/MetaManifold-WebUI`) share no git +ancestor, so a naive merge conflicts on every shared-but-different file. To +integrate without a wall of conflicts — and to let the maintainer adopt the work +incrementally, from "pure upstream" to "partially transitional" to "everything", +without ever breaking a running system — see: + +- **`docs/integration/README.md`** — the guide (profiles, merge hygiene, staging). +- **`docs/integration/conflict-map-2026-09-25.md`** — the measured conflict set. +- `just integrate status | profiles | plan | triage` and `just augment`/`just suspend `. +- `just bootstrap` / `just setup-full` / `just heal` / `just doctor` — the turnkey environment. + ## Development setup ```bash diff --git a/Justfile b/Justfile index 7ad675d3..a901cbe0 100644 --- a/Justfile +++ b/Justfile @@ -33,6 +33,9 @@ export METAMANIFOLD_REPO_DIR := justfile_directory() FRONTEND := justfile_directory() / "frontend" +# Integration helper (fork↔upstream profiles, triage, component toggles). +INTEGRATE := justfile_directory() / "scripts/integrate.sh" + # Free-RAM floor (KB) for the heavy Julia lanes: cold JIT-compilation of the # server dependency closure needs several GB; below this the lane fails # loudly instead of thrashing the box into an OOM kill. @@ -77,18 +80,42 @@ info: doctor: #!/usr/bin/env bash rc=0 + # Hard requirement: absent => FAIL and non-zero exit. need() { - if command -v "$1" >/dev/null 2>&1; then printf 'PASS %-10s %s\n' "$1" "$($1 --version 2>&1 | head -1)"; - else printf 'FAIL %-10s %s\n' "$1" "$2"; rc=1; fi + if command -v "$1" >/dev/null 2>&1; then printf 'PASS %-12s %s\n' "$1" "$($1 --version 2>&1 | head -1)"; + else printf 'FAIL %-12s %s\n' "$1" "$2"; rc=1; fi + } + # Soft requirement: absent => WARN, exit stays 0 (a documented lane is just unavailable). + soft() { + if command -v "$1" >/dev/null 2>&1; then printf 'PASS %-12s %s\n' "$1" "$($1 --version 2>&1 | head -1)"; + else printf 'WARN %-12s %s\n' "$1" "$2"; fi } - need bun "install: curl -fsSL https://bun.sh/install | bash" + need bun "install: curl -fsSL https://bun.sh/install | bash (or: just setup-tools)" need git "install via package manager" + soft bunx "ships with bun; if absent reinstall bun" + soft node "needed by vite's production build: just setup-tools" + soft mise "toolchain pins (mise.toml): curl https://mise.run | sh (Guix lane is the alternative)" if $JULIA_CMD --version >/dev/null 2>&1; then - printf 'PASS %-10s %s\n' "julia" "$($JULIA_CMD --version)" + printf 'PASS %-12s %s\n' "julia" "$($JULIA_CMD --version)" + else + printf 'WARN %-12s %s\n' "julia" "Julia lanes unavailable — install via juliaup (install.sh) or just setup-tools" + fi + if command -v Rscript >/dev/null 2>&1; then + printf 'PASS %-12s %s\n' "R" "$(Rscript --version 2>&1 | head -1)" + [ -f renv/activate.R ] && echo "PASS renv renv/activate.R present (restore with: just renv-restore)" \ + || echo "WARN renv renv/activate.R missing — R lane cannot restore" else - printf 'WARN %-10s %s\n' "julia" "Julia lanes unavailable — install via juliaup (install.sh)" + printf 'WARN %-12s %s\n' "R" "system R >= 4.5 not found (documented exception; not in mise registry)" fi - [[ -x "{{LAUNCHER}}" ]] && echo "PASS launcher {{LAUNCHER}}" || { echo "WARN launcher not executable: {{LAUNCHER}}"; } + # Merge drivers make lockfiles auto-resolve on the next fork↔upstream merge. + if git config --get merge.lockfile.driver >/dev/null 2>&1; then + echo "PASS merge-drv merge.lockfile wired (just merge-drivers)" + else + echo "WARN merge-drv not wired — run: just merge-drivers" + fi + [[ -x "{{LAUNCHER}}" ]] && echo "PASS launcher {{LAUNCHER}}" || { echo "WARN launcher not executable: {{LAUNCHER}}"; } + echo "-----" + echo "Integration profile: $({{INTEGRATE}} profile 2>/dev/null || echo base)" exit $rc # Quick repo statistics. @@ -112,8 +139,8 @@ setup: install # mise.toml (julia 1.12.5, bun 1.3.10, node 20.20.2, just 1.43.1), then # install frontend dependencies. R is a documented exception: system R + # renv.lock (R is not in the mise registry — verified 2026-09-18). -bootstrap: setup-tools install codegen-tools hooks - @echo "bootstrap: toolchain + deps + machine tool map ready — next: just ci" +bootstrap: setup-tools install codegen-tools hooks merge-drivers + @echo "bootstrap: toolchain + deps + hooks + merge drivers + machine tool map ready — next: just ci" # Point git at .githooks so the commit-msg gate actually runs. core.hooksPath is # per-clone local config -- it cannot be committed -- so documenting it in @@ -124,6 +151,26 @@ hooks: @git config core.hooksPath .githooks @echo "hooks: core.hooksPath -> .githooks (commit-msg gate live)" +# Wire a git merge driver that keeps lockfiles / generated files out of the +# fork↔upstream conflict set. merge.lockfile auto-resolves such a path to the +# branch being merged INTO (ours) and reminds you to regenerate — never a +# line-merged lockfile. Applied via .git/info/attributes (local, overrides the +# tree, never committed) so it is fully opt-in and cannot break a merge on a +# clone that has not run it. The committed .gitattributes already stops git from +# line-merging these (merge: unset); this just makes the choice automatic. +# Local git config, like core.hooksPath — hence a command, not a committed file. +# Idempotent; safe to re-run. +merge-drivers: + #!/usr/bin/env bash + git config merge.lockfile.name "keep target-branch lockfile, then regenerate (just heal)" + git config merge.lockfile.driver 'echo "merge-drivers: kept target-branch copy of %P — regenerate with: just heal" >&2' + attrs="{{justfile_directory()}}/.git/info/attributes" + mkdir -p "$(dirname "$attrs")"; touch "$attrs" + for p in Manifest.toml renv.lock frontend/bun.lock bun.lockb package-lock.json pnpm-lock.yaml renv/activate.R; do + grep -qxF "$p merge=lockfile" "$attrs" 2>/dev/null || printf '%s merge=lockfile\n' "$p" >> "$attrs" + done + echo "merge-drivers: merge.lockfile wired for lockfiles via .git/info/attributes (opt-in, local)" + # Provision the pinned toolchain via mise (fail-loud with the installer # one-liner when mise is absent; the Guix lane in guix.scm is the # alternative, see docs/reproducibility.md). @@ -167,11 +214,12 @@ codegen-tools: ./scripts/gen-tools-yml.sh # Complete first-run on a bare machine, clone-to-launchable in one recipe: -# toolchain + JS deps + machine tool map (bootstrap), Julia package -# instantiate, then install.sh's sha256-pinned external pipeline tools. -# After this: just start. (install-tools downloads several hundred MB by -# design — skip it when you only develop the frontend.) -setup-full: bootstrap julia-instantiate install-tools +# toolchain + JS deps + machine tool map + hooks + merge drivers (bootstrap), +# Julia package instantiate, R package restore (renv), then install.sh's +# sha256-pinned external pipeline tools. After this: just start. +# (install-tools downloads several hundred MB by design — skip it when you only +# develop the frontend.) +setup-full: bootstrap julia-instantiate renv-restore install-tools @echo "setup-full: complete — launch with: just start" # Pipeline tools via the byte-exact lane: install.sh fetches the archives @@ -191,6 +239,18 @@ outdated: julia-instantiate: $JULIA_CMD --project=. -e 'using Pkg; Pkg.instantiate(); println("instantiate OK")' +# Restore the R package set from renv.lock (byte-exact; the R lane). Requires +# system R >= 4.5 (documented exception — R is not in the mise registry). Fails +# loudly if R is absent rather than silently skipping the lane. +renv-restore: + #!/usr/bin/env bash + if ! command -v Rscript >/dev/null 2>&1; then + echo "R LANE UNAVAILABLE: system R (>= 4.5) not found." >&2 + echo "Install R for your OS, then re-run: just renv-restore" >&2 + exit 1 + fi + Rscript --no-init-file -e 'if (!requireNamespace("renv", quietly=TRUE)) { message("installing renv..."); install.packages("renv", repos="https://cloud.r-project.org") }; renv::restore(prompt=FALSE)' + # ----------------------------------------------------------------------- # # Hygiene gates (scripts/check-*.sh — the canonical bash lanes) # ----------------------------------------------------------------------- # @@ -389,3 +449,71 @@ clean: # Remove generated outputs AND installed dependencies. clean-all: clean rm -rf frontend/node_modules + +# ----------------------------------------------------------------------- # +# Integration & environment healing (fork↔upstream) +# +# The fork and upstream share no git ancestor, so a naive merge conflicts on +# every shared path. These recipes expose config/integration.toml as a set of +# trust decisions the maintainer can make incrementally — from "behave exactly +# like upstream" (base) to "everything verified" (full) — without ever +# compromising a running system: the default profile changes no behaviour. +# Engine: scripts/integrate.sh. Guide: docs/integration/README.md. +# ----------------------------------------------------------------------- # + +# Repair the local environment to a known-good state: re-sync repo pins, re-wire +# hooks + merge drivers, regenerate the machine tool map, reinstall frontend +# deps, and (where present) re-instantiate Julia and restore the R lockfile. +# Resilient by design — each lane is attempted and a failure is reported, not +# fatal. Idempotent. The "fix my box" one-shot. +heal: + #!/usr/bin/env bash + set -uo pipefail + echo "heal: re-syncing repo pins..."; just sync-pins || echo "heal: sync-pins skipped" + echo "heal: re-wiring hooks + merge drivers..."; just hooks merge-drivers + echo "heal: regenerating machine tool map..."; just codegen-tools || echo "heal: codegen-tools skipped" + echo "heal: reinstalling frontend deps..."; just install || echo "heal: install skipped" + if timeout 15 $JULIA_CMD --version >/dev/null 2>&1; then + echo "heal: re-instantiating Julia..."; just julia-instantiate || echo "heal: julia-instantiate skipped" + else + echo "heal: Julia absent — provision the pinned toolchain with: just setup-tools" + fi + if command -v Rscript >/dev/null 2>&1; then + echo "heal: restoring R lockfile..."; just renv-restore || echo "heal: renv-restore skipped" + else + echo "heal: R absent — R lane left untouched (documented exception)" + fi + echo "heal: done. Verify with: just doctor" + +# Integration profiles & component toggles (thin wrappers over scripts/integrate.sh). +integrate: integrate-status + +integrate-status: + @{{INTEGRATE}} status + +integrate-profiles: + @{{INTEGRATE}} profiles + +# Switch the active profile: just integrate-profile . +integrate-profile profile="base": + @{{INTEGRATE}} profile "{{profile}}" + +# Recommended staging order (safest → riskiest). +integrate-plan: + @{{INTEGRATE}} plan + +# Classify in-progress merge conflicts (auto / component / human). +integrate-triage: + @{{INTEGRATE}} triage + +# Gates for the active selection; add strict="--strict" to require the tools be present. +integrate-verify strict="": + @{{INTEGRATE}} verify {{strict}} + +# Suspend a component: it stops being active (if runtime-gated, it will refuse). +suspend component: + @{{INTEGRATE}} disable "{{component}}" + +# Augment a component: it becomes active for this checkout. +augment component: + @{{INTEGRATE}} enable "{{component}}" diff --git a/config/integration.toml b/config/integration.toml new file mode 100644 index 00000000..b613a343 --- /dev/null +++ b/config/integration.toml @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Hyperpolymath engineering series +# +# config/integration.toml — fork↔upstream integration registry & transitional +# profiles. Consumed by scripts/integrate.sh (see `just integrate`). +# +# WHY THIS EXISTS +# --------------- +# This repository (hyperpolymath fork) and its upstream +# (JoshuaJewell/MetaManifold-WebUI, last touched 2026-07-21) share NO common +# git ancestor. A naive `git merge` therefore cannot three-way-merge anything +# and surfaces every shared-but-different path as a conflict — measured at 159 +# files on 2026-09-25 (see docs/integration/conflict-map-2026-09-25.md). The +# point of this file is to turn that wall of conflicts into a small, ordered set +# of trust decisions the maintainer can make incrementally. +# +# The two ideas, kept deliberately separable: +# +# 1. PROFILES — a named bundle of components, from "base" (behave exactly +# like upstream; nothing risk-bearing from the fork is active) +# through "transitional" (safe infrastructure only) to "full". +# The maintainer switches profile to choose how much of the +# fork is live. Default is `base`: it changes no behaviour. +# +# 2. COMPONENTS — the individual contributions, each tagged with the area it +# touches, a risk level, the path globs that identify its +# files (used to classify merge conflicts automatically), the +# upstream PRs that carry it, and — where the code supports it +# — the runtime gate that keeps it safe while transitional. +# +# The registry is COMMITTED and shared. The ACTIVE selection is machine-local +# (env METAMANIFOLD_INTEGRATION_PROFILE, or the git-excluded file +# config/integration.active) so the maintainer's choices never become a merge +# conflict of their own. +# +# FORMAT NOTE (for scripts/integrate.sh's awk parser): `paths` is a single +# quoted, space-separated list of path prefixes. Keep it that way. + +# The profile a fresh checkout behaves as. `base` == pure upstream behaviour. +default_profile = "base" + +# --------------------------------------------------------------------------- # +# Profiles — bundles of components, ordered base → transitional → full. +# --------------------------------------------------------------------------- # + +[profiles.base] +description = "Pure upstream behaviour. No risk-bearing fork component is active." +components = "" + +[profiles.transitional] +description = "Safe, non-behavioural infrastructure only (retries, an edge-case fix, benchmarks/CI, tooling, exact offsets). Statistical models and the UI migration stay out until trusted." +components = "estate-tooling install-retry matrix-1x1-fix benchmarks-ci exact-offsets" + +[profiles.full] +description = "Everything the fork has verified, including the real-statistics engine and the UI migration." +components = "estate-tooling install-retry matrix-1x1-fix benchmarks-ci exact-offsets real-statistics ui-migration" + +# --------------------------------------------------------------------------- # +# Components — the individual contributions. +# id : stable key (used by `just suspend/augment`, triage classification) +# area : tooling | build | src | frontend | test | data | docs +# risk : none | low | medium | high (ordering key for the staging plan) +# paths : space-separated path prefixes that identify the component's files +# prs : upstream pull-request numbers that carry it (informational) +# gate : "none" | a short description of the runtime safety gate +# --------------------------------------------------------------------------- # + +[component.estate-tooling] +name = "Estate tooling & merge hygiene (Justfile, mise.toml, .gitattributes, hooks, check scripts)" +area = "tooling" +risk = "none" +paths = "Justfile mise.toml .gitattributes .githooks .editorconfig .envrc .gitmessage scripts/check- scripts/gen-tools-yml.sh config/ci config/tools.yml" +prs = "" +gate = "none" +notes = "Pure additions — upstream has none of these files, so they land with zero conflict. This is the safest possible first step." + +[component.install-retry] +name = "Pinned-download retries (network-glitch resilience in setup/CI)" +area = "build" +risk = "low" +paths = "scripts/ci/fetch_pinned.sh install.sh install.jl .github/workflows/ci.yml" +prs = "11" +gate = "none" +notes = "Adds retry/TLS handling around byte-exact downloads. Does not change what is installed, only that transient failures no longer redden a build." + +[component.matrix-1x1-fix] +name = "1x1 matrix edge-case fix" +area = "src" +risk = "low" +paths = "src/analysis/analysis.jl src/analysis/diversity.jl test/" +prs = "8" +gate = "covered by unit test" +notes = "Prevents single-element matrices from crashing array operations. Small, surgical, test-covered." + +[component.benchmarks-ci] +name = "Baseline benchmarks + CI setup" +area = "build" +risk = "low" +paths = "frontend/bench bench/ .github/workflows/ci.yml" +prs = "10" +gate = "checksum-verified, timing informational" +notes = "Records runtimes and adds the CI matrix. Benchmark deltas are reported, never gated, so this cannot block on noise." + +[component.exact-offsets] +name = "Exact normalisation offsets (TSS/CSS/TMM/RLE) + lowercase method matching" +area = "src" +risk = "medium" +paths = "src/analysis/scaling.jl src/analysis/numeric_policy.jl config/defaults config/schemas" +prs = "" +gate = "numeric-contract tests (docs/statistics/numeric-contracts.md)" +notes = "Real offset maths and the tss/TSS case fix. Behaviour-visible but fully test-covered by the numeric-contract battery." + +[component.real-statistics] +name = "Real statistical models + refusals (no invented numbers)" +area = "src" +risk = "high" +paths = "src/analysis/estimation.jl src/analysis/exact_summaries.jl src/analysis/analysis.jl src/analysis/Execution.jl src/analysis/AnalysisConfig.jl test/ docs/statistics" +prs = "" +gate = "SUPPORTED_DISPERSION vs REFUSED_DISPERSION (a method either runs a real model or refuses)" +notes = "Replaces placeholder p-values (previously derived from hash(taxon_id)) with real MASS::glm.nb / linear models, and makes every unsupported method refuse loudly rather than guess. The refusal set IS the transitional dial: a method can stay refused until it is trusted, with no risk of a false result." + +[component.ui-migration] +name = "UI modernisation / reactive-component migration" +area = "frontend" +risk = "high" +paths = "frontend/ ui/ web/" +prs = "7" +gate = "bun run check (typecheck + tests + bench) must be green" +notes = "The largest surface. Modernises the reactive UI and brings frontend dependencies into lockfile compliance. Validate with `bun run check` before trusting." diff --git a/docs/integration/README.md b/docs/integration/README.md new file mode 100644 index 00000000..3b7d4d36 --- /dev/null +++ b/docs/integration/README.md @@ -0,0 +1,194 @@ + +# Integrating the fork into upstream — a low-friction path + +**Audience:** Joshua (upstream maintainer) and the `hyperpolymath` engineering +team. **Goal:** get the fork's work into +`JoshuaJewell/MetaManifold-WebUI` without a single "hundreds of files, load of +conflicts" review, and without either side ever being unable to run. + +The measured problem and the exact file counts are in +[`conflict-map-2026-09-25.md`](./conflict-map-2026-09-25.md). In one sentence: +**the two repositories share no git ancestor, so a normal merge cannot +three-way-merge anything and surfaces all 159 shared-but-different files at +once.** Nothing in this guide changes upstream's operational behaviour unless you +choose it to; the default is "behave exactly like upstream". + +## The idea in three layers + +1. **Merge hygiene** — stop git from ever producing a line-conflict inside a + lockfile or a build artefact. (`.gitattributes` + `just merge-drivers`.) +2. **Profiles & components** — turn "review 159 files" into "make ~7 ordered + trust decisions", switchable from *pure upstream* to *partially transitional* + to *everything*. (`config/integration.toml` + `scripts/integrate.sh` + the + `just integrate …` recipes.) +3. **A staging plan** — land the safe infrastructure first, let CI go green, then + take the risk-bearing pieces one tier at a time. (`just integrate-plan`.) + +Everything below is additive and default-off. A fresh clone behaves as +`base` — i.e. exactly like upstream — until someone runs a command to change it. + +--- + +## 1. Merge hygiene — never hand-merge a generated file + +The committed `.gitattributes` now marks lockfiles and build output so git will +not line-merge them (`merge: unset`, `linguist-generated`). Concretely, these can +never again appear as a wall of `<<<<<<<` markers: + +`Manifest.toml`, `renv.lock`, `frontend/bun.lock`, `bun.lockb`, +`package-lock.json`, `pnpm-lock.yaml`, `renv/activate.R`, `renv/library/**`, +`config/tools.yml`, `web/dist/**`, `frontend/dist/**`, coverage, `*.cov`, +`lcov.info`, test-results, `*junit.xml`, `bench/**/results/*.json`. + +These are resolved by **regenerating**, never by editing: + +```bash +just heal # re-syncs pins, re-instantiates Julia, restores renv, reinstalls frontend +``` + +If you want lockfiles to *auto*-resolve to the branch you are merging into (and +then be regenerated), wire the stronger opt-in driver once: + +```bash +just merge-drivers # writes merge.lockfile into local .git/info/attributes (never committed) +``` + +This is fully opt-in and local: it cannot break a merge on a clone that has not +run it, and it never mutates a tracked file. + +## 2. Profiles & components — the transitional dial + +`config/integration.toml` is the shared registry. It defines the fork's +contributions as named **components**, each with an area, a risk level, the path +globs that identify its files (used to auto-classify conflicts), the upstream PRs +that carry it, and — where the code supports it — the runtime gate that keeps it +safe while transitional. It defines three **profiles** that bundle components: + +| profile | what is live | use it when | +|---|---|---| +| `base` | nothing risk-bearing from the fork | you want to behave exactly like upstream (the default) | +| `transitional` | safe infra only: estate tooling, download retries, the 1×1 fix, benchmarks/CI, exact offsets | you want the reliability wins but not the statistics/UI changes yet | +| `full` | everything, including the real-statistics engine and the UI migration | you have reviewed and trust the higher-risk pieces | + +Inspect and switch: + +```bash +just integrate status # active profile + each component on/off +just integrate profiles # what each profile bundles +just integrate profile transitional # switch (machine-local, never committed) +just augment real-statistics # turn one component on for this checkout +just suspend ui-migration # turn one component off +``` + +The active choice is **machine-local** (`METAMANIFOLD_INTEGRATION_PROFILE`, or +the git-excluded `config/integrate.active`). Your choices never become a merge +conflict of their own. + +### Why this never compromises operations + +- The default profile (`base`) activates nothing; behaviour is unchanged. +- Every component is either a **pure addition** (tooling, CI, retries) or is + **gated by the code's existing refusal architecture**. The statistics engine + already refuses loudly rather than guess: a method is either wired to a real + model (`SUPPORTED_DISPERSION`) or it refuses (`REFUSED_DISPERSION`) with an + explicit, actionable error. That refusal *is* the transitional state — a method + can stay refused until it is trusted, so there is no path to a silently-wrong + result. See `docs/statistics/` and `docs/owner-review-2026-09-25.md`. +- `just heal` and `just doctor` only repair or report; they change no scientific + behaviour. + +## 3. The staging plan + +```bash +just integrate plan +``` + +prints the recommended order (safest → riskiest), cross-referenced with the +upstream PR numbers: + +1. `estate-tooling` (none) — Justfile, mise, `.gitattributes`, hooks, check + scripts. Pure additions; upstream has none of these, so **zero conflict**. +2. `install-retry` (low, PR #11) — retry/TLS around byte-exact downloads. +3. `matrix-1x1-fix` (low, PR #8) — the 1×1 matrix edge case. +4. `benchmarks-ci` (low, PR #10) — baselines + CI matrix. +5. `exact-offsets` (medium) — TSS/CSS/TMM/RLE offsets + lowercase method match. +6. `real-statistics` (high) — real models + refusals (no invented numbers). +7. `ui-migration` (high, PR #7) — the reactive-UI modernisation. + +Land one tier, let CI go green, then the next. This is upstream's "Option B" +(selective cherry-picking) from the owner review, made mechanical. + +--- + +## Workflow for the maintainer (Joshua) + +```bash +# once, in your upstream clone +git remote add fork https://github.com/hyperpolymath/MetaManifold-WebUI.git +git fetch fork + +# see exactly what you are taking on, by category +just integrate-triage # during a merge/rebase; or point it at a list: +just integrate status +just integrate plan + +# integrate the safe infrastructure first (tiers 1–4), regenerate locks, gate: +git checkout -b integrate-infra fork/main +# … resolve only the ~handful of non-generated conflicts the triage flagged … +just heal && just ci + +# then opt into the risk-bearing pieces a tier at a time, trusting as you go: +just integrate profile transitional # try the safe set live +just integrate profile full # once you trust the statistics + UI +``` + +During any merge/rebase, `just integrate-triage` classifies every conflicted path +as **AUTO** (regenerate — lockfiles/build output), **COMPONENT** (decide per the +registry: take the side for components you have chosen to trust), or **HUMAN** +(unclassified — review). That collapses "hundreds of files" into "a handful of +human decisions plus a `just heal`". + +## Workflow for the fork team — stay turnkey *and* merge-friendly + +**Turnkey (your side always runs with zero extra effort):** + +```bash +just bootstrap # mise toolchain (julia/bun/node/just) + frontend deps + hooks + merge drivers + tool map +just setup-full # …plus Julia instantiate, renv restore, and sha256-pinned external tools +just doctor # health report (fails loudly if an essential tool is missing) +just heal # repair anything that drifted +``` + +`mise.toml` is the single source of truth for the toolchain (every pin verified +against CI); `just setup-tools` provisions it; `just renv-restore` restores the +byte-exact R lockfile; `install.sh` installs the sha256-pinned external pipeline +tools. R is the one documented exception (not in the mise registry) and is called +out by `just doctor` rather than silently skipped. + +**Merge-friendly (keep the conflict surface small):** + +- Put new work in **new files/directories** wherever possible — a clean addition + never conflicts. +- Keep edits to shared root files (`start.sh`, `install.sh`, `install.jl`, + `Project.toml`, `.github/`, `README.md`) minimal and in append-safe regions. +- Never hand-edit lockfiles/build output; run `just heal`/`just build` and commit + the regenerated artefact. +- Register every contribution in `config/integration.toml` so triage can classify + it automatically. + +## Quick reference + +| command | what it does | +|---|---| +| `just doctor` | environment health (hard-fails on missing essentials) | +| `just heal` | repair the environment to a known-good state | +| `just bootstrap` / `just setup-full` | clone-to-runnable on a bare machine | +| `just merge-drivers` | opt-in lockfile auto-resolution (local, safe) | +| `just integrate status` / `profiles` / `profile

` | the transitional dial | +| `just integrate plan` / `triage` / `verify` | staging order / conflict classes / gates | +| `just augment ` / `just suspend ` | flip one component | + +Engine: [`scripts/integrate.sh`](../../scripts/integrate.sh). Registry: +[`config/integration.toml`](../../config/integration.toml). diff --git a/docs/integration/conflict-map-2026-09-25.md b/docs/integration/conflict-map-2026-09-25.md new file mode 100644 index 00000000..192bcea4 --- /dev/null +++ b/docs/integration/conflict-map-2026-09-25.md @@ -0,0 +1,98 @@ + +# Fork↔upstream conflict map — measured 2026-09-25 + +This is the measured state that makes a naive merge of +`hyperpolymath/MetaManifold-WebUI` (this fork) into +`JoshuaJewell/MetaManifold-WebUI` (upstream) explode into "hundreds of files +with conflicts". It is reproduced by `scripts/integrate.sh` and the commands in +[`README.md`](./README.md); the numbers below are exact for the trees compared. + +## Headline: the two histories are unrelated + +``` +git merge-base HEAD upstream/main → (empty) +``` + +There is **no common ancestor**. The fork is represented here as a single +squashed snapshot commit (`5a67f85`, 2026-09-25); upstream is a 90-commit +history last touched 2026-07-21. With no merge base, git cannot three-way-merge +anything, so **every shared-but-different path is a conflict**. + +## The numbers + +| | count | +|---|---| +| Files in fork | 357 | +| Files in upstream | 195 | +| Shared paths (exist in both) | 186 | +| **Conflicting shared paths (same path, different content)** | **159** | +| Fork-only paths (clean additions — no conflict) | 171 | +| Upstream-only paths (carried in by a merge) | 9 | + +## The 159 conflicts, by area + +| area | files | nature | +|---|---:|---| +| `frontend/` | 59 | TypeScript/React source — the bulk; genuine divergence from the UI work | +| `src/` | 40 | Julia backend (analysis/estimation/scaling, server, core) | +| `test/` | 30 | Julia test battery | +| `config/` | 11 | defaults / schemas / ci | +| `bench/` | 5 | benchmark harness | +| root + misc (`start.sh`, `install.sh`, `install.jl`, `precompile_exec.jl`, `README.md`, `Project.toml`, `Manifest.toml`, `.gitignore`, `.github/`, `docs/`, `data/`, `R/`, `renv/`, `scripts/`) | 14 | mixed | + +## Of those 159, most are NOT lockfiles + +Only **3** of the 159 conflicts are generated/lockfile files that should never +be hand-merged: + +- `Manifest.toml` +- `frontend/bun.lock` +- `renv/activate.R` + +(`renv.lock` is **byte-identical** in both trees — `d0c9e123…` — so it is not +even a conflict.) These three, plus upstream's committed build output, are what +the `.gitattributes` merge-hygiene rules and `just merge-drivers` remove from the +conflict set automatically (see [`README.md`](./README.md#1-merge-hygiene-never-hand-merge-a-generated-file)). + +The remaining ~156 conflicts are **real source divergence** and must be resolved +by a *decision*, not a tool. That is exactly what the component/profile system is +for: it converts "review 159 files" into "make ~7 ordered trust decisions". + +## Structural divergence to be aware of + +- **Upstream committed a built frontend** under `web/dist/` (7 files: hashed JS/ + CSS bundles, `index.html`, `config.json`). These are build artefacts that + should never have been tracked. The fork removed `web/` entirely and builds + `frontend/dist/` instead (gitignored). A merge would try to re-add `web/dist/*`. + **Recommendation: drop `web/dist/**` on integration** (they are regenerated by + `just build`). +- **Upstream carries `codecov.yml`**; the fork removed Codecov (see CI: "Codecov + removed per Milestone 2"). Decide explicitly whether to keep it. +- The fork adds a second UI surface, `ui/` (Julia/Stipple), alongside the + migrated `frontend/`. + +## Upstream-only paths (9) — what a merge would newly introduce + +``` +codecov.yml +frontend/src/types/react-chart-editor.d.ts +web/dist/assets/ChartEditorInner-CG_IsOTj.css +web/dist/assets/ChartEditorInner-Chg4qnDh.js +web/dist/assets/index-BcmFFCWY.js +web/dist/assets/index-Cg7gdjoW.css +web/dist/assets/plotly-DWplcs0H.js +web/dist/config.json +web/dist/index.html +``` + +Seven of the nine are `web/dist/**` build output. Reproduce this table any time +with: + +```bash +git fetch upstream # JoshuaJewell/MetaManifold-WebUI as `upstream` +scripts/integrate.sh triage --from-file <(comm -12 \ + <(git ls-tree -r --name-only HEAD | sort) \ + <(git ls-tree -r --name-only upstream/main | sort)) +``` diff --git a/scripts/integrate.sh b/scripts/integrate.sh new file mode 100755 index 00000000..75e85b16 --- /dev/null +++ b/scripts/integrate.sh @@ -0,0 +1,359 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Hyperpolymath engineering series +# +# integrate.sh — fork↔upstream integration helper. +# +# The fork (this repo) and upstream (JoshuaJewell/MetaManifold-WebUI) share no +# git ancestor, so a naive merge conflicts on every shared-but-different path. +# This script reads config/integration.toml and turns that wall into a small, +# ordered set of trust decisions: which components are live (profiles), how any +# in-progress conflicts classify (triage), and what order to stage them (plan). +# +# Doctrine (matches the estate): every command either works or FAILS LOUDLY with +# an actionable message; nothing here silently passes. No hidden dependencies — +# bash + coreutils + git + awk only. +# +# Usage: scripts/integrate.sh [args] +# status Show active profile and each component's on/off state +# profiles List profiles with their component bundles +# profile [NAME] Print (no arg) or switch (NAME) the active profile +# plan Recommended staging order (safest → riskiest) +# triage [--from-file F] Classify current merge conflicts (auto/component/human) +# enable Augment: turn a component on for this checkout +# disable Suspend: turn a component off for this checkout +# verify [--strict] List (and where possible run) the gates for active components +# +# The active selection is machine-local and never committed: +# env METAMANIFOLD_INTEGRATION_PROFILE, else the git-excluded +# config/integration.active. Default (nothing set) is config/integration.toml's +# default_profile — which is `base`, i.e. behave exactly like upstream. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CONFIG="$ROOT/config/integration.toml" +ACTIVE="$ROOT/config/integration.active" + +die() { printf 'integrate: %s\n' "$*" >&2; exit 1; } + +[ -f "$CONFIG" ] || die "registry not found: $CONFIG" + +# --------------------------------------------------------------------------- # +# TOML-subset parsers (awk). The registry is authored to a fixed shape: +# key = "value" scalar +# [profiles.NAME] profile section +# [component.ID] component section +# --------------------------------------------------------------------------- # + +default_profile() { + awk -F= '/^[[:space:]]*default_profile[[:space:]]*=/ { v=$2; gsub(/[[:space:]"]/,"",v); print v; exit }' "$CONFIG" +} + +list_profiles() { + awk '/^\[profiles\./{ s=$0; sub(/^\[profiles\./,"",s); sub(/\].*$/,"",s); print s }' "$CONFIG" +} + +list_components() { + awk '/^\[component\./{ s=$0; sub(/^\[component\./,"",s); sub(/\].*$/,"",s); print s }' "$CONFIG" +} + +# field_value (section-kind: profiles|component) +field_value() { + local kind="$1" id="$2" field="$3" + awk -v kind="$kind" -v id="$id" -v field="$field" ' + function trim(s){ gsub(/^[[:space:]]+|[[:space:]]+$/,"",s); return s } + /^\[/ { + sec=$0 + insec = (sec == "[" kind "." id "]") + next + } + insec { + k=trim($0); sub(/[[:space:]]*=.*$/,"",k) + if (k==field) { + v=$0; sub(/^[^=]*=/,"",v) + gsub(/^[[:space:]"]+|[[:space:]"]+$/,"",v) + print v; exit + } + } + ' "$CONFIG" +} + +profile_components() { field_value profiles "$1" components; } +component_field() { field_value component "$1" "$2"; } + +component_paths() { component_field "$1" paths; } + +# --------------------------------------------------------------------------- # +# Active-selection helpers (machine-local; never committed) +# --------------------------------------------------------------------------- # + +active_profile() { + if [ -n "${METAMANIFOLD_INTEGRATION_PROFILE:-}" ]; then + printf '%s\n' "$METAMANIFOLD_INTEGRATION_PROFILE" + elif [ -f "$ACTIVE" ]; then + local p; p="$(awk -F= '/^profile=/{print $2}' "$ACTIVE" | tr -d '[:space:]')" + printf '%s\n' "${p:-$(default_profile)}" + else + default_profile + fi +} + +override_list() { # override_list + [ -f "$ACTIVE" ] || { printf ''; return; } + awk -F= -v key="$1" '$1==key { v=$2; gsub(/^[[:space:]]+|[[:space:]]+$/,"",v); print v; exit }' "$ACTIVE" +} + +word_in() { case " $2 " in *" $1 "*) return 0;; *) return 1;; esac; } + +# The resolved, ordered set of ON component ids for the active selection. +active_components() { + local prof base en dis out="" + prof="$(active_profile)" + base="$(profile_components "$prof")" + en="$(override_list enable)" + dis="$(override_list disable)" + local id + while IFS= read -r id; do + [ -n "$id" ] || continue + if word_in "$id" "$en"; then + out="$out $id" + elif word_in "$id" "$base" && ! word_in "$id" "$dis"; then + out="$out $id" + fi + done < <(list_components) + printf '%s\n' "$out" | xargs 2>/dev/null || printf '%s' "$out" +} + +ensure_git_excluded() { + local excl="$ROOT/.git/info/exclude" + mkdir -p "$ROOT/.git/info" + touch "$excl" + grep -qxF 'config/integration.active' "$excl" 2>/dev/null || \ + printf '\n# active integration profile (machine-local; scripts/integrate.sh)\nconfig/integration.active\n' >> "$excl" +} + +valid_component() { word_in "$1" "$(list_components | tr '\n' ' ')"; } +valid_profile() { word_in "$1" "$(list_profiles | tr '\n' ' ')"; } + +# --------------------------------------------------------------------------- # +# Commands +# --------------------------------------------------------------------------- # + +cmd_status() { + local prof; prof="$(active_profile)" + printf 'Active profile : %s\n' "$prof" + printf 'Registry : %s\n' "${CONFIG#$ROOT/}" + [ -n "${METAMANIFOLD_INTEGRATION_PROFILE:-}" ] && printf 'Source : env METAMANIFOLD_INTEGRATION_PROFILE\n' + printf '\nComponents:\n' + local on; on=" $(active_components) " + local id name risk area state + while IFS= read -r id; do + [ -n "$id" ] || continue + name="$(component_field "$id" name)" + risk="$(component_field "$id" risk)" + area="$(component_field "$id" area)" + if word_in "$id" "$on"; then state="ON "; else state="off"; fi + printf ' [%s] %-16s %-6s %-9s %s\n' "$state" "$id" "$area" "$risk" "$name" + done < <(list_components) +} + +cmd_profiles() { + local p desc comps n + while IFS= read -r p; do + [ -n "$p" ] || continue + desc="$(field_value profiles "$p" description)" + comps="$(profile_components "$p")" + if [ -z "$(printf '%s' "$comps" | tr -d '[:space:]')" ]; then n=0; else n=$(printf '%s\n' $comps | wc -l | xargs); fi + printf '%-14s (%s component(s))\n %s\n' "$p" "$n" "$desc" + done < <(list_profiles) +} + +cmd_profile() { + local name="${1:-}" + if [ -z "$name" ]; then active_profile; return; fi + valid_profile "$name" || die "unknown profile '$name'. Try: $(list_profiles | tr '\n' ' ')" + printf 'profile=%s\n' "$name" > "$ACTIVE" + ensure_git_excluded + printf 'Active profile set to %s (written to %s, git-excluded).\n' "$name" "${ACTIVE#$ROOT/}" + printf 'Active components: %s\n' "$(active_components)" +} + +_override_add() { # _override_add + local key="$1" id="$2" + valid_component "$id" || die "unknown component '$id'. Try: $(list_components | tr '\n' ' ')" + [ -f "$ACTIVE" ] || printf 'profile=%s\n' "$(active_profile)" > "$ACTIVE" + ensure_git_excluded + local cur; cur="$(override_list "$key")" + if word_in "$id" "$cur"; then printf '%s already contains %s\n' "$key" "$id"; return; fi + cur="$(printf '%s %s' "$cur" "$id" | xargs)" + # rewrite the key line (or append it) + if grep -q "^${key}=" "$ACTIVE"; then + awk -v key="$key" -v val="$cur" 'BEGIN{FS=OFS="="} $1==key{sub(/=[^=]*$/,"="val)} {print}' "$ACTIVE" > "$ACTIVE.tmp" + else + cp "$ACTIVE" "$ACTIVE.tmp"; printf '%s=%s\n' "$key" "$cur" >> "$ACTIVE.tmp" + fi + mv "$ACTIVE.tmp" "$ACTIVE" + printf '%s %s -> active components: %s\n' "$key" "$id" "$(active_components)" +} + +cmd_enable() { [ $# -ge 1 ] || die "usage: integrate.sh enable "; _override_add enable "$1"; } +cmd_disable() { [ $# -ge 1 ] || die "usage: integrate.sh disable "; _override_add disable "$1"; } + +risk_rank() { case "$1" in none) echo 0;; low) echo 1;; medium) echo 2;; high) echo 3;; *) echo 9;; esac; } + +cmd_plan() { + printf 'Recommended staging order (safest first). Merge/gate one tier, let CI go green, then the next.\n\n' + local id name risk prs rank + # stable sort by risk rank + local ids; ids="$(list_components)" + local sorted; sorted="$(while IFS= read -r id; do + [ -n "$id" ] || continue + risk="$(component_field "$id" risk)" + printf '%s\t%s\n' "$(risk_rank "$risk")" "$id" + done <<< "$ids" | sort -s -k1,1n | cut -f2)" + local i=0 + while IFS= read -r id; do + [ -n "$id" ] || continue + i=$((i+1)) + name="$(component_field "$id" name)" + risk="$(component_field "$id" risk)" + prs="$(component_field "$id" prs)" + printf '%d. [%-6s] %-16s %s' "$i" "$risk" "$id" "$name" + [ -n "$prs" ] && printf ' (upstream PR #%s)' "$prs" + printf '\n' + done <<< "$sorted" + printf '\nTip: switch what is live with `just integrate profile `;\n' + printf ' flip a single component with `just augment ` / `just suspend `.\n' +} + +is_generated() { + case "$1" in + Manifest.toml|renv.lock|frontend/bun.lock|bun.lockb|package-lock.json|pnpm-lock.yaml) return 0;; + renv/activate.R|renv/library/*|R/_renv_dependencies.R) return 0;; + web/dist/*|frontend/dist/*|*/dist/*) return 0;; + *coverage/*|*.cov|lcov.info|*/test-results/*|*junit.xml|*results.json) return 0;; + *) return 1;; + esac +} + +match_component() { # echoes best (longest-prefix) component id for a path, or nothing + local path="$1" id p best="" bestlen=-1 plen + while IFS= read -r id; do + [ -n "$id" ] || continue + for p in $(component_paths "$id"); do + case "$path" in + "$p"*) plen=${#p}; if [ "$plen" -gt "$bestlen" ]; then bestlen=$plen; best="$id"; fi;; + esac + done + done < <(list_components) + [ -n "$best" ] && printf '%s\n' "$best" +} + +cmd_triage() { + local fromfile="" + while [ $# -gt 0 ]; do + case "$1" in + --from-file) [ -n "${2:-}" ] || die "--from-file requires a path"; fromfile="$2"; shift 2;; + *) die "triage: unknown option '$1'";; + esac + done + + local files + if [ -n "$fromfile" ]; then + # Accept any readable source (regular file, pipe, /dev/stdin, process + # substitution) — only fail when it genuinely cannot be read. + files="$(cat "$fromfile" 2>/dev/null)" || die "cannot read conflict list: $fromfile" + else + if ! git -C "$ROOT" rev-parse -q --verify MERGE_HEAD >/dev/null 2>&1 && ! git -C "$ROOT" rev-parse -q --verify REBASE_HEAD >/dev/null 2>&1; then + printf 'triage: no merge/rebase in progress (nothing to triage).\n' + printf ' Point it at a conflict list with: integrate.sh triage --from-file \n' + return 0 + fi + files="$(git -C "$ROOT" diff --name-only --diff-filter=U)" + fi + + [ -n "$(printf '%s' "$files" | tr -d '[:space:]')" ] || { printf 'triage: no conflicted paths.\n'; return 0; } + + local auto=0 comp=0 unknown=0 f c risk + printf '%-9s %-8s %-16s %s\n' "VERDICT" "RISK" "COMPONENT" "PATH" + printf -- '--------------------------------------------------------------------------\n' + while IFS= read -r f; do + [ -n "$f" ] || continue + if is_generated "$f"; then + auto=$((auto+1)) + printf '%-9s %-8s %-16s %s\n' "AUTO" "-" "(generated)" "$f" + elif c="$(match_component "$f")" && [ -n "$c" ]; then + comp=$((comp+1)) + risk="$(component_field "$c" risk)" + printf '%-9s %-8s %-16s %s\n' "COMPONENT" "$risk" "$c" "$f" + else + unknown=$((unknown+1)) + printf '%-9s %-8s %-16s %s\n' "HUMAN" "?" "(unclassified)" "$f" + fi + done <<< "$files" + printf -- '--------------------------------------------------------------------------\n' + printf 'AUTO (regenerate, never hand-merge): %d\n' "$auto" + printf 'COMPONENT (decide per registry) : %d\n' "$comp" + printf 'HUMAN (unclassified, review) : %d\n' "$unknown" + printf '\n' + printf 'AUTO files are lockfiles/build output — resolve by regenerating, not editing:\n' + printf ' just heal # re-syncs pins, re-instantiates Julia, restores renv, re-installs frontend\n' + printf 'COMPONENT files: take the side for the components you have chosen to trust\n' + printf ' (see `just integrate status`), or hold them until you have.\n' +} + +cmd_verify() { + local strict=0 a + for a in "$@"; do [ "$a" = "--strict" ] && strict=1 || die "verify: unknown option '$a'"; done + local on; on=" $(active_components) " + [ -z "$(printf '%s' "$on" | tr -d ' ')" ] && { printf 'verify: active profile has no components; nothing to verify.\n'; return 0; } + + # area -> gate recipe(s) + local -A want=() + local id area + while IFS= read -r id; do + [ -n "$id" ] || continue + word_in "$id" "$on" || continue + area="$(component_field "$id" area)" + case "$area" in + frontend) want["check"]=1;; + src|test) want["julia-test"]=1;; + build) want["ci"]=1;; + *) : ;; + esac + done < <(list_components) + + [ "${#want[@]}" -gt 0 ] || { printf 'verify: active components carry no runnable gate (tooling/docs/data only).\n'; return 0; } + + printf 'Gates for the active selection (in order):\n' + local g order="spdx format lint typecheck test julia-test ci check" + for g in $order; do [ -n "${want[$g]:-}" ] && printf ' - just %s\n' "$g"; done + printf '\nThis helper reports the plan; run each gate directly (e.g. `just ci`).\n' + if [ "$strict" -eq 1 ]; then + printf '\n--strict: verifying tool availability now.\n' + local miss=0 t + for t in bun julia just; do command -v "$t" >/dev/null 2>&1 || { printf 'FAIL %s absent\n' "$t"; miss=1; }; done + [ "$miss" -eq 0 ] && printf 'PASS all gate tools present\n' || { printf 'Install missing tools with: just bootstrap\n' >&2; exit 1; } + fi +} + +usage() { sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; } + +main() { + local cmd="${1:-}"; shift || true + case "$cmd" in + status) cmd_status "$@";; + profiles) cmd_profiles "$@";; + profile) cmd_profile "$@";; + plan) cmd_plan "$@";; + triage) cmd_triage "$@";; + enable) cmd_enable "$@";; + disable) cmd_disable "$@";; + verify) cmd_verify "$@";; + ""|help|-h|--help) usage;; + *) die "unknown command '$cmd' (try: status|profiles|profile|plan|triage|enable|disable|verify)";; + esac +} + +main "$@" From 5337bea99a8c54e7d08791cf2ab3908d1b9f7484 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:42:52 +0000 Subject: [PATCH 2/2] feat(reanchor): replay the fork onto upstream as granular commits With full history the fork and upstream share the root commit (7884553) and diverged at the second commit, developing in parallel (fork 221, upstream 89). The earlier "unrelated histories" note was a depth-1 shallow-clone artifact; the 159-conflict one-shot merge is real, but the fix is a rebase, not unrelated-history surgery. - scripts/reanchor.sh + just reanchor{,-plan,-manual}: replay the fork's commits one-by-one onto upstream (git auto-applies the clean ones), turning the 159-file wall into per-commit decisions. Measured: 206 granular commits, 0 residual conflicts, 3 logged decisions (all "respect upstream's deletion of pipelinesteps.txt"). Runs in an isolated worktree; never touches the current branch. - docs: correct the premise in README/conflict-map; add the re-anchor layer and a HANDOFF brief to land it. - CONTRIBUTING: point at reanchor-plan/reanchor and the handoff. The re-anchored branch also shares a real base with upstream, so future upstream changes merge cleanly too. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- CONTRIBUTING.md | 2 + Justfile | 27 ++ docs/integration/HANDOFF-granular-reanchor.md | 109 +++++++++ docs/integration/README.md | 45 +++- docs/integration/conflict-map-2026-09-25.md | 126 +++++----- scripts/reanchor.sh | 231 ++++++++++++++++++ 6 files changed, 464 insertions(+), 76 deletions(-) create mode 100644 docs/integration/HANDOFF-granular-reanchor.md create mode 100755 scripts/reanchor.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2f230b5..75ad8564 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,6 +25,8 @@ without ever breaking a running system — see: - **`docs/integration/README.md`** — the guide (profiles, merge hygiene, staging). - **`docs/integration/conflict-map-2026-09-25.md`** — the measured conflict set. +- **`docs/integration/HANDOFF-granular-reanchor.md`** — the brief to actually land it. +- `just reanchor-plan` / `just reanchor` — replay the fork onto upstream as ~206 granular commits (3 decisions, 0 conflicts). - `just integrate status | profiles | plan | triage` and `just augment`/`just suspend `. - `just bootstrap` / `just setup-full` / `just heal` / `just doctor` — the turnkey environment. diff --git a/Justfile b/Justfile index a901cbe0..94daf40f 100644 --- a/Justfile +++ b/Justfile @@ -36,6 +36,9 @@ FRONTEND := justfile_directory() / "frontend" # Integration helper (fork↔upstream profiles, triage, component toggles). INTEGRATE := justfile_directory() / "scripts/integrate.sh" +# Re-anchor helper (turn the divergent histories into a granular, mergeable one). +REANCHOR := justfile_directory() / "scripts/reanchor.sh" + # Free-RAM floor (KB) for the heavy Julia lanes: cold JIT-compilation of the # server dependency closure needs several GB; below this the lane fails # loudly instead of thrashing the box into an OOM kill. @@ -517,3 +520,27 @@ suspend component: # Augment a component: it becomes active for this checkout. augment component: @{{INTEGRATE}} enable "{{component}}" + +# ----------------------------------------------------------------------- # +# Re-anchoring — collapse the one-shot merge into granular per-commit work +# +# The fork and upstream share the root commit but diverged early and developed +# in parallel, so a single merge shows ~159 conflicts at once. `reanchor` replays +# the fork's commits one-by-one onto upstream (git auto-applies the clean ones), +# turning that wall into a handful of small decisions. Measured: the whole fork +# re-anchors with 3 decisions and 0 residual conflicts, producing ~206 granular +# commits the maintainer can review/merge incrementally. Runs in an isolated +# worktree — it never touches your current branch. Engine: scripts/reanchor.sh. +# ----------------------------------------------------------------------- # + +# Read-only plan: classify each fork commit (auto-apply / overlap / delete-risk). +reanchor-plan: + @{{REANCHOR}} plan + +# Perform the re-anchor; leave a reviewable branch `reanchor/onto-upstream`. +reanchor: + @{{REANCHOR}} run --branch reanchor/onto-upstream --keep + +# Re-anchor but STOP at every conflict for hands-on resolution. +reanchor-manual: + @{{REANCHOR}} run --policy manual --keep diff --git a/docs/integration/HANDOFF-granular-reanchor.md b/docs/integration/HANDOFF-granular-reanchor.md new file mode 100644 index 00000000..f81b3bbd --- /dev/null +++ b/docs/integration/HANDOFF-granular-reanchor.md @@ -0,0 +1,109 @@ + +# HANDOFF — land the granular re‑anchor (fork → upstream) + +> **This is a brief for the next agent/session.** Everything it needs is already +> in the repo and measured. The goal: turn the fork↔upstream divergence into a +> **granular, reviewable, mergeable history** and land it, without ever +> compromising a running system. Read [`README.md`](./README.md) and +> [`conflict-map-2026-09-25.md`](./conflict-map-2026-09-25.md) first. + +## Objective + +Produce a branch that is **upstream's history with the fork's ~206 commits +cleanly replayed on top** (`reanchor/onto-upstream`), review the handful of +decisions it makes, verify the gates are green on it, and land it into +`JoshuaJewell/MetaManifold-WebUI` — either whole, or tier‑by‑tier — with a PR +that links the integration guide. + +## What already exists (do not rebuild) + +- `scripts/reanchor.sh` + `just reanchor` / `reanchor-plan` / `reanchor-manual` + — the re‑anchor engine. **Proven:** the whole fork re‑anchors with **3 + decisions, 0 residual conflicts, ~206 commits**. +- `scripts/integrate.sh` + `just integrate …` + `config/integration.toml` — + profiles (`base`/`transitional`/`full`), per‑component toggles, conflict triage. +- `.gitattributes` merge hygiene + `just merge-drivers`; `just heal`/`doctor`/ + `bootstrap`/`setup-full`/`renv-restore` for the turnkey environment. +- Measured facts in `conflict-map-2026-09-25.md`. + +## Ground truth (re‑verify, don't assume) + +```bash +git remote -v # upstream = JoshuaJewell/…, origin = hyperpolymath/… +git fetch --unshallow upstream origin # full history (the Arena clone is depth‑1!) +git merge-base origin/main upstream/main # → 7884553 "Initial commit" +``` + +The histories share the root commit and diverged at the 2nd commit (fork 221 +commits, upstream 89). A one‑shot merge = 159 conflicts; a re‑anchor = 3 decisions. + +## Steps + +1. **Plan.** `just reanchor-plan`. Record the auto/overlap/delete‑risk counts + (expect ~55 auto‑apply, ~152 overlap, delete‑risk on `pipelinesteps.txt`). +2. **Re‑anchor.** `just reanchor` (== `scripts/reanchor.sh run --branch + reanchor/onto-upstream --keep`). It runs in an isolated worktree; your current + branch is untouched. Confirm the summary: `206 commits / 0 residual / 3 + decisions`. +3. **Review the 3 decisions.** They should all be `DELETE (upstream): + pipelinesteps.txt` at commits `06d85ba`, `252299e`, `fed106b` — upstream + deleted the file; the fork's early commits touched it. Confirm accepting the + deletion is correct (it is: the file is gone upstream and superseded). +4. **Drop stray build output.** The re‑anchored tree keeps upstream's committed + `web/dist/*` (7 files) that the fork had removed. On the branch: + `git -C rm -r web/dist` and amend/commit (`chore: drop committed + web/dist build output`). These are build artefacts and must not be tracked. +5. **Confirm no upstream fix was silently reverted.** The default policy is + fork‑wins on content, so the re‑anchored tree ≈ the fork tree (+ `web/dist`). + Diff it against a *manual* three‑way merge to prove the only differences are + the documented items: + ```bash + git -C diff origin/main HEAD # expect only web/dist (+ any you fixed) + ``` + Spot‑check the high‑risk components named in `config/integration.toml` + (`real-statistics`, `ui-migration`) against upstream to be sure nothing + upstream‑authored was clobbered. If it was, resolve it by hand on the branch. +6. **Verify gates on the re‑anchored branch.** `just doctor`, then `just ci` + (frontend: typecheck + tests + bench) and, where the environment allows, + `just julia-test` and `just renv-restore`. The tree ≈ the fork's, so gates + should behave exactly as on `origin/main`. Record the verdicts. +7. **Land it.** Pick one, with Joshua (upstream owner): + - **Whole:** open a PR `hyperpolymath:reanchor/onto-upstream → + JoshuaJewell:main`; it is now a normal, granular, mergeable branch. + - **Tiered (recommended if trust is being built):** keep the branch, and + cherry‑pick / merge components in the `just integrate plan` order + (`estate-tooling → install-retry → matrix-1x1-fix → benchmarks-ci → + exact-offsets → real-statistics → ui-migration`), gating each with CI before + the next. Use `just integrate profile transitional|full` to mirror the + adoption on a running checkout. + Link [`docs/integration/README.md`](./README.md) in the PR body. + +## Guardrails (non‑negotiable) + +- **Never compromise operations.** The re‑anchored branch must pass its gates and + must not silently drop an upstream fix. The 3 decisions + step 5 cover the known + cases; anything else you find, resolve by hand and log it. +- **Branch discipline.** This session is pinned to `arena/01a0daa1-metamanifold-webui`. + Produce the `reanchor/onto-upstream` branch *via the tool at runtime* (a user + action), not by committing 206 commits onto the session branch. Do not switch + the session branch. +- **Keep the aid merge‑clean.** Any further changes to the integration tooling + stay on fork‑only / new paths (they add zero conflict surface for upstream). +- **Conventional commits** (`feat|fix|docs|…: …`, ≤72‑char subject) and run + `scripts/check-spdx.sh` + `scripts/check-format.sh` before pushing. + +## Decisions to confirm with Joshua (from the owner review, Options A–D) + +Full merge · selective/tiered cherry‑pick · dual‑track fork · handover notice. +Also: keep `codecov.yml`? (fork removed Codecov) — recommend **drop**. + +## Done when + +- [ ] `reanchor/onto-upstream` exists, `git rev-list --count upstream/main..HEAD` + ≈ 206, residual conflicts 0. +- [ ] `web/dist/*` removed on the branch. +- [ ] Diff vs `origin/main` is only the documented items (no surprise reversions). +- [ ] Gates green on the branch (recorded). +- [ ] PR opened against upstream, guide linked, adoption path agreed with Joshua. diff --git a/docs/integration/README.md b/docs/integration/README.md index 3b7d4d36..7de63763 100644 --- a/docs/integration/README.md +++ b/docs/integration/README.md @@ -10,27 +10,51 @@ conflicts" review, and without either side ever being unable to run. The measured problem and the exact file counts are in [`conflict-map-2026-09-25.md`](./conflict-map-2026-09-25.md). In one sentence: -**the two repositories share no git ancestor, so a normal merge cannot -three-way-merge anything and surfaces all 159 shared-but-different files at -once.** Nothing in this guide changes upstream's operational behaviour unless you -choose it to; the default is "behave exactly like upstream". - -## The idea in three layers - -1. **Merge hygiene** — stop git from ever producing a line-conflict inside a +**the fork and upstream share only the root commit, diverged at the second +commit, and then developed in parallel for months — so the merge base is nearly +empty and a normal merge surfaces all 159 shared‑but‑different files at once.** +Nothing in this guide changes upstream's operational behaviour unless you choose +it to; the default is "behave exactly like upstream". + +## The idea, in layers + +0. **Re‑anchor** — replay the fork's commits one‑by‑one onto upstream so git + auto‑applies the clean ones and only the genuine overlaps surface, per commit. + Measured: the whole fork re‑anchors with **3 decisions** and **0 residual + conflicts**, producing ~206 granular, reviewable commits. (`just reanchor`.) +1. **Merge hygiene** — stop git from ever producing a line‑conflict inside a lockfile or a build artefact. (`.gitattributes` + `just merge-drivers`.) 2. **Profiles & components** — turn "review 159 files" into "make ~7 ordered trust decisions", switchable from *pure upstream* to *partially transitional* to *everything*. (`config/integration.toml` + `scripts/integrate.sh` + the `just integrate …` recipes.) 3. **A staging plan** — land the safe infrastructure first, let CI go green, then - take the risk-bearing pieces one tier at a time. (`just integrate-plan`.) + take the risk‑bearing pieces one tier at a time. (`just integrate-plan`.) -Everything below is additive and default-off. A fresh clone behaves as +Everything below is additive and default‑off. A fresh clone behaves as `base` — i.e. exactly like upstream — until someone runs a command to change it. --- +## 0. Re‑anchor — from one 159‑file wall to ~206 granular commits + +This is the deepest fix and the reason the fork *looks* unmergeable. Instead of +one giant three‑way merge, re‑anchor replays the fork's history onto upstream: + +```bash +just reanchor-plan # read-only: classify every fork commit (auto / overlap / delete-risk) +just reanchor # do it; leaves branch reanchor/onto-upstream for review +just reanchor-manual # same, but stop at every conflict for hands-on resolution +``` + +It runs in an isolated worktree and never touches your current branch. The result +is upstream's history with the fork's ~206 commits cleanly on top — reviewable and +mergeable incrementally, and sharing a real base so future upstream work merges +cleanly. See `scripts/reanchor.sh` and the measured numbers in +[`conflict-map-2026-09-25.md`](./conflict-map-2026-09-25.md). + +--- + ## 1. Merge hygiene — never hand-merge a generated file The committed `.gitattributes` now marks lockfiles and build output so git will @@ -186,6 +210,7 @@ out by `just doctor` rather than silently skipped. | `just heal` | repair the environment to a known-good state | | `just bootstrap` / `just setup-full` | clone-to-runnable on a bare machine | | `just merge-drivers` | opt-in lockfile auto-resolution (local, safe) | +| `just reanchor-plan` / `just reanchor` | re‑anchor the fork onto upstream as granular commits | | `just integrate status` / `profiles` / `profile

` | the transitional dial | | `just integrate plan` / `triage` / `verify` | staging order / conflict classes / gates | | `just augment ` / `just suspend ` | flip one component | diff --git a/docs/integration/conflict-map-2026-09-25.md b/docs/integration/conflict-map-2026-09-25.md index 192bcea4..9672514f 100644 --- a/docs/integration/conflict-map-2026-09-25.md +++ b/docs/integration/conflict-map-2026-09-25.md @@ -3,96 +3,90 @@ SPDX-License-Identifier: CC-BY-SA-4.0 --> # Fork↔upstream conflict map — measured 2026-09-25 -This is the measured state that makes a naive merge of +This is the measured state behind "a naive merge of `hyperpolymath/MetaManifold-WebUI` (this fork) into -`JoshuaJewell/MetaManifold-WebUI` (upstream) explode into "hundreds of files -with conflicts". It is reproduced by `scripts/integrate.sh` and the commands in -[`README.md`](./README.md); the numbers below are exact for the trees compared. +`JoshuaJewell/MetaManifold-WebUI` (upstream) is hundreds of files with +conflicts", and how the re‑anchor collapses it. Numbers are exact for the trees +compared and reproducible with `scripts/reanchor.sh plan` and the commands below. -## Headline: the two histories are unrelated +## The histories: shared root, early divergence, long parallel development ``` -git merge-base HEAD upstream/main → (empty) +git merge-base origin/main upstream/main +→ 7884553 "Initial commit" (2026-01-30) ``` -There is **no common ancestor**. The fork is represented here as a single -squashed snapshot commit (`5a67f85`, 2026-09-25); upstream is a 90-commit -history last touched 2026-07-21. With no merge base, git cannot three-way-merge -anything, so **every shared-but-different path is a conflict**. +They **do** share the root commit, but they split at the *second* commit and then +developed in parallel for months: -## The numbers +- fork (`origin/main`): **221** commits since the base (last: `17d8c0d`) +- upstream (`upstream/main`): **89** commits since the base (last push 2026-07-21) + +Because both sides changed almost everything from a near‑empty base, the +three‑way merge base (the "Initial commit") is nearly useless — almost every file +differs on both sides — so a single `git merge` conflicts on all of them at once. + +> Note: an earlier revision of this document claimed the histories were +> *unrelated* (empty merge base). That was an artifact of a **depth‑1 shallow +> clone** hiding the shared root. With full history the root is `7884553`. The +> conflict *count* below is unchanged; only the mechanism (and therefore the fix) +> is different: this is ordinary divergence, so it is fixable by **rebase**, not +> by unrelated‑history surgery. + +## The numbers (one‑shot merge) | | count | -|---|---| +|---|---:| | Files in fork | 357 | | Files in upstream | 195 | | Shared paths (exist in both) | 186 | | **Conflicting shared paths (same path, different content)** | **159** | -| Fork-only paths (clean additions — no conflict) | 171 | +| Fork-only paths (clean additions) | 171 | | Upstream-only paths (carried in by a merge) | 9 | +| Files upstream changed since the base | 196 | ## The 159 conflicts, by area -| area | files | nature | -|---|---:|---| -| `frontend/` | 59 | TypeScript/React source — the bulk; genuine divergence from the UI work | -| `src/` | 40 | Julia backend (analysis/estimation/scaling, server, core) | -| `test/` | 30 | Julia test battery | -| `config/` | 11 | defaults / schemas / ci | -| `bench/` | 5 | benchmark harness | -| root + misc (`start.sh`, `install.sh`, `install.jl`, `precompile_exec.jl`, `README.md`, `Project.toml`, `Manifest.toml`, `.gitignore`, `.github/`, `docs/`, `data/`, `R/`, `renv/`, `scripts/`) | 14 | mixed | - -## Of those 159, most are NOT lockfiles - -Only **3** of the 159 conflicts are generated/lockfile files that should never -be hand-merged: +| area | files | +|---|---:| +| `frontend/` (TS/React) | 59 | +| `src/` (Julia) | 40 | +| `test/` | 30 | +| `config/` · `bench/` · root files | 32 | -- `Manifest.toml` -- `frontend/bun.lock` -- `renv/activate.R` +Only **3** are lockfiles (`Manifest.toml`, `frontend/bun.lock`, `renv/activate.R`) +— handled automatically by merge hygiene (see `README.md`). `renv.lock` is +byte‑identical in both. The rest is genuine source divergence that needs a +*decision*. -(`renv.lock` is **byte-identical** in both trees — `d0c9e123…` — so it is not -even a conflict.) These three, plus upstream's committed build output, are what -the `.gitattributes` merge-hygiene rules and `just merge-drivers` remove from the -conflict set automatically (see [`README.md`](./README.md#1-merge-hygiene-never-hand-merge-a-generated-file)). +## The fix: re‑anchor into a granular history (measured) -The remaining ~156 conflicts are **real source divergence** and must be resolved -by a *decision*, not a tool. That is exactly what the component/profile system is -for: it converts "review 159 files" into "make ~7 ordered trust decisions". - -## Structural divergence to be aware of - -- **Upstream committed a built frontend** under `web/dist/` (7 files: hashed JS/ - CSS bundles, `index.html`, `config.json`). These are build artefacts that - should never have been tracked. The fork removed `web/` entirely and builds - `frontend/dist/` instead (gitignored). A merge would try to re-add `web/dist/*`. - **Recommendation: drop `web/dist/**` on integration** (they are regenerated by - `just build`). -- **Upstream carries `codecov.yml`**; the fork removed Codecov (see CI: "Codecov - removed per Milestone 2"). Decide explicitly whether to keep it. -- The fork adds a second UI surface, `ui/` (Julia/Stipple), alongside the - migrated `frontend/`. - -## Upstream-only paths (9) — what a merge would newly introduce +`git rebase --onto upstream/main 7884553 origin/main` replays the fork's commits +one at a time onto upstream. Git auto‑applies every commit that doesn't collide +and surfaces only the genuine overlaps, per commit. `scripts/reanchor.sh run` +automates it (fork‑wins on content; respect upstream's deletions). **Measured +result:** ``` -codecov.yml -frontend/src/types/react-chart-editor.d.ts -web/dist/assets/ChartEditorInner-CG_IsOTj.css -web/dist/assets/ChartEditorInner-Chg4qnDh.js -web/dist/assets/index-BcmFFCWY.js -web/dist/assets/index-Cg7gdjoW.css -web/dist/assets/plotly-DWplcs0H.js -web/dist/config.json -web/dist/index.html +granular commits on top of upstream : 206 +residual conflicts : 0 +decisions the policy had to make : 3 + DELETE (upstream): pipelinesteps.txt [06d85ba Create run_cutadapt.jl.] + DELETE (upstream): pipelinesteps.txt [252299e Merge tables logic.] + DELETE (upstream): pipelinesteps.txt [fed106b Added dada2 R module.] +files the re-anchored tree keeps that the fork dropped (review) : 7 (all web/dist/* build output) ``` -Seven of the nine are `web/dist/**` build output. Reproduce this table any time -with: +So the "hundreds of files, load of conflicts" becomes: **auto‑apply 55 clean +commits, make 3 trivial decisions (all the same deleted file), review 7 stray +committed build artefacts, and land ~206 reviewable commits.** The resulting +branch also shares a real merge base with upstream, so *future* upstream changes +merge cleanly too. + +Reproduce: ```bash -git fetch upstream # JoshuaJewell/MetaManifold-WebUI as `upstream` -scripts/integrate.sh triage --from-file <(comm -12 \ - <(git ls-tree -r --name-only HEAD | sort) \ - <(git ls-tree -r --name-only upstream/main | sort)) +git fetch upstream origin +scripts/reanchor.sh plan # per-commit classification +scripts/reanchor.sh run # or: just reanchor (leaves branch reanchor/onto-upstream) ``` diff --git a/scripts/reanchor.sh b/scripts/reanchor.sh new file mode 100755 index 00000000..4b17cd1c --- /dev/null +++ b/scripts/reanchor.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Hyperpolymath engineering series +# +# reanchor.sh — turn the fork↔upstream divergence into a GRANULAR history. +# +# WHY +# --- +# The fork and upstream share the root commit ("Initial commit", 7884553) but +# diverged at the second commit and then developed in parallel (fork: ~221 +# commits; upstream: ~89). Because BOTH sides changed almost everything from a +# near-empty base, a single `git merge` surfaces ~159 conflicting files at once +# (see docs/integration/conflict-map-2026-09-25.md). +# +# Re-anchoring fixes this: `git rebase --onto upstream/main ` +# replays the fork's commits ONE AT A TIME onto upstream. Git auto-applies every +# commit that does not collide, and surfaces only the genuine overlaps — as a +# sequence of tiny, per-commit decisions instead of one wall. Measured 2026-09-25 +# with the default policy: the whole fork re-anchors with THREE decisions (all +# "upstream deleted this file"), leaving 0 residual conflicts. +# +# The result is a granular branch — upstream's history, then the fork's ~206 +# commits on top — that (a) the maintainer can review/merge incrementally or all +# at once, and (b) shares a real merge base with upstream, so FUTURE upstream +# changes merge cleanly too. +# +# Doctrine: fail loudly; no hidden dependencies (bash + git + awk); never touch +# the caller's working tree or current branch (all work happens in a worktree). +# +# Usage: scripts/reanchor.sh [options] +# plan Read-only: classify each fork commit (auto / overlap / delete-risk) +# run [options] Perform the re-anchor in a worktree (see options) +# status Report an in-progress re-anchor +# +# run options: +# --policy theirs|manual theirs (default): auto-resolve content in the fork's +# favour and respect upstream's deletions — the whole +# fork lands with a handful of logged decisions. +# manual: stop at every conflict for hands-on review. +# --branch NAME Create/point branch NAME at the result (default: none, detached) +# --worktree PATH Where to build (default: $TMPDIR/metamanifold-reanchor) +# --keep Leave the worktree in place afterwards (default: remove on success) +# +# Env: UPSTREAM_REMOTE (default upstream), FORK_REMOTE (default origin), +# UPSTREAM_REF (default main), FORK_REF (default main). + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +UPSTREAM_REMOTE="${UPSTREAM_REMOTE:-upstream}" +FORK_REMOTE="${FORK_REMOTE:-origin}" +UPSTREAM_REF="${UPSTREAM_REF:-main}" +FORK_REF="${FORK_REF:-main}" +UPSTREAM="$UPSTREAM_REMOTE/$UPSTREAM_REF" +FORK="$FORK_REMOTE/$FORK_REF" + +die() { printf 'reanchor: %s\n' "$*" >&2; exit 1; } + +require_ref() { git rev-parse -q --verify "$1" >/dev/null 2>&1 || die "ref not found: $1 (fetch it first, e.g. 'git fetch $UPSTREAM_REMOTE')"; } + +find_base() { + local b; b="$(git merge-base "$UPSTREAM" "$FORK" 2>/dev/null || true)" + [ -n "$b" ] || die "no common ancestor between $UPSTREAM and $FORK — re-anchoring is not possible; use the profile/triage path instead (just integrate)" + printf '%s\n' "$b" +} + +preflight() { + require_ref "$UPSTREAM" + require_ref "$FORK" + # Only uncommitted TRACKED changes block; the rebase runs in an isolated + # worktree at the fork ref, so untracked files are irrelevant to it. + [ -z "$(git status --porcelain --untracked-files=no)" ] || die "uncommitted tracked changes present — commit or stash before re-anchoring" + local gd; gd="$(git rev-parse --git-dir)" + if [ -d "$gd/rebase-merge" ] || [ -d "$gd/rebase-apply" ]; then + die "a rebase is already in progress — finish or 'git rebase --abort' first" + fi +} + +# --------------------------------------------------------------------------- # +cmd_plan() { + require_ref "$UPSTREAM"; require_ref "$FORK" + local base; base="$(find_base)" + printf 'Fork : %s (%s)\n' "$FORK" "$(git rev-parse --short "$FORK")" + printf 'Upstream : %s (%s)\n' "$UPSTREAM" "$(git rev-parse --short "$UPSTREAM")" + printf 'Merge base : %s %s\n' "$(git rev-parse --short "$base")" "$(git log -1 --format='%s' "$base")" + printf 'Fork commits since base : %s\n' "$(git rev-list --count "$base..$FORK")" + printf 'Upstream commits since base : %s\n' "$(git rev-list --count "$base..$UPSTREAM")" + echo "" + + git diff --name-only "$base" "$UPSTREAM" | sort -u > /tmp/.reanchor_up.$$ 2>/dev/null || true + local upchanged=/tmp/.reanchor_up.$$ + printf 'Upstream changed %s file(s) since base.\n\n' "$(wc -l < "$upchanged" | xargs)" + + # Files upstream DELETED but the fork still touches → modify/delete hotspots. + git diff --diff-filter=D --name-only "$base" "$UPSTREAM" | sort -u > /tmp/.reanchor_del.$$ 2>/dev/null || true + local updeleted=/tmp/.reanchor_del.$$ + + local clean=0 overlap=0 c files ov tot + printf '%-8s %-6s %-9s %s\n' "CLASS" "OVLAP" "RISK" "COMMIT" + while IFS= read -r c; do + [ -n "$c" ] || continue + files="$(git show --name-only --format= "$c" 2>/dev/null || true)" + [ -n "$files" ] || continue + ov="$(comm -12 <(printf '%s\n' "$files" | sort -u) "$upchanged" | wc -l | xargs)" + tot="$(printf '%s\n' "$files" | grep -c . || true)" + if [ "$ov" -eq 0 ]; then + clean=$((clean+1)) + else + overlap=$((overlap+1)) + # delete-risk: does this commit touch a file upstream deleted? + local delrisk="-" + comm -12 <(printf '%s\n' "$files" | sort -u) "$updeleted" | grep -q . && delrisk="DELETE" + printf '%-8s %-6s %-9s %s\n' "overlap" "$ov/$tot" "$delrisk" "$(git log -1 --format='%h %s' "$c")" + fi + done < <(git rev-list "$base..$FORK") + + echo "" + printf 'SUMMARY: %s commit(s) auto-apply (touch only fork-owned files); %s overlap upstream.\n' "$clean" "$overlap" + printf 'A one-shot merge would show all overlaps at once; a re-anchor resolves them per commit.\n' + printf 'Next: scripts/reanchor.sh run (automated, fork-wins, logged decisions)\n' + rm -f "$upchanged" "$updeleted" +} + +# --------------------------------------------------------------------------- # +cmd_run() { + local policy="theirs" branch="" wt="${TMPDIR:-/tmp}/metamanifold-reanchor" keep=0 + while [ $# -gt 0 ]; do + case "$1" in + --policy) policy="${2:?--policy needs a value}"; shift 2;; + --branch) branch="${2:?--branch needs a name}"; shift 2;; + --worktree) wt="${2:?--worktree needs a path}"; shift 2;; + --keep) keep=1; shift;; + *) die "run: unknown option '$1'";; + esac + done + case "$policy" in theirs|manual) ;; *) die "--policy must be 'theirs' or 'manual'";; esac + + preflight + local base; base="$(find_base)" + + rm -rf "$wt" + git worktree add --detach "$wt" "$FORK" >/dev/null 2>&1 || die "could not create worktree at $wt" + # shellcheck disable=SC2064 + trap "cd '$ROOT'; git worktree remove --force '$wt' >/dev/null 2>&1 || true" EXIT + + cd "$wt" + export GIT_EDITOR=true + printf 'reanchor: rebasing %s onto %s (policy=%s, base=%s)\n' "$FORK" "$UPSTREAM" "$policy" "$(git rev-parse --short "$base")" + local decisions=/tmp/.reanchor_decisions.$$; : > "$decisions" + git -c core.editor=true rebase --onto "$UPSTREAM" "$base" -X theirs >/dev/null 2>&1 || true + + local gd; gd="$(git rev-parse --git-dir)" + local iters=0 + while [ -d "$gd/rebase-merge" ] || [ -d "$gd/rebase-apply" ]; do + iters=$((iters+1)) + [ "$iters" -gt 400 ] && { printf 'reanchor: SAFETY CAP hit (%s iterations) — leaving the rebase in place for inspection.\n' "$iters" >&2; return 3; } + local cf; cf="$(git diff --name-only --diff-filter=U)" + if [ -z "$cf" ]; then git -c core.editor=true rebase --continue >/dev/null 2>&1 || true; continue; fi + local cur; cur="$(git log -1 --format='%h %s' REBASE_HEAD 2>/dev/null || echo '?')" + if [ "$policy" = "manual" ]; then + printf '\nreanchor: STOPPED for manual review at %s\n' "$cur" >&2 + printf 'Conflicted files:\n%s\n' "$cf" >&2 + printf 'Resolve, then: git -C %s rebase --continue (or: git -C %s rebase --abort)\n' "$wt" "$wt" >&2 + return 2 + fi + local f + while IFS= read -r f; do + [ -n "$f" ] || continue + if git cat-file -e "$UPSTREAM":"$f" 2>/dev/null; then + if git checkout --theirs -- "$f" 2>/dev/null && git add -- "$f" 2>/dev/null; then + printf 'CONTENT fork-wins : %s [%s]\n' "$f" "$cur" >> "$decisions" + else + git rm -f -- "$f" >/dev/null 2>&1 && printf 'DELETE (fork) : %s [%s]\n' "$f" "$cur" >> "$decisions" + fi + else + git rm -f -- "$f" >/dev/null 2>&1 && printf 'DELETE (upstream): %s [%s]\n' "$f" "$cur" >> "$decisions" + fi + done <<< "$cf" + git -c core.editor=true rebase --continue >/dev/null 2>&1 || true + done + + local ahead; ahead="$(git rev-list --count HEAD --not "$UPSTREAM")" + local residual; residual="$(git diff --name-only --diff-filter=U | wc -l | xargs)" + printf '\nreanchor: COMPLETE.\n' + printf ' granular commits on top of upstream : %s\n' "$ahead" + printf ' residual conflicts : %s\n' "$residual" + printf ' decisions the policy made : %s\n' "$(grep -c . "$decisions" || echo 0)" + if [ -s "$decisions" ]; then printf '\n --- decisions ---\n'; sed 's/^/ /' "$decisions"; fi + local drift; drift="$(git diff --name-only "$FORK" HEAD | wc -l | xargs)" + printf '\n files the re-anchored tree keeps that the fork dropped (review these): %s\n' "$drift" + git diff --name-only "$FORK" HEAD | sed 's/^/ /' | head -20 + + if [ -n "$branch" ]; then + git branch -f "$branch" HEAD >/dev/null 2>&1 && printf '\n branch %s -> %s\n' "$branch" "$(git rev-parse --short HEAD)" + fi + printf '\nNext:\n' + printf ' review : git -C %s log --oneline %s..HEAD | head\n' "$wt" "$UPSTREAM" + printf ' stray build output (e.g. web/dist) is upstream-committed; drop it: git -C %s rm -r web/dist\n' "$wt" + printf ' land : merge this granular branch into upstream, or cherry-pick tiers.\n' + + rm -f "$decisions" + if [ "$keep" -eq 1 ]; then trap - EXIT; printf '\n(worktree kept at %s)\n' "$wt"; fi + return 0 +} + +cmd_status() { + local gd; gd="$(git rev-parse --git-dir)" + if [ -d "$gd/rebase-merge" ] || [ -d "$gd/rebase-apply" ]; then + echo "reanchor: a rebase is in progress." + git status | sed -n '1,8p' + else + echo "reanchor: no re-anchor in progress." + fi +} + +usage() { sed -n '2,44p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; } + +main() { + local cmd="${1:-}"; shift || true + case "$cmd" in + plan) cmd_plan "$@";; + run) cmd_run "$@";; + status) cmd_status "$@";; + ""|help|-h|--help) usage;; + *) die "unknown command '$cmd' (try: plan|run|status)";; + esac +} + +main "$@"