diff --git a/.beads/config.yaml b/.beads/config.yaml new file mode 100644 index 0000000000..acdf367371 --- /dev/null +++ b/.beads/config.yaml @@ -0,0 +1,16 @@ +issue_prefix: gc +issue-prefix: gc +dolt.auto-start: false +dolt.local-only: true +dolt.auto-push: false +no-push: true +export.auto: false +gc.endpoint_origin: inherited_city +gc.endpoint_status: verified +types.custom: molecule,convoy,message,event,gate,merge-request,agent,role,rig,session,spec,convergence,step +dolt: + disable-event-flush: true +backup.enabled: false +dolt.auto-commit: "batch" +import.auto: false +sync.remote: "git+ssh://git@github.com/zookanalytics/gascity.git" diff --git a/.beads/identity.toml b/.beads/identity.toml new file mode 100644 index 0000000000..c7374e83ed --- /dev/null +++ b/.beads/identity.toml @@ -0,0 +1,5 @@ +# .beads/identity.toml — canonical, git-tracked. +# Edited only at scope creation or by deliberate human/`gc` migration. + +[project] +id = "gc-local-76b302ae41577800e6d45fe488752941" diff --git a/.beads/metadata.json b/.beads/metadata.json new file mode 100644 index 0000000000..65436c2a95 --- /dev/null +++ b/.beads/metadata.json @@ -0,0 +1,7 @@ +{ + "backend": "dolt", + "database": "dolt", + "dolt_database": "gc", + "dolt_mode": "server", + "project_id": "gc-local-76b302ae41577800e6d45fe488752941" +} diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 301684a700..dfe38e6450 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -1,10 +1,38 @@ #!/usr/bin/env bash set -euo pipefail +# Skip the two heaviest pre-commit steps (test-fast-parallel and +# dashboard-check + dashboard-smoke) by default in agent contexts. +# Polecats commit at agent pace; running the full per-iteration validation +# N times concurrent saturates FS/CPU/Dolt (see gc-53c8k4 for the 2026-05-24 +# supervisor SIGTERM cascade). Refinery's pre-publish review gate and +# GitHub Actions CI run the full validation before the human-visible +# artifact (PR), which is the gate that actually matters. +# +# Override either direction explicitly: +# GC_PRECOMMIT_SKIP_HEAVY=1 → always skip (regardless of GC_AGENT) +# GC_PRECOMMIT_SKIP_HEAVY=0 → always run (force full validation in agent) +# unset → skip iff GC_AGENT is set +if [ "${GC_PRECOMMIT_SKIP_HEAVY:-${GC_AGENT:+1}}" = "1" ]; then + SKIP_HEAVY=1 +else + SKIP_HEAVY=0 +fi + staged_go_files=$(git diff --cached --name-only --diff-filter=ACM -- '*.go' || true) staged_web_src=$(git diff --cached --name-only --diff-filter=ACM -- 'internal/api/dashboardspa/web/shared/src/' 'internal/api/dashboardspa/web/frontend/src/' 'internal/api/dashboardspa/web/frontend/index.html' 'internal/api/dashboardspa/web/frontend/public/' 'internal/api/dashboardspa/web/package.json' 'internal/api/dashboardspa/web/shared/package.json' 'internal/api/dashboardspa/web/frontend/package.json' 'internal/api/dashboardspa/web/frontend/vite.config.ts' 'internal/api/dashboardspa/web/frontend/tsconfig.json' 'internal/api/dashboardspa/web/shared/tsconfig.json' || true) staged_docs=$(git diff --cached --name-only --diff-filter=ACM -- '*.md' 'docs/**' 'engdocs/**' 'plans/**' 'specs/**' 'AGENTS.md' 'CONTRIBUTING.md' 'README.md' 'TESTING.md' || true) +# When mid-rebase, skip the heavy gates (test/vet/codegen). The +# formula's post-rebase `test` step is the safety net that runs the +# full suite once on the final state. Per-commit testing during a +# rebase that replays 30+ commits is purely redundant. +git_dir=$(git rev-parse --git-dir) +if [ -d "$git_dir/rebase-merge" ] || [ -d "$git_dir/rebase-apply" ]; then + echo "pre-commit: rebase in progress — skipping vet/test/codegen (post-rebase test step is the gate)" >&2 + exit 0 +fi + if [ -z "$staged_go_files" ] && [ -z "$staged_web_src" ] && [ -z "$staged_docs" ]; then exit 0 fi @@ -51,13 +79,22 @@ fi if command -v npm >/dev/null 2>&1; then spec_changed=$(git diff --cached --name-only --diff-filter=ACM -- 'internal/api/openapi.json' || true) if [ -n "$spec_changed" ] || [ -n "$staged_web_src" ]; then - # Typecheck BEFORE build: vite's build transpiles TS to JS and - # silently ignores type errors. The Makefile target also runs the - # Vitest suite, builds dist/, and smoke-runs the compiled SPA via - # Vite preview so a bundle that builds but won't serve is caught - # before CI. - make dashboard-check dashboard-smoke - git add internal/api/dashboardspa/dist + if [ "$SKIP_HEAVY" != "1" ]; then + # Typecheck BEFORE build: vite's build transpiles TS to JS and + # silently ignores type errors. The Makefile target also runs the + # Vitest suite, builds dist/, and smoke-runs the compiled SPA via + # Vite preview so a bundle that builds but won't serve is caught + # before CI. + make dashboard-check dashboard-smoke + git add internal/api/dashboardspa/dist + fi + # SKIP_HEAVY=1: we did not rebuild the bundle, so we must not stage + # dist/ — staging a stale bundle would ship drift in the polecat's + # commits. The refinery + CI (dashboard-ci + the `dashboard` Actions + # job) catch any drift at PR time and bounce the bead back via + # rejection_reason; the next polecat picks it up and either fixes the + # regen explicitly or commits with GC_PRECOMMIT_SKIP_HEAVY=0 to run + # the full block. fi else echo "warning: npm not on PATH — skipped dashboard SPA typecheck + rebuild. CI will enforce this." >&2 diff --git a/Makefile b/Makefile index a3cea19f38..58ea77daf3 100644 --- a/Makefile +++ b/Makefile @@ -332,11 +332,33 @@ vet: ## city-wide (ga-w2kh1r). Do not add them. For a bare `go test` that bypasses ## this wrapper, internal/testenv scrubs these vars at test-binary init in every ## covered package (enforced by TestRequiresDedicatedTestenvImportFile). +## +## TMPDIR defaults to /var/tmp, not /tmp. `make check` runs `go test -p=4 +## ./...`; the parallel link phase of large CGO/ICU test binaries can peak +## past a small tmpfs-backed /tmp and fail with "No space left on device" +## mid-link — a spurious rebase/refinery preflight failure (gc-v2z1p). +## /var/tmp is conventionally a persistent (non-tmpfs) disk with more room. +## Every make test target flows through this wrapper, so scripts it invokes +## (test-*-shard, test-local-parallel) inherit the same TMPDIR. Export +## TMPDIR to override (e.g. TMPDIR=/tmp make check). GOPATH_VAL := $(shell go env GOPATH) GOCACHE_VAL := $(shell go env GOCACHE) GOMODCACHE_VAL := $(shell go env GOMODCACHE) GOTMPDIR_VAL := $(shell go env GOTMPDIR) GOROOT_VAL := $(shell go env GOROOT) + +## ISOLATED_GITCONFIG: seeded global git config for tests. HOME is allowlisted +## below but SSH_AUTH_SOCK is not, so a host ~/.gitconfig with commit.gpgsign=true +## + gpg.format=ssh makes every test that execs `git commit` fail with "Couldn't +## get agent socket?". Pointing GIT_CONFIG_GLOBAL here neutralizes the host config +## for every test binary at once, so individual tests need no per-test opt-in. +## Must be a real writable file, never /dev/null: ensure_beads_role runs +## `git config --global beads.role maintainer` and cannot lock /dev/null. Tests +## that WRITE global config still call testutil.IsolatedGitConfig for a per-test +## file, so writes don't leak through this shared one. Keep contents in sync with +## internal/testutil/gitconfig.go (isolatedGitConfigContents). +ISOLATED_GITCONFIG := $(shell d="$${TMPDIR:-/var/tmp}/gascity-testcfg"; mkdir -p "$$d" && printf '[user]\n\tname = Gas City Test\n\temail = gascity-test@example.invalid\n[commit]\n\tgpgsign = false\n[tag]\n\tgpgsign = false\n[init]\n\tdefaultBranch = main\n' > "$$d/gitconfig" && printf '%s' "$$d/gitconfig") + TEST_ENV = env -i \ PATH="$$PATH" \ HOME="$$HOME" \ @@ -344,7 +366,9 @@ TEST_ENV = env -i \ LOGNAME="$$LOGNAME" \ SHELL="$$SHELL" \ LANG="$$LANG" \ - TMPDIR="$${TMPDIR:-/tmp}" \ + TMPDIR="$${TMPDIR:-/var/tmp}" \ + GIT_CONFIG_GLOBAL="$(ISOLATED_GITCONFIG)" \ + GIT_CONFIG_SYSTEM=/dev/null \ OBSERVABLE_TEST_LOG="$${OBSERVABLE_TEST_LOG-}" \ OBSERVABLE_FAILURE_LINES="$${OBSERVABLE_FAILURE_LINES-}" \ GC_TEST_NO_SLICE="$${GC_TEST_NO_SLICE-}" \ diff --git a/TESTING.md b/TESTING.md index 8fef4a6717..c5446806e6 100644 --- a/TESTING.md +++ b/TESTING.md @@ -129,30 +129,30 @@ all-source audit while staying outside untagged and Small debt. | Ledger kind | Source scope | Resource baseline | Tracking owner | Invariant / resource owner | Migration | Expiry | | --- | --- | --- | --- | --- | --- | --- | -| Audit baseline | all tracked test source | fixed_sleep: 441 calls / 158 files (historical regex census: 447 / 157) | ga-80po0c.2 | tracked test source totals remain visible as audit evidence; ga-80po0c.2 owns this point-in-time source census | P0.4a | 2026-10-01 | -| Audit baseline | all tracked test source | subprocess: 528 calls / 154 files (historical regex census: 495 / 135) | ga-80po0c.2 | tracked test source totals remain visible as audit evidence; ga-80po0c.2 owns this point-in-time source census | P0.4a | 2026-10-01 | +| Audit baseline | all tracked test source | fixed_sleep: 442 calls / 159 files (historical regex census: 447 / 157) | ga-80po0c.2 | tracked test source totals remain visible as audit evidence; ga-80po0c.2 owns this point-in-time source census | P0.4a | 2026-10-01 | +| Audit baseline | all tracked test source | subprocess: 535 calls / 156 files (historical regex census: 495 / 135) | ga-80po0c.2 | tracked test source totals remain visible as audit evidence; ga-80po0c.2 owns this point-in-time source census | P0.4a | 2026-10-01 | | Medium owner | `cmd/gc` package `main` | TestMain: environment | ga-80po0c.2.1 | cmd/gc TestMain is the checked package-level Medium owner; only environment calls lexically inside TestMain leave Small debt | P0.4b | 2026-10-01 | | Medium owner | `internal/api` package `api` | TestEveryEmittedErrorCodeIsRegistered: subprocess | ga-80po0c.2.1 | internal/api tracked-source error URN guard is a checked Medium owner; only the git ls-files call lexically inside TestEveryEmittedErrorCodeIsRegistered leaves Small debt | P0.4b | 2026-10-01 | | Medium owner | `scripts` package `scripts_test` | TestProviderOverridesAndSuiteContractsCrossMakeIsolation: subprocess | ga-80po0c.2.1 | Make/provider and suite-contract proof is a checked Medium owner; the six isolated Make invocations are confined to TestProviderOverridesAndSuiteContractsCrossMakeIsolation | P0.1 | 2026-10-01 | -| Small debt ratchet | `cmd/gc` untagged test source | cwd: 284 calls / 43 files | ga-80po0c.2.1 | untagged Small cmd/gc cwd call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners restore or eliminate every cwd mutation | D5/D6 | 2026-10-01 | -| Small debt ratchet | `cmd/gc` untagged test source | environment: 4326 calls / 200 files (historical regex census: 4339 / 199) | ga-80po0c.2.1 | untagged Small cmd/gc environment call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners restore or eliminate every process-environment mutation | D5/D6/E6 | 2026-10-01 | +| Small debt ratchet | `cmd/gc` untagged test source | cwd: 295 calls / 44 files (historical regex census: 284 / 43) | ga-80po0c.2.1 | untagged Small cmd/gc cwd call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners restore or eliminate every cwd mutation | D5/D6 | 2026-10-01 | +| Small debt ratchet | `cmd/gc` untagged test source | environment: 4371 calls / 203 files (historical regex census: 4339 / 199) | ga-80po0c.2.1 | untagged Small cmd/gc environment call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners restore or eliminate every process-environment mutation | D5/D6/E6 | 2026-10-01 | | Small debt ratchet | `cmd/gc` untagged test source | slow_process_gate: 74 calls / 25 files (historical regex census: 75 / 25) | ga-80po0c.2.1 | untagged Small cmd/gc slow-process marker totals cannot grow; reductions must lower this baseline; each non-Medium marked caller retains an explicit process-suite migration owner | D5/D6/E6 | 2026-10-01 | -| Small debt ratchet | all untagged test source | fixed_sleep: 287 calls / 113 files | ga-80po0c.2.1 | untagged Small fixed-sleep call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners replace elapsed wall time with lifecycle signals | W1-W5 | 2026-10-01 | +| Small debt ratchet | all untagged test source | fixed_sleep: 288 calls / 114 files (historical regex census: 287 / 113) | ga-80po0c.2.1 | untagged Small fixed-sleep call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners replace elapsed wall time with lifecycle signals | W1-W5 | 2026-10-01 | | Small debt ratchet | all untagged test source | http_test_server: 300 calls / 66 files | ga-80po0c.2.2 | untagged Small HTTP test server call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners move server-backed tests to exact Medium ownership or replace the listener | P0.4c | 2026-10-01 | | Small debt ratchet | all untagged test source | net_listen: 92 calls / 34 files | ga-80po0c.2.2 | untagged Small net.Listen call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners move listener-backed tests to exact Medium ownership or replace the listener | P0.4c | 2026-10-01 | | Small debt ratchet | all untagged test source | net_listen_config: 1 calls / 1 files | ga-80po0c.2.2 | untagged Small net.ListenConfig.Listen call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners move ListenConfig-backed tests to exact Medium ownership or replace the listener | P0.4c | 2026-10-01 | | Small debt ratchet | all untagged test source | net_listen_unixgram: 3 calls / 2 files | ga-80po0c.2.2 | untagged Small net.ListenUnixgram call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners move Unix datagram listener-backed tests to exact Medium ownership or replace the listener | P0.4c | 2026-10-01 | -| Small debt ratchet | all untagged test source | subprocess: 399 calls / 107 files (historical regex census: 394 / 105) | ga-80po0c.2.1 | untagged Small subprocess call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners remove or replace each process call site | D1/D2/D5/D6/E6 | 2026-10-01 | +| Small debt ratchet | all untagged test source | subprocess: 406 calls / 109 files (historical regex census: 394 / 105) | ga-80po0c.2.1 | untagged Small subprocess call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners remove or replace each process call site | D1/D2/D5/D6/E6 | 2026-10-01 | | Small debt ratchet | all untagged test source | syscall_listen: 1 calls / 1 files | ga-80po0c.2.2 | untagged Small syscall.Listen call/file totals cannot grow; reductions must lower this baseline; non-Medium lexical owners move syscall-backed listener tests to exact Medium ownership or replace the listener | P0.4c | 2026-10-01 | -| Source debt ratchet | `cmd/gc` untagged test source | cwd: 284 calls / 43 files (historical regex census: 98 / 13) | ga-80po0c.2.3 | untagged cmd/gc cwd call/file totals cannot grow; reductions must lower this baseline; cmd/gc callers restore or eliminate every recognized cwd mutation | D5/D6 | 2026-10-01 | -| Source debt ratchet | `cmd/gc` untagged test source | environment: 4332 calls / 200 files (historical regex census: 3960 / 184) | ga-80po0c.2.3 | untagged cmd/gc environment call/file totals cannot grow; reductions must lower this baseline; cmd/gc callers restore or eliminate every recognized process-environment mutation | D5/D6/E6 | 2026-10-01 | +| Source debt ratchet | `cmd/gc` untagged test source | cwd: 295 calls / 44 files (historical regex census: 98 / 13) | ga-80po0c.2.3 | untagged cmd/gc cwd call/file totals cannot grow; reductions must lower this baseline; cmd/gc callers restore or eliminate every recognized cwd mutation | D5/D6 | 2026-10-01 | +| Source debt ratchet | `cmd/gc` untagged test source | environment: 4379 calls / 203 files (historical regex census: 3960 / 184) | ga-80po0c.2.3 | untagged cmd/gc environment call/file totals cannot grow; reductions must lower this baseline; cmd/gc callers restore or eliminate every recognized process-environment mutation | D5/D6/E6 | 2026-10-01 | | Source debt ratchet | `cmd/gc` untagged test source | slow_process_gate: 74 calls / 25 files (historical regex census: 78 / 27) | ga-80po0c.2.3 | untagged cmd/gc slow-process marker totals cannot grow; reductions must lower this baseline; the helper definition and every marked caller retain an explicit process-suite migration owner | D5/D6/E6 | 2026-10-01 | -| Source debt ratchet | all untagged test source | fixed_sleep: 287 calls / 113 files (historical regex census: 295 / 114) | ga-80po0c.2 | untagged fixed-sleep call/file totals cannot grow; reductions must lower this baseline; each owning test replaces elapsed wall time with its lifecycle signal | W1-W5 | 2026-10-01 | +| Source debt ratchet | all untagged test source | fixed_sleep: 288 calls / 114 files (historical regex census: 295 / 114) | ga-80po0c.2 | untagged fixed-sleep call/file totals cannot grow; reductions must lower this baseline; each owning test replaces elapsed wall time with its lifecycle signal | W1-W5 | 2026-10-01 | | Source debt ratchet | all untagged test source | http_test_server: 300 calls / 66 files (historical regex census: 255 / 56) | ga-80po0c.2.2 | untagged HTTP test server call/file totals cannot grow; reductions must lower this baseline; each owning test closes its loopback server and removes duplicate server-backed coverage | P0.4c | 2026-10-01 | | Source debt ratchet | all untagged test source | net_listen: 92 calls / 34 files | ga-80po0c.2.2 | untagged net.Listen call/file totals cannot grow; reductions must lower this baseline; each owning test closes its listener and removes duplicate listener-backed coverage | P0.4c | 2026-10-01 | | Source debt ratchet | all untagged test source | net_listen_config: 1 calls / 1 files | ga-80po0c.2.2 | untagged net.ListenConfig.Listen call/file totals cannot grow; reductions must lower this baseline; each owning test closes its configured listener and removes duplicate listener-backed coverage | P0.4c | 2026-10-01 | | Source debt ratchet | all untagged test source | net_listen_unixgram: 3 calls / 2 files | ga-80po0c.2.2 | untagged net.ListenUnixgram call/file totals cannot grow; reductions must lower this baseline; each owning test closes its Unix datagram listener and removes duplicate listener-backed coverage | P0.4c | 2026-10-01 | -| Source debt ratchet | all untagged test source | subprocess: 401 calls / 108 files (historical regex census: 380 / 98) | ga-80po0c.2 | untagged subprocess call/file totals cannot grow; reductions must lower this baseline; each process-owning test removes or replaces its source call site | D1/D2/D5/D6/E6 | 2026-10-01 | +| Source debt ratchet | all untagged test source | subprocess: 408 calls / 110 files (historical regex census: 380 / 98) | ga-80po0c.2 | untagged subprocess call/file totals cannot grow; reductions must lower this baseline; each process-owning test removes or replaces its source call site | D1/D2/D5/D6/E6 | 2026-10-01 | | Source debt ratchet | all untagged test source | syscall_listen: 1 calls / 1 files | ga-80po0c.2.2 | untagged syscall.Listen call/file totals cannot grow; reductions must lower this baseline; each owning test closes its listening file descriptor and removes duplicate listener-backed coverage | P0.4c | 2026-10-01 | | Reviewed hermetic body | Effective runnable size | Medium reason | Retained real composition owner | diff --git a/cmd/gc/api_state.go b/cmd/gc/api_state.go index 874feaccba..8eec7cd8a1 100644 --- a/cmd/gc/api_state.go +++ b/cmd/gc/api_state.go @@ -77,6 +77,7 @@ type controllerState struct { ct crashTracker // nil if crash tracking disabled pokeCh chan struct{} // nil when poke is not available; triggers immediate reconciler tick configDirty *atomic.Bool // optional dirty flag shared with the reconciler reload path + demandDirty *atomic.Bool // optional one-shot flag shared with the reconciler; forces a pool-demand rebuild services workspacesvc.Registry extmsgSvc *extmsg.Services adapterReg *extmsg.AdapterRegistry @@ -522,10 +523,43 @@ func (cs *controllerState) startMaintenanceLoop(ctx context.Context) { DiskMinFreeBytes: doltDiskMinFreeBytes(), DiskWarnFreeBytes: doltDiskWarnFreeBytes(), } - active := deps.OpenDoltOps != nil && deps.OpenDoltBackup != nil + // Wire CALL DOLT_GC() against the managed Dolt server. The factory + // resolves the port lazily — each maintenance cycle reads the + // currently-published port — so the loop recovers automatically if Dolt + // had not finished publishing when the controller started. An + // unresolvable port or open failure surfaces as a stage="gc" failure for + // that cycle (classified the same as "Dolt unreachable") and is retried + // on the next tick. managedDoltMaintenanceOps iterates every managed user + // database because DOLT_GC compacts only the session's selected database. + deps.OpenDoltOps = func(_ context.Context) (supervisor.DoltOps, error) { + port := currentResolvableManagedDoltPort(cityPath) + if port == "" { + return nil, fmt.Errorf("managed Dolt port not resolvable for store maintenance") + } + ops, err := newManagedDoltMaintenanceOps(managedDoltConnectHost(""), port, "root") + if err != nil { + return nil, err + } + return ops, nil + } // Always log the loop's startup so operators can confirm initialization // (and its mode) from the supervisor log, not just the observe-only case. + // active gates on the GC opener, not upstream's both-openers test: this + // fork wires DOLT_GC (OpenDoltOps) but intentionally leaves OpenDoltBackup + // unwired (multi-database snapshot tracked in gc-thnww), so a both-wired + // test would mislabel a GC-active loop "observe-only". + active := deps.OpenDoltOps != nil fmt.Fprintln(os.Stderr, maintenanceStartupLine(cfg.Maintenance.Dolt.IntervalOrDefault(), active)) //nolint:errcheck // best-effort stderr + // OpenDoltBackup (pre-GC snapshot) stays unwired: internal/supervisor's + // snapshot layout assumes a single Dolt store, but this fork runs a + // multi-database managed server (.beads/dolt/). Wiring a single-dir + // backup here would fail runSnapshot every cycle and — because + // executeCycleLocked runs the snapshot before the gc — block DOLT_GC + // entirely. Multi-database snapshotting is tracked in gc-thnww. CALL + // DOLT_GC() is online and safe to run without it. + if deps.OpenDoltBackup == nil { + fmt.Fprintln(os.Stderr, "store-maintenance: DOLT_GC enabled; pre-GC snapshot not wired (multi-database snapshot tracked in gc-thnww)") + } loop := supervisor.NewStoreMaintenanceLoop(deps) // Retain the handle so the API layer can expose // /v0/city/{city}/maintenance/* (status reads + manual trigger) @@ -566,19 +600,49 @@ func (cs *controllerState) applyBeadEventToStores(evt events.Event) { } cs.mu.RUnlock() - for _, store := range stores { - if cached, ok := store.(*beads.CachingStore); ok { - cached.ApplyEvent(evt.Type, evt.Payload) - } - } + // Skip events we emitted ourselves (reconciler-detected changes): + // don't re-apply them to the caching stores, and don't poke. The + // originating CachingStore already updated its own cache during + // reconcile; redelivering through ApplyEvent risks a self-feedback + // loop because mergeCacheEventPatch is field-aware (driven by which + // JSON keys are present) while notifyChange marshals the full bead + // with omitempty — fields that became empty are dropped from the + // payload and the merge silently keeps the prior cache value, so the + // next reconcile cycle still sees a diff and re-fires. Other stores + // filter by ownsBeadID, so the only meaningful delivery was a self- + // echo back to the originating store anyway. Bead-close autoclose + // below still runs regardless of actor: a close first observed via + // reconcile must still cascade convoy/wisp/molecule autoclose. if evt.Actor != "cache-reconcile" { - cs.Poke() + for _, store := range stores { + if cached, ok := store.(*beads.CachingStore); ok { + cached.ApplyEvent(evt.Type, evt.Payload) + } + } + // Don't poke on the controller's own order-tracking writes. + if !isOrderTrackingBeadEvent(evt.Payload) { + cs.Poke() + } } if evt.Type == events.BeadClosed && evt.Subject != "" && len(stores) > 0 { cs.runBeadCloseAutoclose(evt.Subject, stores[0], storeRef) } } +// isOrderTrackingBeadEvent reports whether a bead-event payload describes a +// controller-authored order-tracking bead (label labelOrderTracking, stamped by +// dispatchOrders in order_dispatch.go). +func isOrderTrackingBeadEvent(payload json.RawMessage) bool { + if len(payload) == 0 { + return false + } + var p api.BeadEventPayload + if err := json.Unmarshal(payload, &p); err != nil { + return false + } + return hasBeadLabel(p.Bead.Labels, labelOrderTracking) +} + // autocloseStoreRefLocked returns the storeRef string for the store that owns // beadID. Called under cs.mu read lock. func (cs *controllerState) autocloseStoreRefLocked(beadID string) string { @@ -2600,6 +2664,17 @@ func (cs *controllerState) Poke() { } } +// PokeDemand forces a pool-demand rebuild on the next tick and signals one +// immediately. It is the demand-aware Poke: a plain Poke leaves the cached +// demand snapshot in place, so callers that change pool demand without touching +// a session bead use this instead. +func (cs *controllerState) PokeDemand() { + if cs.demandDirty != nil { + cs.demandDirty.Store(true) + } + cs.Poke() +} + // WaitForSessionCommandable waits until the controller has reconciled an async // session create into a lifecycle state that can accept normal commands. func (cs *controllerState) WaitForSessionCommandable(ctx context.Context, sessionID string) (session.Info, error) { diff --git a/cmd/gc/api_state_test.go b/cmd/gc/api_state_test.go index 36e2e449ed..fc26d7cb61 100644 --- a/cmd/gc/api_state_test.go +++ b/cmd/gc/api_state_test.go @@ -2006,7 +2006,11 @@ func TestControllerStateSchema2CreateThenDeleteConventionAgent(t *testing.T) { } } -func TestControllerStateAppliesCacheReconcileBeadEventsToStores(t *testing.T) { +func TestControllerStateSkipsCacheReconcileBeadEventDelivery(t *testing.T) { + // cache-reconcile events on the bus must NOT be re-applied to caching + // stores via ApplyEvent. The originating store already wrote its cache + // during reconcile; redelivering risks the omitempty + mergeCacheEventPatch + // self-feedback loop documented in applyBeadEventToStores. backing := beads.NewMemStore() created, err := backing.Create(beads.Bead{Title: "root"}) if err != nil { @@ -2017,6 +2021,8 @@ func TestControllerStateAppliesCacheReconcileBeadEventsToStores(t *testing.T) { t.Fatalf("Prime: %v", err) } + // Cache is primed with status="open". Construct a payload claiming + // status="in_progress" and deliver it via the cache-reconcile path. updated := created updated.Status = "in_progress" payload, err := json.Marshal(updated) @@ -2042,8 +2048,8 @@ func TestControllerStateAppliesCacheReconcileBeadEventsToStores(t *testing.T) { if len(items) != 1 || items[0].ID != created.ID { t.Fatalf("cached items = %+v, want only %s", items, created.ID) } - if items[0].Status != "in_progress" { - t.Fatalf("status after cache-reconcile event = %q, want in_progress", items[0].Status) + if items[0].Status != "open" { + t.Fatalf("status after cache-reconcile bus event = %q, want unchanged %q (cache-reconcile events must not be redelivered via ApplyEvent)", items[0].Status, "open") } } @@ -3757,6 +3763,70 @@ func TestControllerStateApplyCacheReconcileEventDoesNotPokeController(t *testing } } +// TestControllerStateApplyOrderTrackingEventDoesNotPokeController asserts that an +// order-tracking bead event does not poke the controller. +func TestControllerStateApplyOrderTrackingEventDoesNotPokeController(t *testing.T) { + cs := &controllerState{ + pokeCh: make(chan struct{}, 1), + } + + payload, err := json.Marshal(api.BeadEventPayload{ + Bead: beads.Bead{ + ID: "gc-ot-1", + Title: "order:gascity/some-order", + Labels: []string{"order-run:gascity/some-order", labelOrderTracking}, + }, + }) + if err != nil { + t.Fatalf("marshal order-tracking payload: %v", err) + } + + cs.applyBeadEventToStores(events.Event{ + Type: events.BeadCreated, + Actor: "agent-runtime", + Subject: "gc-ot-1", + Payload: payload, + }) + + select { + case <-cs.pokeCh: + t.Fatal("order-tracking bead event must not poke controller") + default: + } +} + +// TestControllerStateApplyNonOrderTrackingEventStillPokesController asserts that +// an ordinary work-bead event still pokes the controller. +func TestControllerStateApplyNonOrderTrackingEventStillPokesController(t *testing.T) { + cs := &controllerState{ + pokeCh: make(chan struct{}, 1), + } + + payload, err := json.Marshal(api.BeadEventPayload{ + Bead: beads.Bead{ + ID: "gc-work-1", + Title: "do the thing", + Labels: []string{"some-other-label"}, + }, + }) + if err != nil { + t.Fatalf("marshal work-bead payload: %v", err) + } + + cs.applyBeadEventToStores(events.Event{ + Type: events.BeadUpdated, + Actor: "agent-runtime", + Subject: "gc-work-1", + Payload: payload, + }) + + select { + case <-cs.pokeCh: + default: + t.Fatal("ordinary work-bead event must still poke controller") + } +} + func newControllerStateMutationHarness(t *testing.T) (*controllerState, string) { t.Helper() diff --git a/cmd/gc/apiroute.go b/cmd/gc/apiroute.go index 7c29a31023..96892dd647 100644 --- a/cmd/gc/apiroute.go +++ b/cmd/gc/apiroute.go @@ -23,22 +23,25 @@ var ( apiRouteSupervisorClientHook = supervisorCityAPIClient ) -// apiClient returns an API client if a controller with a mutable API server -// is running for the city at cityPath. Returns nil if no controller is running, -// the API is not configured, GC_NO_API is set truthy (operator escape hatch), -// or the API is bound to a non-localhost address without allow_mutations. -// CLI commands use this to route reads/writes through the API when available, -// falling back to direct bd or file mutation. +// apiClient returns an API client when one is reachable for the city at +// cityPath, or nil if the caller should use its local bd/file fallback. +// Returns nil if neither a standalone controller nor a supervisor is +// reachable, if no API is configured anywhere, if GC_NO_API is set truthy +// (operator escape hatch), or if the only reachable standalone API is bound to +// a non-localhost address without allow_mutations and no supervisor API is +// available. CLI commands use this to route reads/writes through the API when +// available, falling back to direct bd or file mutation. // // A standalone controller (gc controller / gc serve) and a supervisor-managed // city both answer the per-city controller socket — the supervisor hosts that -// controller in-process. When the socket is alive, apiClient routes to the -// standalone HTTP endpoint if the city configures an [api] port, otherwise -// returns nil so the caller uses its local fallback; when the socket is not -// alive it returns the supervisor-managed client. Maintenance commands have no -// local fallback, so they use maintenanceAPIClient, which additionally routes a -// supervisor-managed city (alive socket, no standalone [api] port) to the -// supervisor client rather than reporting controller-down. (gascity ga-tp7) +// controller in-process. When the socket is alive, apiClient prefers the +// standalone HTTP endpoint if the city configures a usable [api] port; +// otherwise it falls through to the supervisor's HTTP API (which services +// /v0/city/{name}/... routes for every supervisor-managed city). Fall-through +// is load-bearing on supervisor-managed cities: typical city.toml has no [api] +// section, so without it every general `gc` invocation would take the slow +// direct-Dolt path even though the supervisor's API is fully functional. +// (gascity gc-1rr12w) func apiClient(cityPath string) *api.Client { // Remote routing is NOT handled here. A remote target is refused upstream by // the capability gate in resolveContext (Phase 1) and, once enabled, will be @@ -58,11 +61,15 @@ func apiClient(cityPath string) *api.Client { fmt.Fprintln(os.Stderr, "warning: "+warn) //nolint:errcheck // best-effort stderr } if apiRouteControllerAliveHook(cityPath) != 0 { - // Alive socket: use the standalone HTTP endpoint when configured, else - // return nil so the caller takes its local fallback. A supervisor-managed - // city (no standalone [api] port) reaches the supervisor client only via - // maintenanceAPIClient, which has no local fallback. - return standaloneControllerClient(cityPath) + // Alive socket: prefer the standalone HTTP endpoint when it's usable. + if c := standaloneControllerClient(cityPath); c != nil { + return c + } + // Standalone-controller API isn't usable (config absent, [api] port + // unset, or a non-loopback bind without allow_mutations). A + // supervisor-managed city hits this on every call — fall through to the + // supervisor's HTTP API rather than returning nil, so general commands + // take the fast API path instead of slow direct-Dolt. (gascity gc-1rr12w) } return apiRouteSupervisorClientHook(cityPath) } @@ -103,22 +110,38 @@ func standaloneControllerCityName(cfg *config.City, cityPath string) string { // returned nil for cityPath. Read-path CLI commands call this when the // client is nil to emit a route=fallback reason= log line. // -// The closed set mirrors the enabler's reason codes (ga-71l): "escape-hatch" -// (GC_NO_API truthy), "non-loopback-bind" (API bound to non-localhost with -// mutations disallowed), "controller-down" (everything else — no controller, -// config missing, API port unset). +// The closed set: "escape-hatch" (GC_NO_API truthy), "non-loopback-bind" +// (standalone API bound to non-localhost with mutations disallowed and no +// supervisor reachable), "standalone-api-disabled" (standalone controller +// alive but no API port configured and no supervisor reachable), +// "controller-down" (no standalone controller and no supervisor reachable). +// +// The "standalone-api-disabled" code distinguishes the +// supervisor-managed-city case (where the controller socket answers ping +// but city.toml has no [api] section) from a genuinely down controller, +// so operators debugging route=fallback don't chase the wrong thread. func apiClientFallbackReason(cityPath string) string { if disabled, _ := classifyGCNoAPI(os.Getenv("GC_NO_API")); disabled { return "escape-hatch" } - if controllerAlive(cityPath) != 0 { + if apiRouteControllerAliveHook(cityPath) != 0 { tomlPath := filepath.Join(cityPath, "city.toml") if cfg, err := config.Load(fsys.OSFS{}, tomlPath); err == nil && cfg.API.Port > 0 { bind := cfg.API.BindOrDefault() if bind != "127.0.0.1" && bind != "localhost" && bind != "::1" && !cfg.API.AllowMutations { return "non-loopback-bind" } + // Standalone API is usable; apiClient would have returned a + // client. Reaching here is unexpected — fall through to the + // catch-all rather than emit a misleading code. + return "controller-down" } + // Standalone controller is alive but its HTTP API isn't configured. + // apiClient fell through to supervisorCityAPIClient and that also + // returned nil — surface the standalone-side cause so operators + // don't chase a phantom "controller-down" diagnosis on a perfectly + // healthy supervisor host. + return "standalone-api-disabled" } return "controller-down" } diff --git a/cmd/gc/apiroute_test.go b/cmd/gc/apiroute_test.go index 72990b495a..60072e52c7 100644 --- a/cmd/gc/apiroute_test.go +++ b/cmd/gc/apiroute_test.go @@ -54,12 +54,17 @@ func TestStandaloneControllerClient(t *testing.T) { } } -// TestAPIClientRouting covers apiClient's routing: the standalone endpoint when -// the socket is alive and an [api] port is configured, nil (the caller's local -// fallback) when alive without a standalone port, the supervisor client when the -// socket is down, and nil under the GC_NO_API escape hatch. The supervisor -// fall-through for a managed city with no [api] port is scoped to maintenance — -// see TestMaintenanceAPIClientRoutesToSupervisor. (gascity ga-tp7) +// TestAPIClientRouting covers apiClient's routing: the standalone endpoint +// when the socket is alive and a usable [api] port is configured, the +// supervisor client when the socket is alive but no standalone API is usable +// (the supervisor-managed-city fall-through), the supervisor client when the +// socket is down, nil when nothing is reachable, and nil under the GC_NO_API +// escape hatch. +// +// The general-command fall-through to the supervisor is the gc-1rr12w fork +// behavior: a supervisor-managed city answers the controller socket in-process +// but omits a standalone [api] port, so without the fall-through every `gc` +// read on such a city takes the slow direct-Dolt path. (gascity gc-1rr12w) func TestAPIClientRouting(t *testing.T) { sentinel := api.NewClient("http://supervisor.sentinel:1") @@ -70,14 +75,15 @@ func TestAPIClientRouting(t *testing.T) { origAlive, origSup := apiRouteControllerAliveHook, apiRouteSupervisorClientHook t.Cleanup(func() { restore(origAlive, origSup) }) - t.Run("controller-alive-no-api-port-returns-nil", func(t *testing.T) { - // General commands have a local fallback, so apiClient returns nil here - // (no global supervisor fall-through). + t.Run("controller-alive-no-api-port-falls-through-to-supervisor", func(t *testing.T) { + // Supervisor-managed city: controller socket alive, no standalone [api] + // port. apiClient must fall through to the supervisor rather than + // returning nil, so general commands use the API fast path. (gc-1rr12w) t.Setenv("GC_NO_API", "") restore(func(string) int { return 4242 }, func(string) *api.Client { return sentinel }) dir := writeCityTOMLForRoute(t, t.TempDir(), "name = \"t\"\n") - if got := apiClient(dir); got != nil { - t.Fatalf("apiClient = %p, want nil (general commands use local fallback)", got) + if got := apiClient(dir); got != sentinel { + t.Fatalf("apiClient = %p, want supervisor sentinel %p (managed-city fall-through)", got, sentinel) } }) @@ -94,6 +100,17 @@ func TestAPIClientRouting(t *testing.T) { } }) + t.Run("controller-alive-non-loopback-bind-falls-through", func(t *testing.T) { + // A non-loopback [api] without allow_mutations is not a usable standalone + // endpoint, so apiClient falls through to the supervisor. (gc-1rr12w) + t.Setenv("GC_NO_API", "") + restore(func(string) int { return 4242 }, func(string) *api.Client { return sentinel }) + dir := writeCityTOMLForRoute(t, t.TempDir(), "name = \"t\"\n[api]\nport = 8080\nbind = \"0.0.0.0\"\n") + if got := apiClient(dir); got != sentinel { + t.Fatalf("apiClient = %p, want supervisor sentinel %p (non-loopback fall-through)", got, sentinel) + } + }) + t.Run("controller-down-uses-supervisor", func(t *testing.T) { t.Setenv("GC_NO_API", "") restore(func(string) int { return 0 }, func(string) *api.Client { return sentinel }) @@ -103,6 +120,16 @@ func TestAPIClientRouting(t *testing.T) { } }) + t.Run("nothing-reachable-returns-nil", func(t *testing.T) { + // Controller down and supervisor unreachable: nil, caller uses local fallback. + t.Setenv("GC_NO_API", "") + restore(func(string) int { return 0 }, func(string) *api.Client { return nil }) + dir := writeCityTOMLForRoute(t, t.TempDir(), "name = \"t\"\n") + if got := apiClient(dir); got != nil { + t.Fatalf("apiClient = %p, want nil when neither standalone nor supervisor is reachable", got) + } + }) + t.Run("escape-hatch-returns-nil", func(t *testing.T) { t.Setenv("GC_NO_API", "1") restore(func(string) int { return 4242 }, func(string) *api.Client { return sentinel }) @@ -113,11 +140,40 @@ func TestAPIClientRouting(t *testing.T) { }) } -// TestMaintenanceAPIClientRoutesToSupervisor proves the maintenance-scoped -// fall-through: when the controller socket is alive but the supervisor-managed -// city omits a standalone [api] port, maintenanceAPIClient routes to the -// supervisor-managed client (maintenance has no local fallback), where general -// commands' apiClient returns nil. (gascity ga-tp7) +// TestAPIClientFallbackReason covers the reason codes that the gc-1rr12w +// supervisor fall-through introduces. The "standalone-api-disabled" code +// distinguishes a supervisor-managed city (controller socket alive, no [api] +// section, supervisor unreachable) from a genuinely down controller, so +// operators debugging route=fallback don't chase a phantom controller-liveness +// bug. The "escape-hatch" and "controller-down" codes are covered by +// route_log_test.go. (gascity gc-1rr12w) +func TestAPIClientFallbackReason(t *testing.T) { + origAlive := apiRouteControllerAliveHook + t.Cleanup(func() { apiRouteControllerAliveHook = origAlive }) + + t.Run("standalone-api-disabled", func(t *testing.T) { + t.Setenv("GC_NO_API", "") + apiRouteControllerAliveHook = func(string) int { return 4242 } + dir := writeCityTOMLForRoute(t, t.TempDir(), "name = \"t\"\n") + if got := apiClientFallbackReason(dir); got != "standalone-api-disabled" { + t.Fatalf("apiClientFallbackReason = %q, want %q", got, "standalone-api-disabled") + } + }) + + t.Run("non-loopback-bind", func(t *testing.T) { + t.Setenv("GC_NO_API", "") + apiRouteControllerAliveHook = func(string) int { return 4242 } + dir := writeCityTOMLForRoute(t, t.TempDir(), "name = \"t\"\n[api]\nport = 8080\nbind = \"0.0.0.0\"\n") + if got := apiClientFallbackReason(dir); got != "non-loopback-bind" { + t.Fatalf("apiClientFallbackReason = %q, want %q", got, "non-loopback-bind") + } + }) +} + +// TestMaintenanceAPIClientRoutesToSupervisor proves maintenanceAPIClient +// resolves the supervisor client for a supervisor-managed city (controller +// socket alive, no standalone [api] port) and honors the GC_NO_API escape +// hatch. (gascity ga-tp7) func TestMaintenanceAPIClientRoutesToSupervisor(t *testing.T) { sentinel := api.NewClient("http://supervisor.sentinel:1") origAlive, origSup := apiRouteControllerAliveHook, apiRouteSupervisorClientHook diff --git a/cmd/gc/bd_env.go b/cmd/gc/bd_env.go index b11a7f2a4f..5a3776b6e2 100644 --- a/cmd/gc/bd_env.go +++ b/cmd/gc/bd_env.go @@ -305,32 +305,23 @@ func applyCanonicalDoltTargetEnv(env map[string]string, target contract.DoltConn if env == nil { return } - // GC-owned projections must use the resolved target, not ambient parent - // shell host/port. Stale GC_DOLT_HOST/PORT was causing gc bd and projected - // session flows to drift away from the canonical external endpoint. - if shouldProjectResolvedDoltHost(target) { - env["GC_DOLT_HOST"] = strings.TrimSpace(target.Host) + // Always project the resolved host/port together — never inherit ambient + // parent shell host/port (PR #790 intent). Coupling them prevents a + // latent broken state where HOST alone would route bd via SQL with no + // port to connect to. Projecting the resolved loopback host in + // managed_city eliminates bd's empty-HOST CLI fallback (which forks + // `dolt remote -v` per call and saturates the host on multi-DB data dirs). + host := strings.TrimSpace(target.Host) + port := strings.TrimSpace(target.Port) + if host != "" && port != "" { + env["GC_DOLT_HOST"] = host + env["GC_DOLT_PORT"] = port } else { delete(env, "GC_DOLT_HOST") - } - if strings.TrimSpace(target.Port) != "" { - env["GC_DOLT_PORT"] = target.Port - } else { delete(env, "GC_DOLT_PORT") } } -func shouldProjectResolvedDoltHost(target contract.DoltConnectionTarget) bool { - host := strings.TrimSpace(target.Host) - if host == "" { - return false - } - if target.External { - return true - } - return !managedLocalDoltHost(host) -} - func applyCanonicalDoltAuthEnv(env map[string]string, cityPath, scopeRoot string, target contract.DoltConnectionTarget) { if env == nil { return diff --git a/cmd/gc/bd_env_test.go b/cmd/gc/bd_env_test.go index 50448bd5d9..4f4cb7fa90 100644 --- a/cmd/gc/bd_env_test.go +++ b/cmd/gc/bd_env_test.go @@ -661,6 +661,86 @@ dolt.auto-start: false } } +func TestApplyCanonicalDoltTargetEnvCouplesHostAndPort(t *testing.T) { + // applyCanonicalDoltTargetEnv must project HOST and PORT together or + // not at all. HOST without PORT would route bd via SQL with no port to + // connect to; PORT without HOST puts bd into its CLI fallback which + // forks `dolt remote -v` per call. + for _, tc := range []struct { + name string + target contract.DoltConnectionTarget + wantHost string // "" means key absent + wantPort string + }{ + { + name: "managed loopback projects both", + target: contract.DoltConnectionTarget{Host: "127.0.0.1", Port: "3307"}, + wantHost: "127.0.0.1", + wantPort: "3307", + }, + { + name: "external host projects both", + target: contract.DoltConnectionTarget{Host: "db.example.com", Port: "3307", External: true}, + wantHost: "db.example.com", + wantPort: "3307", + }, + { + name: "empty host strips both", + target: contract.DoltConnectionTarget{Port: "3307"}, + }, + { + name: "empty port strips both", + target: contract.DoltConnectionTarget{Host: "127.0.0.1"}, + }, + { + name: "empty target strips both", + target: contract.DoltConnectionTarget{}, + }, + { + name: "whitespace-only host strips both", + target: contract.DoltConnectionTarget{Host: " ", Port: "3307"}, + }, + { + name: "whitespace-only port strips both", + target: contract.DoltConnectionTarget{Host: "127.0.0.1", Port: " "}, + }, + { + name: "host with surrounding whitespace gets trimmed", + target: contract.DoltConnectionTarget{Host: " 127.0.0.1 ", Port: " 3307 "}, + wantHost: "127.0.0.1", + wantPort: "3307", + }, + } { + t.Run(tc.name, func(t *testing.T) { + // Seed env with stale values to confirm strip behavior. + env := map[string]string{ + "GC_DOLT_HOST": "stale.example.com", + "GC_DOLT_PORT": "9999", + } + applyCanonicalDoltTargetEnv(env, tc.target) + if got, ok := env["GC_DOLT_HOST"]; tc.wantHost == "" { + if ok { + t.Errorf("GC_DOLT_HOST = %q, want absent", got) + } + } else if got != tc.wantHost { + t.Errorf("GC_DOLT_HOST = %q, want %q", got, tc.wantHost) + } + if got, ok := env["GC_DOLT_PORT"]; tc.wantPort == "" { + if ok { + t.Errorf("GC_DOLT_PORT = %q, want absent", got) + } + } else if got != tc.wantPort { + t.Errorf("GC_DOLT_PORT = %q, want %q", got, tc.wantPort) + } + }) + } +} + +func TestApplyCanonicalDoltTargetEnvNilEnvIsNoop(_ *testing.T) { + // Should not panic; nothing to assert beyond not crashing. + applyCanonicalDoltTargetEnv(nil, contract.DoltConnectionTarget{Host: "127.0.0.1", Port: "3307"}) +} + func TestManagedLocalDoltHostRecognizesIPv6LoopbackAndWildcard(t *testing.T) { for _, tc := range []struct { host string @@ -2434,11 +2514,13 @@ dolt.auto-start: false if got := env["BEADS_DOLT_SERVER_PORT"]; got != wantPort { t.Fatalf("BEADS_DOLT_SERVER_PORT = %q, want %q", got, wantPort) } - if got := env["GC_DOLT_HOST"]; got != "" { - t.Fatalf("GC_DOLT_HOST = %q, want empty for managed target", got) + // Loopback host is projected so bd routes via SQL instead of falling + // back to its CLI mode (which forks `dolt remote -v` on each call). + if got := env["GC_DOLT_HOST"]; got != "127.0.0.1" { + t.Fatalf("GC_DOLT_HOST = %q, want %q for managed target", got, "127.0.0.1") } - if got := env["BEADS_DOLT_SERVER_HOST"]; got != "" { - t.Fatalf("BEADS_DOLT_SERVER_HOST = %q, want empty for managed target", got) + if got := env["BEADS_DOLT_SERVER_HOST"]; got != "127.0.0.1" { + t.Fatalf("BEADS_DOLT_SERVER_HOST = %q, want %q for managed target", got, "127.0.0.1") } } @@ -2542,6 +2624,15 @@ func TestBdRuntimeEnvForRigFallsBackToManagedCityPort(t *testing.T) { if got := env["BEADS_DOLT_SERVER_PORT"]; got != want { t.Fatalf("BEADS_DOLT_SERVER_PORT = %q, want %q", got, want) } + // HOST must be projected alongside PORT — bd CLI mode (empty HOST) + // forks `dolt remote -v` per call and saturates the host on multi-DB + // data dirs. + if got := env["GC_DOLT_HOST"]; got != "127.0.0.1" { + t.Fatalf("GC_DOLT_HOST = %q, want %q", got, "127.0.0.1") + } + if got := env["BEADS_DOLT_SERVER_HOST"]; got != "127.0.0.1" { + t.Fatalf("BEADS_DOLT_SERVER_HOST = %q, want %q", got, "127.0.0.1") + } if got := env["BEADS_DIR"]; got != filepath.Join(rigDir, ".beads") { t.Fatalf("BEADS_DIR = %q, want %q", got, filepath.Join(rigDir, ".beads")) } diff --git a/cmd/gc/beads_provider_lifecycle.go b/cmd/gc/beads_provider_lifecycle.go index 32b10e496e..4a7b8e92a4 100644 --- a/cmd/gc/beads_provider_lifecycle.go +++ b/cmd/gc/beads_provider_lifecycle.go @@ -773,9 +773,24 @@ func ensureBeadsProvider(cityPath string) error { return nil } +// stopManagedDoltProcessForShutdown reaps the managed Dolt sql-server during +// city teardown. It is a package var so tests can stub the process-terminating +// syscalls; production wiring is stopManagedDoltProcess, which discovers the +// live PID from the process table rather than trusting runtime state files. +var stopManagedDoltProcessForShutdown = stopManagedDoltProcess + // shutdownBeadsProvider stops the bead store's backing service. // Called by gc stop after agents have been terminated. // For exec providers, fires "stop". For file providers, always available. +// +// When the city owns its managed Dolt lifecycle (endpoint_origin=managed_city), +// the provider "stop" op terminates the sql-server only through a shell +// round-trip (gc-beads-bd.sh -> gc dolt-state stop-managed) that can silently +// miss a Dolt started outside the normal lifecycle (e.g. dolt.auto-start=false +// plus a manual `gc dolt start`), leaking the sql-server past teardown +// (bead gc-wf0f6o). shutdownBeadsProvider therefore follows the +// provider stop with a direct, idempotent reap that discovers the live PID from +// the process table; it is a no-op when the process is already gone. func shutdownBeadsProvider(cityPath string) error { if cityUsesBdStoreContract(cityPath) && gcDoltSkip() { return clearManagedDoltRuntimeStateUnlessPostgres(cityPath) @@ -784,26 +799,40 @@ func shutdownBeadsProvider(cityPath string) error { return clearManagedDoltRuntimeStateUnlessPostgres(cityPath) } provider := beadsProvider(cityPath) - if strings.HasPrefix(provider, "exec:") { - if providerUsesBdStoreContract(provider) { - owned, err := managedDoltLifecycleOwned(cityPath) - if err != nil { - return err - } - if !owned { - return clearManagedDoltRuntimeStateUnlessPostgres(cityPath) - } - } - script := strings.TrimPrefix(provider, "exec:") - providerEnv, err := providerLifecycleProcessEnvWithError(cityPath, provider) + if !strings.HasPrefix(provider, "exec:") { + return nil + } + ownsManagedDolt := false + if providerUsesBdStoreContract(provider) { + owned, err := managedDoltLifecycleOwned(cityPath) if err != nil { return err } - if err := runProviderOpWithEnv(script, providerEnv, "stop"); err != nil { - return err + if !owned { + return clearManagedDoltRuntimeStateUnlessPostgres(cityPath) } - if err := clearManagedDoltRuntimeStateIfOwned(cityPath); err != nil { - return err + ownsManagedDolt = true + } + // Capture the managed Dolt port before the provider stop op clears runtime + // state, so the post-stop reap can still target the port holder directly. + managedDoltPort := "" + if ownsManagedDolt { + managedDoltPort = currentResolvableManagedDoltPort(cityPath) + } + script := strings.TrimPrefix(provider, "exec:") + providerEnv, err := providerLifecycleProcessEnvWithError(cityPath, provider) + if err != nil { + return err + } + if err := runProviderOpWithEnv(script, providerEnv, "stop"); err != nil { + return err + } + if err := clearManagedDoltRuntimeStateIfOwned(cityPath); err != nil { + return err + } + if ownsManagedDolt { + if _, err := stopManagedDoltProcessForShutdown(cityPath, managedDoltPort); err != nil { + return fmt.Errorf("reap managed dolt process for %s: %w", cityPath, err) } } return nil diff --git a/cmd/gc/beads_provider_lifecycle_test.go b/cmd/gc/beads_provider_lifecycle_test.go index 52e860850a..51d3c7f80d 100644 --- a/cmd/gc/beads_provider_lifecycle_test.go +++ b/cmd/gc/beads_provider_lifecycle_test.go @@ -10635,6 +10635,109 @@ dolt.auto-start: false } } +// TestShutdownBeadsProviderReapsOwnedManagedDoltProcess verifies that tearing +// down a city that owns its managed Dolt lifecycle (endpoint_origin=managed_city) +// follows the provider "stop" op with a direct stopManagedDoltProcess reap. The +// provider stop alone terminates the sql-server through a shell round-trip that +// can miss a Dolt started outside the normal lifecycle (dolt.auto-start=false +// plus a manual `gc dolt start`); the reap guarantees the process is gone. +// Regression test for the gc stop managed-Dolt leak (bead gc-wf0f6o). +func TestShutdownBeadsProviderReapsOwnedManagedDoltProcess(t *testing.T) { + cityPath := t.TempDir() + callLog := filepath.Join(cityPath, "op-calls.log") + script := gcBeadsBdScriptPath(cityPath) + if err := os.MkdirAll(filepath.Dir(script), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(script, []byte("#!/bin/sh\necho \"$1\" >> "+callLog+"\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(cityPath, ".beads"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cityPath, ".beads", "config.yaml"), []byte(`issue_prefix: gc +gc.endpoint_origin: managed_city +gc.endpoint_status: verified +dolt.auto-start: false +`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cityPath, ".beads", "metadata.json"), + []byte(`{"backend":"dolt","dolt_database":"hq","dolt_mode":"server"}`), 0o644); err != nil { + t.Fatal(err) + } + + var reapCalls int + var reapCity string + old := stopManagedDoltProcessForShutdown + stopManagedDoltProcessForShutdown = func(cp, _ string) (managedDoltStopReport, error) { + reapCalls++ + reapCity = cp + return managedDoltStopReport{}, nil + } + t.Cleanup(func() { stopManagedDoltProcessForShutdown = old }) + + t.Setenv("GC_BEADS", "exec:"+script) + t.Setenv("GC_BEADS_SCOPE_ROOT", cityPath) + if err := shutdownBeadsProvider(cityPath); err != nil { + t.Fatalf("shutdownBeadsProvider() error = %v", err) + } + + // The provider stop op must still run for the owned managed city. + if got := string(mustReadFile(t, callLog)); !strings.Contains(got, "stop") { + t.Fatalf("provider stop op not invoked; op log = %q", got) + } + // And the direct process reap must follow it. + if reapCalls != 1 { + t.Fatalf("managed dolt reap calls = %d, want 1", reapCalls) + } + if reapCity != cityPath { + t.Fatalf("reap cityPath = %q, want %q", reapCity, cityPath) + } +} + +// TestShutdownBeadsProviderSkipsReapForExternalTarget confines the managed-Dolt +// reap to cities that own their Dolt lifecycle: a city pointed at an external +// canonical endpoint must not have someone else's sql-server reaped. +func TestShutdownBeadsProviderSkipsReapForExternalTarget(t *testing.T) { + cityPath := t.TempDir() + script := gcBeadsBdScriptPath(cityPath) + if err := os.MkdirAll(filepath.Dir(script), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(script, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(cityPath, ".beads"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cityPath, ".beads", "config.yaml"), []byte(`issue_prefix: gc +gc.endpoint_origin: city_canonical +gc.endpoint_status: verified +dolt.host: 127.0.0.1 +dolt.port: "4406" +`), 0o644); err != nil { + t.Fatal(err) + } + + reapCalled := false + old := stopManagedDoltProcessForShutdown + stopManagedDoltProcessForShutdown = func(string, string) (managedDoltStopReport, error) { + reapCalled = true + return managedDoltStopReport{}, nil + } + t.Cleanup(func() { stopManagedDoltProcessForShutdown = old }) + + t.Setenv("GC_BEADS", "exec:"+script) + t.Setenv("GC_BEADS_SCOPE_ROOT", cityPath) + if err := shutdownBeadsProvider(cityPath); err != nil { + t.Fatalf("shutdownBeadsProvider() error = %v", err) + } + if reapCalled { + t.Fatal("managed dolt reap must not run for an external canonical endpoint") + } +} + // ── startBeadsLifecycle skips provider for external ─────────────────── func TestStartBeadsLifecycleSkipsProviderForExternalHost(t *testing.T) { diff --git a/cmd/gc/city_runtime.go b/cmd/gc/city_runtime.go index 046bf9242a..1a9aa30c3b 100644 --- a/cmd/gc/city_runtime.go +++ b/cmd/gc/city_runtime.go @@ -106,6 +106,11 @@ type CityRuntime struct { asyncStarts asyncStartTracker asyncStops asyncStartTracker demandSnapshot *runtimeDemandSnapshot + // demandDirty forces a one-shot pool-demand rebuild on the next tick. It is + // the signal for demand that the session fingerprint can't see — routed work + // for a sleeping pool mutates no session bead — set from any goroutine + // (controller socket, API state) and consumed by the reconciler. + demandDirty *atomic.Bool fsPressureConsecutiveSkips int fsPressureEpisodeLogged bool @@ -163,6 +168,7 @@ type CityRuntimeParams struct { WatchTargets []config.WatchTarget ConfigRev string ConfigDirty *atomic.Bool + DemandDirty *atomic.Bool Cfg *config.City SP runtime.Provider @@ -229,6 +235,10 @@ func newCityRuntime(p CityRuntimeParams) *CityRuntime { if configDirty == nil { configDirty = &atomic.Bool{} } + demandDirty := p.DemandDirty + if demandDirty == nil { + demandDirty = &atomic.Bool{} + } it := buildIdleTracker(p.Cfg, p.CityName, p.CityPath, p.SP) mat := buildMaxSessionAgeTracker(p.Cfg, p.CityName, p.SP) @@ -287,6 +297,7 @@ func newCityRuntime(p CityRuntimeParams) *CityRuntime { watchTargets: p.WatchTargets, configRev: p.ConfigRev, configDirty: configDirty, + demandDirty: demandDirty, cfg: p.Cfg, sp: p.SP, publication: p.Publication, @@ -1176,7 +1187,7 @@ func (cr *CityRuntime) tick( return } phaseStart = time.Now() - demand := cr.loadDemandSnapshot(sessionBeads, trace, trigger, configChanged) + demand := cr.loadDemandSnapshot(sessionBeads, trace, configChanged) recordPhase(TraceSiteDemandSnapshot, "load_demand_snapshot", phaseStart, map[string]any{ "config_changed": configChanged, "trigger": trigger, @@ -1558,7 +1569,7 @@ func (cr *CityRuntime) runNudgeMailSweepWatchdog(now time.Time) { } func (cr *CityRuntime) orderTrackingSweepStores() ([]beads.Store, []orderTrackingSweepTarget, func(), error) { //nolint:unparam // targets slice returned for callers that need sweep scope metadata; current call sites discard it - targets := orderTrackingSweepTargetsForConfig(cr.cityPath, cr.cfg) + targets := orderTrackingSweepTargetsForConfig(cr.cityPath, cr.cfg, cr.stderr) rigStores := cr.rigBeadStores() var freshlyOpened []beads.Store stores, err := orderTrackingSweepStoresFromTargets(targets, func(sweepTarget orderTrackingSweepTarget) (beads.Store, error) { @@ -3189,11 +3200,10 @@ func (cr *CityRuntime) buildDesiredState(sessionBeads *sessionBeadSnapshot, trac func (cr *CityRuntime) loadDemandSnapshot( sessionBeads *sessionBeadSnapshot, trace *sessionReconcilerTraceCycle, - trigger string, configChanged bool, ) runtimeDemandSnapshot { sessionFingerprint := sessionBeadSnapshotFingerprint(sessionBeads) - if cr.shouldRefreshDemandSnapshot(trigger, configChanged, sessionFingerprint) { + if cr.shouldRefreshDemandSnapshot(configChanged, sessionFingerprint) { result := cr.buildDesiredState(sessionBeads, trace) var openSessionInfos []sessionpkg.Info if sessionBeads != nil { @@ -3226,15 +3236,21 @@ func (cr *CityRuntime) loadDemandSnapshot( return snapshot } +// shouldRefreshDemandSnapshot reports whether the cached pool-demand snapshot +// must be rebuilt this tick. It rebuilds on a config change, a one-shot +// demandDirty signal, a session-fingerprint change, or snapshot age; otherwise +// poke-driven ticks reuse the cached snapshot, keeping the high-frequency +// session-lifecycle wake path off the per-tick demand rebuild. func (cr *CityRuntime) shouldRefreshDemandSnapshot( - trigger string, configChanged bool, sessionFingerprint string, ) bool { - // Non-patrol triggers (config reloads and controller pokes — e.g. from - // sling-dispatched work) always rebuild immediately; only patrol-cadence - // re-evaluation is throttled below. - if configChanged || trigger != "patrol" { + // Consume the one-shot demand signal before any other rebuild reason can + // short-circuit past it, so it is never left set for a redundant rebuild on + // the next tick. The flag is fail-safe: it can only force an extra rebuild, + // never suppress one, so routed work for a sleeping pool is never missed. + demandDirty := cr.demandDirty != nil && cr.demandDirty.Swap(false) + if configChanged || demandDirty { return true } if cr.demandSnapshot == nil { @@ -3252,7 +3268,9 @@ func (cr *CityRuntime) shouldRefreshDemandSnapshot( // demandSnapshotPatrolMaxAge reports how long a cached demand snapshot may be // reused across consecutive patrol ticks, or 0 when patrol must rebuild every -// tick. Non-patrol triggers bypass this entirely (see shouldRefreshDemandSnapshot). +// tick. A work-routing poke (demandDirty) forces an immediate rebuild regardless +// of this age, so routed work for a sleeping pool is observed without waiting for +// the floor (see shouldRefreshDemandSnapshot). func (cr *CityRuntime) demandSnapshotPatrolMaxAge() time.Duration { if cr.demandSnapshotsEnabled() { return runtimeDemandSnapshotMaxAge diff --git a/cmd/gc/city_runtime_test.go b/cmd/gc/city_runtime_test.go index 5d666a3e3d..0fa9bb3d74 100644 --- a/cmd/gc/city_runtime_test.go +++ b/cmd/gc/city_runtime_test.go @@ -812,8 +812,8 @@ func TestCityRuntimeDemandSnapshotReusesStablePatrolDemand(t *testing.T) { }, }}) - first := cr.loadDemandSnapshot(sessionBeads, nil, "patrol", false) - second := cr.loadDemandSnapshot(sessionBeads, nil, "patrol", false) + first := cr.loadDemandSnapshot(sessionBeads, nil, false) + second := cr.loadDemandSnapshot(sessionBeads, nil, false) if buildCalls != 1 { t.Fatalf("buildDesiredState call count = %d, want 1 for stable patrol reuse", buildCalls) @@ -832,14 +832,139 @@ func TestCityRuntimeDemandSnapshotReusesStablePatrolDemand(t *testing.T) { "pending_create_claim": "true", }, }}) - _ = cr.loadDemandSnapshot(changedSessionBeads, nil, "patrol", false) + _ = cr.loadDemandSnapshot(changedSessionBeads, nil, false) if buildCalls != 2 { t.Fatalf("buildDesiredState call count after session change = %d, want 2", buildCalls) } - _ = cr.loadDemandSnapshot(changedSessionBeads, nil, "poke", false) + // A poke-driven (non-patrol) tick with a stable fingerprint reuses the fresh + // cached snapshot instead of rebuilding — the same gate patrol ticks use. + _ = cr.loadDemandSnapshot(changedSessionBeads, nil, false) + if buildCalls != 2 { + t.Fatalf("buildDesiredState call count after stable poke = %d, want 2 (poke reuses fresh cache)", buildCalls) + } + + // A config change still forces a rebuild — templates actually changed. + _ = cr.loadDemandSnapshot(changedSessionBeads, nil, true) if buildCalls != 3 { - t.Fatalf("buildDesiredState call count after poke = %d, want 3", buildCalls) + t.Fatalf("buildDesiredState call count after config change = %d, want 3 (config change rebuilds)", buildCalls) + } +} + +// TestCityRuntimeDemandSnapshotRebuildsForWorkRoutingPoke asserts that a +// work-routing poke (demandDirty) forces a demand rebuild even when the session +// fingerprint is stable, so routed work for a sleeping pool is observed this +// tick — while a plain poke still reuses the cached snapshot. +func TestCityRuntimeDemandSnapshotRebuildsForWorkRoutingPoke(t *testing.T) { + buildCalls := 0 + routedWork := false + cr := &CityRuntime{ + cityName: "test-city", + cityPath: t.TempDir(), + cfg: &config.City{ + Workspace: config.Workspace{Name: "test-city"}, + }, + cs: &controllerState{ + eventProv: events.NewFake(), + }, + demandDirty: &atomic.Bool{}, + stderr: io.Discard, + } + cr.buildFnWithSessionBeads = func(*config.City, runtime.Provider, beads.Store, map[string]beads.Store, *sessionBeadSnapshot, *sessionReconcilerTraceCycle) DesiredStateResult { + buildCalls++ + st := map[string]TemplateParams{} + if routedWork { + // Newly-routed pool work produces a desired session (the wake). + st["worker-bd-1"] = TemplateParams{SessionName: "worker-bd-1"} + } + return DesiredStateResult{State: st} + } + + // Sleeping pool: no session beads at all. Prime a no-demand snapshot. + sessionBeads := newSessionBeadSnapshot(nil) + primed := cr.loadDemandSnapshot(sessionBeads, nil, false) + if buildCalls != 1 { + t.Fatalf("buildDesiredState call count after prime = %d, want 1", buildCalls) + } + if len(primed.result.State) != 0 { + t.Fatalf("primed desired sessions = %d, want 0 (no work routed yet)", len(primed.result.State)) + } + + // A plain poke with a stable fingerprint and no demand-dirty signal reuses + // the cache — the perf win we must not regress. + _ = cr.loadDemandSnapshot(sessionBeads, nil, false) + if buildCalls != 1 { + t.Fatalf("buildDesiredState call count after plain poke = %d, want 1 (cache reuse preserved)", buildCalls) + } + + // Routing new work for the sleeping pool fires a work-routing poke that marks + // demand dirty. The fingerprint is unchanged, so only the flag forces a rebuild. + routedWork = true + cr.demandDirty.Store(true) + woken := cr.loadDemandSnapshot(sessionBeads, nil, false) + if buildCalls != 2 { + t.Fatalf("buildDesiredState call count after work-routing poke = %d, want 2 (forced rebuild)", buildCalls) + } + if len(woken.result.State) != 1 { + t.Fatalf("woken desired sessions = %d, want 1 (routed work observed immediately)", len(woken.result.State)) + } + if cr.demandDirty.Load() { + t.Fatal("demandDirty must be cleared once a rebuild consumes it (one-shot)") + } + + // The signal is one-shot: the next stable poke reuses the rebuilt cache. + _ = cr.loadDemandSnapshot(sessionBeads, nil, false) + if buildCalls != 2 { + t.Fatalf("buildDesiredState call count after consuming demand-dirty = %d, want 2 (reuse)", buildCalls) + } +} + +// TestCityRuntimeDemandSnapshotConsumesDirtyFlagOnConfigChange asserts the +// demandDirty consume cannot be bypassed by another rebuild reason: when a tick +// already rebuilds for a config change while a work-routing poke also marked +// demand dirty, the flag is still consumed, so it can't trigger a second, +// redundant rebuild on the next stable poke. +func TestCityRuntimeDemandSnapshotConsumesDirtyFlagOnConfigChange(t *testing.T) { + buildCalls := 0 + cr := &CityRuntime{ + cityName: "test-city", + cityPath: t.TempDir(), + cfg: &config.City{ + Workspace: config.Workspace{Name: "test-city"}, + }, + cs: &controllerState{ + eventProv: events.NewFake(), + }, + demandDirty: &atomic.Bool{}, + stderr: io.Discard, + } + cr.buildFnWithSessionBeads = func(*config.City, runtime.Provider, beads.Store, map[string]beads.Store, *sessionBeadSnapshot, *sessionReconcilerTraceCycle) DesiredStateResult { + buildCalls++ + return DesiredStateResult{State: map[string]TemplateParams{}} + } + + sessionBeads := newSessionBeadSnapshot(nil) + _ = cr.loadDemandSnapshot(sessionBeads, nil, false) + if buildCalls != 1 { + t.Fatalf("buildDesiredState call count after prime = %d, want 1", buildCalls) + } + + // A config-change tick that also carries a work-routing demand-dirty signal: + // the config change forces the rebuild, but the flag must still be consumed. + cr.demandDirty.Store(true) + _ = cr.loadDemandSnapshot(sessionBeads, nil, true) + if buildCalls != 2 { + t.Fatalf("buildDesiredState call count after config-change tick = %d, want 2", buildCalls) + } + if cr.demandDirty.Load() { + t.Fatal("demandDirty must be consumed even when a config change also forces the rebuild") + } + + // The next stable poke reuses the cache, proving no leftover flag forced a + // redundant rebuild. + _ = cr.loadDemandSnapshot(sessionBeads, nil, false) + if buildCalls != 2 { + t.Fatalf("buildDesiredState call count after stable poke = %d, want 2 (no redundant rebuild)", buildCalls) } } @@ -1332,7 +1457,7 @@ func TestCityRuntimeDemandSnapshotRetainsOnlyPoolScaleCheckPartials(t *testing.T return tc.result } - snapshot := cr.loadDemandSnapshot(sessionBeads, nil, "poke", false) + snapshot := cr.loadDemandSnapshot(sessionBeads, nil, false) if got := snapshot.result.PoolDesiredCounts["worker"]; got != tc.want { t.Fatalf("PoolDesiredCounts[worker] = %d, want %d", got, tc.want) @@ -1887,6 +2012,11 @@ func TestOrderTrackingSweepWatchdogClosesRigStoreSweepTracking(t *testing.T) { cityPath := t.TempDir() rigPath := filepath.Join(cityPath, "frontend") + // The rig directory must exist on disk: sweep target enumeration skips + // configured rigs whose scope root is missing (gc-q40pm Problem B). + if err := os.MkdirAll(rigPath, 0o755); err != nil { + t.Fatalf("MkdirAll(rigPath): %v", err) + } cr := &CityRuntime{ cityPath: cityPath, cityName: "test-city", @@ -1944,6 +2074,11 @@ func TestOrderTrackingSweepWatchdogFallsBackToConfiguredRigStore(t *testing.T) { } cityPath := t.TempDir() rigPath := filepath.Join(cityPath, "frontend") + // The rig directory must exist on disk: sweep target enumeration skips + // configured rigs whose scope root is missing (gc-q40pm Problem B). + if err := os.MkdirAll(rigPath, 0o755); err != nil { + t.Fatalf("MkdirAll(rigPath): %v", err) + } prevOpenSweepStore := newCityRuntimeOpenSweepStore newCityRuntimeOpenSweepStore = func(scopeRoot, gotCityPath string) (beads.Store, error) { if gotCityPath != cityPath { @@ -2039,8 +2174,8 @@ func TestCityRuntimeDemandSnapshotCachesCustomDemandCommands(t *testing.T) { } sessionBeads := newSessionBeadSnapshot(nil) - _ = cr.loadDemandSnapshot(sessionBeads, nil, "patrol", false) - _ = cr.loadDemandSnapshot(sessionBeads, nil, "patrol", false) + _ = cr.loadDemandSnapshot(sessionBeads, nil, false) + _ = cr.loadDemandSnapshot(sessionBeads, nil, false) if buildCalls != tc.wantBuilds { t.Fatalf("buildDesiredState call count = %d, want %d", buildCalls, tc.wantBuilds) @@ -2064,7 +2199,8 @@ func TestCityRuntimeDemandSnapshotThrottlesScaleCheckPatrolReeval(t *testing.T) cs: &controllerState{ eventProv: events.NewFake(), }, - stderr: io.Discard, + demandDirty: &atomic.Bool{}, + stderr: io.Discard, } cr.buildFnWithSessionBeads = func(*config.City, runtime.Provider, beads.Store, map[string]beads.Store, *sessionBeadSnapshot, *sessionReconcilerTraceCycle) DesiredStateResult { buildCalls++ @@ -2078,25 +2214,35 @@ func TestCityRuntimeDemandSnapshotThrottlesScaleCheckPatrolReeval(t *testing.T) sessionBeads := newSessionBeadSnapshot(nil) - // First patrol builds; a second immediate patrol is throttled. - _ = cr.loadDemandSnapshot(sessionBeads, nil, "patrol", false) - _ = cr.loadDemandSnapshot(sessionBeads, nil, "patrol", false) + // First patrol builds; a second immediate patrol is throttled by the + // scale_check floor (scaleCheckDemandMinInterval). The floor only bites once + // the fork's always-rebuild top-guard is gone: a scale_check pool now reuses a + // sub-floor-age snapshot instead of re-running the probe on every patrol tick. + _ = cr.loadDemandSnapshot(sessionBeads, nil, false) + _ = cr.loadDemandSnapshot(sessionBeads, nil, false) if buildCalls != 1 { t.Fatalf("buildDesiredState calls after two immediate patrols = %d, want 1 (throttled)", buildCalls) } // Once the floor elapses, the next patrol re-runs the probe. cr.demandSnapshot.createdAt = time.Now().Add(-2 * scaleCheckDemandMinInterval) - _ = cr.loadDemandSnapshot(sessionBeads, nil, "patrol", false) + _ = cr.loadDemandSnapshot(sessionBeads, nil, false) if buildCalls != 2 { t.Fatalf("buildDesiredState calls after interval elapsed = %d, want 2", buildCalls) } - // Non-patrol triggers (routed-work pokes/events) bypass the floor so pools - // wake immediately, even within the throttle window. - _ = cr.loadDemandSnapshot(sessionBeads, nil, "poke", false) + // A work-routing poke (demandDirty) bypasses the floor so a sleeping pool + // wakes immediately for routed work, even within the throttle window. The + // snapshot is still fresh from the rebuild above, so only the flag — not age — + // forces this rebuild. (A plain poke reuses the cache; see + // TestCityRuntimeDemandSnapshotReusesStablePatrolDemand.) + cr.demandDirty.Store(true) + _ = cr.loadDemandSnapshot(sessionBeads, nil, false) if buildCalls != 3 { - t.Fatalf("buildDesiredState calls after poke = %d, want 3 (event-driven wake must bypass throttle)", buildCalls) + t.Fatalf("buildDesiredState calls after work-routing poke = %d, want 3 (demandDirty bypasses throttle)", buildCalls) + } + if cr.demandDirty.Load() { + t.Fatal("demandDirty must be cleared once a rebuild consumes it (one-shot)") } // A session change within the window also forces an immediate rebuild. @@ -2105,7 +2251,7 @@ func TestCityRuntimeDemandSnapshotThrottlesScaleCheckPatrolReeval(t *testing.T) Status: "open", Metadata: map[string]string{"session_name": "polecat-1", "template": "polecat", "state": "active"}, }}) - _ = cr.loadDemandSnapshot(changed, nil, "patrol", false) + _ = cr.loadDemandSnapshot(changed, nil, false) if buildCalls != 4 { t.Fatalf("buildDesiredState calls after session change = %d, want 4", buildCalls) } @@ -2131,7 +2277,7 @@ func TestCityRuntimeDemandSnapshotDoesNotRunControllerWorkQuery(t *testing.T) { return DesiredStateResult{State: map[string]TemplateParams{}} } - snapshot := cr.loadDemandSnapshot(newSessionBeadSnapshot(nil), nil, "patrol", false) + snapshot := cr.loadDemandSnapshot(newSessionBeadSnapshot(nil), nil, false) if len(snapshot.result.WorkSet) != 0 { t.Fatalf("WorkSet = %#v, want empty; controller demand must not run work_query", snapshot.result.WorkSet) @@ -2165,7 +2311,7 @@ func TestCityRuntimeDemandSnapshotReplaysACPRoutesOnCacheHit(t *testing.T) { stderr: io.Discard, } - _ = cr.loadDemandSnapshot(nil, nil, "patrol", false) + _ = cr.loadDemandSnapshot(nil, nil, false) if err := sp.Attach("headless-agent"); err == nil || !strings.Contains(err.Error(), "ACP transport") { t.Fatalf("Attach(headless-agent) error = %v, want ACP transport route", err) diff --git a/cmd/gc/cmd_bd_test.go b/cmd/gc/cmd_bd_test.go index 13f5f0e8e8..a617d1f360 100644 --- a/cmd/gc/cmd_bd_test.go +++ b/cmd/gc/cmd_bd_test.go @@ -720,14 +720,16 @@ set -eu if got["GC_BEADS_PREFIX"] != "repo" { t.Fatalf("GC_BEADS_PREFIX = %q, want %q", got["GC_BEADS_PREFIX"], "repo") } - if got["GC_DOLT_HOST"] != "" { - t.Fatalf("GC_DOLT_HOST = %q, want empty for managed target", got["GC_DOLT_HOST"]) + // Loopback host is projected so bd routes via SQL instead of falling + // back to its CLI mode (which forks `dolt remote -v` on each call). + if got["GC_DOLT_HOST"] != "127.0.0.1" { + t.Fatalf("GC_DOLT_HOST = %q, want %q for managed target", got["GC_DOLT_HOST"], "127.0.0.1") } if got["GC_DOLT_PORT"] != wantPort { t.Fatalf("GC_DOLT_PORT = %q, want %q", got["GC_DOLT_PORT"], wantPort) } - if got["BEADS_DOLT_SERVER_HOST"] != "" { - t.Fatalf("BEADS_DOLT_SERVER_HOST = %q, want empty for managed target", got["BEADS_DOLT_SERVER_HOST"]) + if got["BEADS_DOLT_SERVER_HOST"] != "127.0.0.1" { + t.Fatalf("BEADS_DOLT_SERVER_HOST = %q, want %q for managed target", got["BEADS_DOLT_SERVER_HOST"], "127.0.0.1") } if got["BEADS_DOLT_SERVER_PORT"] != wantPort { t.Fatalf("BEADS_DOLT_SERVER_PORT = %q, want %q", got["BEADS_DOLT_SERVER_PORT"], wantPort) diff --git a/cmd/gc/cmd_convoy.go b/cmd/gc/cmd_convoy.go index 0941c9af47..8f15f52046 100644 --- a/cmd/gc/cmd_convoy.go +++ b/cmd/gc/cmd_convoy.go @@ -85,24 +85,33 @@ control beads.`, } type convoyCreateOptions struct { - Fields ConvoyFields - Owned bool + Fields ConvoyFields + Owned bool + CityScope bool } func newConvoyCreateCmd(stdout, stderr io.Writer) *cobra.Command { var owner, notify, merge, target string - var owned, jsonOut bool + var owned, jsonOut, cityScope bool cmd := &cobra.Command{ Use: "create [issue-ids...]", Short: "Create a convoy and optionally track issues", Long: `Create a convoy and optionally link existing issues to it. Creates a convoy bead and tracks any provided issue IDs. Issues can -also be added later with "gc convoy add".`, +also be added later with "gc convoy add". + +The convoy is created in the same rig scope that "gc bd" would resolve: +an explicit --rig flag wins, then the current directory's rig, then the +GC_RIG environment variable, otherwise city scope. When tracked issues +are supplied they anchor the scope so the convoy and its issues share a +store. Pass --city-scope to force city scope (and silence the city-scope +fall-through warning).`, Example: ` gc convoy create sprint-42 gc convoy create sprint-42 issue-1 issue-2 issue-3 gc convoy create deploy --owner mayor --notify mayor --merge mr - gc convoy create auth-rewrite --owned --target integration/auth-rewrite`, + gc convoy create auth-rewrite --owned --target integration/auth-rewrite + gc convoy create infra-sweep --city-scope`, Args: cobra.ArbitraryArgs, RunE: func(_ *cobra.Command, args []string) error { opts := convoyCreateOptions{ @@ -112,7 +121,8 @@ also be added later with "gc convoy add".`, Merge: merge, Target: target, }, - Owned: owned, + Owned: owned, + CityScope: cityScope, } code := 0 if jsonOut { @@ -131,6 +141,7 @@ also be added later with "gc convoy add".`, cmd.Flags().StringVar(&merge, "merge", "", "merge strategy: direct, mr, local") cmd.Flags().StringVar(&target, "target", "", "target branch inherited by child work beads") cmd.Flags().BoolVar(&owned, "owned", false, "mark convoy as owned (manual lifecycle, no auto-close)") + cmd.Flags().BoolVar(&cityScope, "city-scope", false, "force city scope (overrides --rig/GC_RIG/cwd detection and silences the city-scope warning)") cmd.Flags().BoolVar(&jsonOut, "json", false, "emit JSONL result") return cmd } @@ -161,12 +172,17 @@ func cmdConvoyCreateWithOptionsJSON(args []string, opts convoyCreateOptions, jso } } - // Determine which store to use: if children are provided, use the - // first child's rig store so convoy and children share a database. - // This avoids cross-store parent references that bd can't resolve. - storeDir := cityPath - if len(issueIDs) > 0 { - storeDir = convoyCreateStoreRoot(cfg, cityPath, issueIDs[0]) + // Resolve which store the convoy lands in, mirroring `gc bd`'s rig + // resolution (--rig flag, cwd, GC_RIG, then city scope). Tracked issues, + // when supplied, anchor the scope so the convoy and its children share a + // store and parent references stay resolvable. + storeDir, scopeWarning, err := resolveConvoyCreateScope(cfg, cityPath, rigFlag, opts.CityScope, issueIDs) + if err != nil { + fmt.Fprintf(stderr, "gc convoy create: %v\n", err) //nolint:errcheck // best-effort stderr + return 1 + } + if scopeWarning != "" { + fmt.Fprintln(stderr, scopeWarning) //nolint:errcheck // best-effort stderr } store, err := openStoreAtForCity(storeDir, cityPath) if err != nil { @@ -184,6 +200,72 @@ func doConvoyCreate(store beads.Store, rec events.Recorder, args []string, stdou return doConvoyCreateWithOptions(store, rec, args, convoyCreateOptions{}, stdout, stderr) } +// resolveConvoyCreateScope decides which bead store a new convoy is created +// in. It mirrors `gc bd`'s rig resolution so `gc convoy create` is no longer +// asymmetric with `gc bd create`: an explicit --rig flag wins, then a +// cwd-detected rig, then the GC_RIG env var, then city scope — the same order +// resolveBdScopeTarget applies. +// +// When child issue IDs are supplied they anchor the scope: a convoy and the +// issues it tracks must share a store because bd cannot resolve cross-store +// parent references. The children's store therefore overrides the resolved +// scope, and a warning is returned when that override contradicts an explicit +// rig/city signal (silent issue-prefix routing remains the no-signal default). +// +// It returns the absolute store scope root, an optional stderr warning, and an +// error — the latter only for contradictory flags or an unresolvable --rig. +func resolveConvoyCreateScope(cfg *config.City, cityPath, rigName string, cityScope bool, issueIDs []string) (string, string, error) { + rigName = strings.TrimSpace(rigName) + if cityScope && rigName != "" { + return "", "", fmt.Errorf("--city-scope conflicts with --rig %q: choose one", rigName) + } + + var resolved execStoreTarget + if cityScope { + resolved = bdCityScopeTarget(cityPath, cfg) + } else { + // Pass no bead args: the convoy name is not a bead and child issues + // are handled below, so resolveBdScopeTarget is restricted to its + // flag/cwd/env resolution — the same store selection `gc bd` uses. + target, err := resolveBdScopeTarget(cfg, cityPath, rigName, nil, false) + if err != nil { + return "", "", err + } + resolved = target + } + + if len(issueIDs) > 0 { + childRoot := convoyCreateStoreRoot(cfg, cityPath, issueIDs[0]) + if !samePath(childRoot, resolved.ScopeRoot) && (cityScope || resolved.ScopeKind == "rig") { + // An explicit signal (a resolved rig, or --city-scope) is being + // overridden so the convoy can co-locate with its issues. + return childRoot, "gc convoy create: warning: tracked issues live outside the resolved scope; creating the convoy alongside them so parent references resolve", nil + } + return childRoot, "", nil + } + + if !cityScope && resolved.ScopeKind != "rig" && cityHasBoundRig(cfg) { + return resolved.ScopeRoot, "gc convoy create: warning: no rig resolved from --rig, GC_RIG, or the current directory; creating the convoy in city scope. Pass --rig to target a rig, or --city-scope to silence this warning.", nil + } + return resolved.ScopeRoot, "", nil +} + +// cityHasBoundRig reports whether cfg declares at least one rig with a path +// binding. The city-scope fall-through warning is only meaningful when a rig +// was an available alternative; a rigless city has nowhere else to put a +// convoy, so warning there would be pure noise. +func cityHasBoundRig(cfg *config.City) bool { + if cfg == nil { + return false + } + for _, rig := range cfg.Rigs { + if strings.TrimSpace(rig.Path) != "" { + return true + } + } + return false +} + func convoyCreateStoreRoot(cfg *config.City, cityPath, beadID string) string { if cfg != nil { if rd := rigDirForBead(cfg, beadID); rd != "" { diff --git a/cmd/gc/cmd_convoy_scope_test.go b/cmd/gc/cmd_convoy_scope_test.go new file mode 100644 index 0000000000..74f94bd4c3 --- /dev/null +++ b/cmd/gc/cmd_convoy_scope_test.go @@ -0,0 +1,223 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/gastownhall/gascity/internal/config" +) + +// newConvoyScopeTestCity builds a city tempdir with a single bound rig +// "myrig" (prefix mp) at rigs/myrig and returns the city path, the rig's +// resolved store root, and a fresh config. A fresh config is returned per +// call because resolveBdScopeTarget mutates Rig.Path to an absolute path. +func newConvoyScopeTestCity(t *testing.T) (cityPath, rigRoot string, cfg *config.City) { + t.Helper() + cityPath = t.TempDir() + rigRoot = filepath.Join(cityPath, "rigs", "myrig") + if err := os.MkdirAll(rigRoot, 0o755); err != nil { + t.Fatal(err) + } + cfg = &config.City{ + Workspace: config.Workspace{Name: "demo", Prefix: "gc"}, + Rigs: []config.Rig{ + {Name: "myrig", Path: filepath.Join("rigs", "myrig"), Prefix: "mp"}, + }, + } + return cityPath, filepath.Clean(rigRoot), cfg +} + +// TestResolveConvoyCreateScope exercises the rig-resolution contract for +// `gc convoy create` (gc-nm4d2h): the convoy store is resolved like `gc bd` +// (--rig flag, cwd, GC_RIG, then city scope), tracked issues anchor the +// scope, and the city-scope fall-through warns unless --city-scope opts in. +// +// Each subtest sets cwd and GC_RIG explicitly so resolution is deterministic +// regardless of where the test binary runs. +func TestResolveConvoyCreateScope(t *testing.T) { + t.Run("rig flag wins from city cwd", func(t *testing.T) { + cityPath, rigRoot, cfg := newConvoyScopeTestCity(t) + t.Chdir(cityPath) + t.Setenv("GC_RIG", "") + + got, warn, err := resolveConvoyCreateScope(cfg, cityPath, "myrig", false, nil) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if !samePath(got, rigRoot) { + t.Errorf("storeDir = %q, want rig root %q", got, rigRoot) + } + if warn != "" { + t.Errorf("warning = %q, want none", warn) + } + }) + + t.Run("GC_RIG env resolves from city cwd", func(t *testing.T) { + cityPath, rigRoot, cfg := newConvoyScopeTestCity(t) + t.Chdir(cityPath) + t.Setenv("GC_RIG", "myrig") + + got, warn, err := resolveConvoyCreateScope(cfg, cityPath, "", false, nil) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if !samePath(got, rigRoot) { + t.Errorf("storeDir = %q, want rig root %q", got, rigRoot) + } + if warn != "" { + t.Errorf("warning = %q, want none", warn) + } + }) + + t.Run("cwd inside rig resolves to rig", func(t *testing.T) { + cityPath, rigRoot, cfg := newConvoyScopeTestCity(t) + t.Chdir(rigRoot) + t.Setenv("GC_RIG", "") + + got, warn, err := resolveConvoyCreateScope(cfg, cityPath, "", false, nil) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if !samePath(got, rigRoot) { + t.Errorf("storeDir = %q, want rig root %q", got, rigRoot) + } + if warn != "" { + t.Errorf("warning = %q, want none", warn) + } + }) + + t.Run("no signal falls back to city scope with warning", func(t *testing.T) { + cityPath, _, cfg := newConvoyScopeTestCity(t) + t.Chdir(cityPath) + t.Setenv("GC_RIG", "") + + got, warn, err := resolveConvoyCreateScope(cfg, cityPath, "", false, nil) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if !samePath(got, cityPath) { + t.Errorf("storeDir = %q, want city path %q", got, cityPath) + } + if !strings.Contains(warn, "city scope") { + t.Errorf("warning = %q, want one mentioning city scope", warn) + } + }) + + t.Run("city-scope flag forces city and silences warning", func(t *testing.T) { + cityPath, _, cfg := newConvoyScopeTestCity(t) + // Even from inside the rig, --city-scope wins. + t.Chdir(filepath.Join(cityPath, "rigs", "myrig")) + t.Setenv("GC_RIG", "myrig") + + got, warn, err := resolveConvoyCreateScope(cfg, cityPath, "", true, nil) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if !samePath(got, cityPath) { + t.Errorf("storeDir = %q, want city path %q", got, cityPath) + } + if warn != "" { + t.Errorf("warning = %q, want none", warn) + } + }) + + t.Run("city-scope conflicts with rig flag", func(t *testing.T) { + cityPath, _, cfg := newConvoyScopeTestCity(t) + t.Chdir(cityPath) + t.Setenv("GC_RIG", "") + + _, _, err := resolveConvoyCreateScope(cfg, cityPath, "myrig", true, nil) + if err == nil { + t.Fatal("err = nil, want a conflict error") + } + if !strings.Contains(err.Error(), "city-scope") || !strings.Contains(err.Error(), "rig") { + t.Errorf("err = %v, want mention of both --city-scope and --rig", err) + } + }) + + t.Run("unknown rig flag errors", func(t *testing.T) { + cityPath, _, cfg := newConvoyScopeTestCity(t) + t.Chdir(cityPath) + t.Setenv("GC_RIG", "") + + _, _, err := resolveConvoyCreateScope(cfg, cityPath, "nope", false, nil) + if err == nil { + t.Fatal("err = nil, want 'rig not found' error") + } + }) + + t.Run("city issue anchors to city without warning", func(t *testing.T) { + cityPath, _, cfg := newConvoyScopeTestCity(t) + t.Chdir(cityPath) + t.Setenv("GC_RIG", "") + + got, warn, err := resolveConvoyCreateScope(cfg, cityPath, "", false, []string{"gc-1"}) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if !samePath(got, cityPath) { + t.Errorf("storeDir = %q, want city path %q", got, cityPath) + } + if warn != "" { + t.Errorf("warning = %q, want none", warn) + } + }) + + t.Run("rig issue routes to rig store without warning", func(t *testing.T) { + cityPath, rigRoot, cfg := newConvoyScopeTestCity(t) + t.Chdir(cityPath) + t.Setenv("GC_RIG", "") + + // No explicit signal: issue-prefix routing (the path that + // historically worked) places the convoy with its issue. + got, warn, err := resolveConvoyCreateScope(cfg, cityPath, "", false, []string{"mp-1"}) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if !samePath(got, rigRoot) { + t.Errorf("storeDir = %q, want rig root %q", got, rigRoot) + } + if warn != "" { + t.Errorf("warning = %q, want none", warn) + } + }) + + t.Run("cross-rig: issue store overrides resolved rig with warning", func(t *testing.T) { + cityPath, _, cfg := newConvoyScopeTestCity(t) + t.Chdir(cityPath) + t.Setenv("GC_RIG", "") + + // --rig resolves to myrig, but the tracked issue lives in the city + // store; the convoy must co-locate with the issue, and we warn. + got, warn, err := resolveConvoyCreateScope(cfg, cityPath, "myrig", false, []string{"gc-1"}) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if !samePath(got, cityPath) { + t.Errorf("storeDir = %q, want city path %q (co-located with issue)", got, cityPath) + } + if !strings.Contains(warn, "tracked issues") { + t.Errorf("warning = %q, want one mentioning tracked issues", warn) + } + }) + + t.Run("rigless city does not warn on city scope", func(t *testing.T) { + cityPath := t.TempDir() + t.Chdir(cityPath) + t.Setenv("GC_RIG", "") + cfg := &config.City{Workspace: config.Workspace{Name: "solo", Prefix: "gc"}} + + got, warn, err := resolveConvoyCreateScope(cfg, cityPath, "", false, nil) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if !samePath(got, cityPath) { + t.Errorf("storeDir = %q, want city path %q", got, cityPath) + } + if warn != "" { + t.Errorf("warning = %q, want none (no rig to warn about)", warn) + } + }) +} diff --git a/cmd/gc/cmd_doctor.go b/cmd/gc/cmd_doctor.go index 3e6cbea9fa..ef1ecde775 100644 --- a/cmd/gc/cmd_doctor.go +++ b/cmd/gc/cmd_doctor.go @@ -305,6 +305,7 @@ func buildDoctorChecks(cityPath string, cfg *config.City, cfgErr error, opts bui // Data checks. if cfgErr == nil && cfg != nil { register(doctor.NewBDSplitStoreCheck(cityPath)) + register(doctor.NewBdConfigParseCheck(cityPath)) register(doctor.NewBeadsStoreCheck(cityPath, storeFactory)) register(newV2RoutedToNamespaceCheck(cfg, cityPath, storeFactory)) register(newRunTargetRoutedToBackfillCheck(cfg, cityPath, storeFactory)) @@ -312,12 +313,20 @@ func buildDoctorChecks(cityPath string, cfg *config.City, cfgErr error, opts bui register(newBacklogDepthCheck(cityPath, storeFactory)) register(newOrderTrackingRetentionCheck(cityPath, storeFactory)) register(&sessionModelDoctorCheck{cfg: cfg, cityPath: cityPath, newStore: storeFactory}) + register(&stuckCreatingDoctorCheck{cfg: cfg, cityPath: cityPath, newStore: storeFactory}) } register(newDoctorDoltServerCheck(cityPath, opts.SkipCityDoltCheck)) // Host-level fork-rate watch: surfaces the per-command data-plane fork storm // (gc -> bd.real -> dolt) that operators routinely misread as CPU saturation. // Advisory + read-only (/proc/stat); no config needed. register(newForkRateCheck()) + // Host-level build-scratch watch: surfaces low free space on the filesystem + // backing $TMPDIR (default /tmp, a size-capped tmpfs on this class of host). + // Parallel `make check` CGO/ICU linking can exhaust it and surface as + // spurious ENOSPC test failures; the managed-Dolt/store-maintenance guards + // watch only the Dolt data dir, so this closes that monitoring gap (gc-yiqil). + // Advisory + read-only (statfs); no config needed. + register(newTmpScratchSpaceCheck()) if cfgErr == nil && doctorWorkspaceHasPostgresScope(cityPath, cfg) { register(doctor.NewPostgresAuthCheck(cityPath, cfg)) } @@ -374,6 +383,7 @@ func buildDoctorChecks(cityPath string, cfg *config.City, cfgErr error, opts bui register(doctor.NewRigGitCheck(rig)) register(doctor.NewRigRootBranchCheck(rig)) register(doctor.NewRigBDSplitStoreCheck(cityPath, rig)) + register(doctor.NewRigBdConfigParseCheck(rig)) register(doctor.NewRigBeadsCheck(cityPath, rig, storeFactory)) register(newDoctorRigDoltServerCheck(cityPath, rig, !rigUsesManagedBdStoreContract(cityPath, rig) || gcDoltSkip())) // Custom types check — rig store. @@ -394,6 +404,7 @@ func buildDoctorChecks(cityPath string, cfg *config.City, cfgErr error, opts bui // Pack doctor checks — scripts shipped with packs. if cfgErr == nil && cfg != nil { + packScriptTimeout := doctorCfg.PackScriptTimeout() for _, entry := range cfg.PackDoctors { register(&doctor.PackScriptCheck{ CheckName: entry.PackName + ":" + entry.Name, @@ -402,6 +413,7 @@ func buildDoctorChecks(cityPath string, cfg *config.City, cfgErr error, opts bui PackDir: entry.PackDir, PackName: entry.PackName, Warmup: entry.Warmup, + Timeout: packScriptTimeout, }) } registerLocalDoctorChecksTo(register, cityPath, cfg.Doctor.Checks) diff --git a/cmd/gc/cmd_formula_test.go b/cmd/gc/cmd_formula_test.go index dcdbc42f31..eea839403d 100644 --- a/cmd/gc/cmd_formula_test.go +++ b/cmd/gc/cmd_formula_test.go @@ -403,7 +403,6 @@ func TestFormulaCatalogEntriesUseFormulaRefForDiscovery(t *testing.T) { runGitForFormulaTest(t, root, "init", "-b", "main") runGitForFormulaTest(t, root, "config", "user.email", "test@example.com") runGitForFormulaTest(t, root, "config", "user.name", "test") - runGitForFormulaTest(t, root, "config", "commit.gpgsign", "false") formulaDir := filepath.Join(root, "formulas") writeFormulaTestFile(t, formulaDir, "from-ref", `formula = "from-ref" diff --git a/cmd/gc/cmd_lint.go b/cmd/gc/cmd_lint.go index dca490d0ff..5e532dd523 100644 --- a/cmd/gc/cmd_lint.go +++ b/cmd/gc/cmd_lint.go @@ -400,6 +400,10 @@ func lintPromptContext(packDir string, agentCfg config.Agent, providers map[stri qualifiedName = "lint-agent" } providerKey := agentCfg.Provider + configDir := packDir + if agentCfg.SourceDir != "" { + configDir = agentCfg.SourceDir + } return PromptContext{ CityRoot: packDir, AgentName: qualifiedName, @@ -412,6 +416,7 @@ func lintPromptContext(packDir string, agentCfg config.Agent, providers map[stri IssuePrefix: "lint", Branch: "feature/lint", DefaultBranch: "main", + ConfigDir: configDir, AssignedInProgressQuery: agentCfg.EffectiveAssignedInProgressQueryForBeads(config.BeadsConfig{}), AssignedReadyQuery: agentCfg.EffectiveAssignedReadyQueryForBeads(config.BeadsConfig{}), RoutedPoolQuery: agentCfg.EffectiveRoutedPoolQueryForBeads(config.BeadsConfig{}), diff --git a/cmd/gc/cmd_order.go b/cmd/gc/cmd_order.go index 4c8bce9252..1ed39be633 100644 --- a/cmd/gc/cmd_order.go +++ b/cmd/gc/cmd_order.go @@ -1609,7 +1609,7 @@ func cmdOrderSweepTrackingWithOptions(staleAfter time.Duration, includeWisps, dr fmt.Fprintf(stderr, "gc order sweep-tracking: %v\n", err) //nolint:errcheck // best-effort stderr return 1 } - stores, openErr := orderTrackingSweepStoresForConfigTargets(cityPath, cfg, requiredTargets) + stores, openErr := orderTrackingSweepStoresForConfigTargets(cityPath, cfg, requiredTargets, stderr) if len(stores) == 0 { if openErr != nil { fmt.Fprintf(stderr, "gc order sweep-tracking: %v\n", openErr) //nolint:errcheck // best-effort stderr diff --git a/cmd/gc/cmd_prime.go b/cmd/gc/cmd_prime.go index 514f48ed47..fb435d3049 100644 --- a/cmd/gc/cmd_prime.go +++ b/cmd/gc/cmd_prime.go @@ -321,7 +321,7 @@ func doPrimeWithHookFormat(args []string, stdout, stderr io.Writer, hookMode boo } var ctx PromptContext if a.PromptTemplate != "" || hookMode || sessionTemplateContext { - ctx = buildPrimeContextForBeads(cityPath, cityName, &a, cfg.Rigs, cfg.Beads, stderr) + ctx = buildPrimeContextForBeads(cityPath, cityName, &a, cfg, cfg.Rigs, cfg.Beads, stderr) ctx.ProviderKey, ctx.ProviderDisplayName = providerInfoForAgent(&a, &cfg.Workspace, cfg.Providers) ctx.InstructionsFile = instructionsFileForAgent(&a, &cfg.Workspace, cfg.Providers) } @@ -754,17 +754,28 @@ func findAgentByName(cfg *config.City, name string) (config.Agent, bool) { // buildPrimeContext constructs a PromptContext for gc prime. Uses GC_* // environment variables when running inside a managed session, falls back -// to currentRigContext when run manually. +// to currentRigContext when run manually. It passes a nil City config to +// buildPrimeContextForBeads; callers that need the city-level fallback for +// fields like DefaultMergeStrategy call buildPrimeContextForBeads directly. func buildPrimeContext(cityPath, cityName string, a *config.Agent, rigs []config.Rig, stderr io.Writer) PromptContext { - return buildPrimeContextForBeads(cityPath, cityName, a, rigs, config.BeadsConfig{}, stderr) + return buildPrimeContextForBeads(cityPath, cityName, a, nil, rigs, config.BeadsConfig{}, stderr) } -func buildPrimeContextForBeads(cityPath, cityName string, a *config.Agent, rigs []config.Rig, beadsCfg config.BeadsConfig, stderr io.Writer) PromptContext { +// buildPrimeContextForBeads constructs a PromptContext for gc prime with an +// explicit beads config. `cfg` is the loaded City config and may be nil in +// tests; it supplies the city-level fallback for fields like +// DefaultMergeStrategy. +func buildPrimeContextForBeads(cityPath, cityName string, a *config.Agent, cfg *config.City, rigs []config.Rig, beadsCfg config.BeadsConfig, stderr io.Writer) PromptContext { + configDir := cityPath + if a.SourceDir != "" { + configDir = a.SourceDir + } ctx := PromptContext{ CityRoot: cityPath, TemplateName: a.Name, BindingName: a.BindingName, BindingPrefix: a.BindingPrefix(), + ConfigDir: configDir, Env: a.Env, } @@ -798,6 +809,7 @@ func buildPrimeContextForBeads(cityPath, cityName string, a *config.Agent, rigs ctx.Branch = os.Getenv("GC_BRANCH") ctx.DefaultBranch = defaultBranchForRig(ctx.RigName, rigs, ctx.WorkDir) + ctx.DefaultMergeStrategy = mergeStrategyForRig(ctx.RigName, rigs, cfg) ctx.WorkQuery = expandAgentCommandTemplate(cityPath, cityName, a, rigs, "work_query", a.EffectiveWorkQueryForBeads(beadsCfg), stderr) ctx.AssignedInProgressQuery = expandAgentCommandTemplate(cityPath, cityName, a, rigs, "assigned_in_progress_query", a.EffectiveAssignedInProgressQueryForBeads(beadsCfg), stderr) ctx.AssignedReadyQuery = expandAgentCommandTemplate(cityPath, cityName, a, rigs, "assigned_ready_query", a.EffectiveAssignedReadyQueryForBeads(beadsCfg), stderr) diff --git a/cmd/gc/cmd_prime_test.go b/cmd/gc/cmd_prime_test.go index bbfcf58daa..13631c4747 100644 --- a/cmd/gc/cmd_prime_test.go +++ b/cmd/gc/cmd_prime_test.go @@ -64,7 +64,7 @@ func TestBuildPrimeContextUsesBD105ReadyCompatibility(t *testing.T) { cityPath := filepath.Join(t.TempDir(), "demo-city") ctx := buildPrimeContextForBeads(cityPath, "", &config.Agent{ Name: "worker", - }, nil, config.BeadsConfig{BDCompatibility: config.BeadsBDCompatibility105}, nil) + }, nil, nil, config.BeadsConfig{BDCompatibility: config.BeadsBDCompatibility105}, nil) if !strings.Contains(ctx.AssignedReadyQuery, `bd ready --include-ephemeral --assignee="$id"`) { t.Fatalf("AssignedReadyQuery = %q, want bd-1.0.5-compatible assigned ready query", ctx.AssignedReadyQuery) diff --git a/cmd/gc/cmd_session.go b/cmd/gc/cmd_session.go index 3dcc32f340..295186d325 100644 --- a/cmd/gc/cmd_session.go +++ b/cmd/gc/cmd_session.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "os" "os/exec" "sort" "strconv" @@ -1741,16 +1742,19 @@ func cmdSessionSuspend(args []string, stdout, stderr io.Writer, jsonOutput ...bo return 0 } -// newSessionCloseCmd creates the "gc session close " command. +// newSessionCloseCmd creates the "gc session close [id-or-alias]" command. func newSessionCloseCmd(stdout, stderr io.Writer) *cobra.Command { var jsonOutput bool cmd := &cobra.Command{ - Use: "close ", + Use: "close [session-id-or-alias]", Short: "Close a session permanently", Long: `End a conversation. Stops the runtime if active and closes the bead. -Accepts a session ID (e.g., gc-42) or session alias (e.g., mayor).`, - Args: cobra.ExactArgs(1), +Accepts a session ID (e.g., gc-42) or session alias (e.g., mayor). +When called with no argument, defaults to $GC_SESSION_ID — the +canonical way for an agent to self-close from inside its own +runtime.`, + Args: cobra.MaximumNArgs(1), RunE: func(_ *cobra.Command, args []string) error { if cmdSessionClose(args, stdout, stderr, jsonOutput) != 0 { return errExit @@ -1764,8 +1768,23 @@ Accepts a session ID (e.g., gc-42) or session alias (e.g., mayor).`, } // cmdSessionClose is the CLI entry point for "gc session close". +// When args is empty, falls back to $GC_SESSION_ID so an agent can +// self-close from inside its own runtime without re-substituting +// its own session id. func cmdSessionClose(args []string, stdout, stderr io.Writer, jsonOutput ...bool) int { asJSON := sessionJSONRequested(jsonOutput) + target := "" + if len(args) > 0 { + target = args[0] + } + if target == "" { + target = strings.TrimSpace(os.Getenv("GC_SESSION_ID")) + } + if target == "" { + fmt.Fprintln(stderr, "gc session close: no session id given and $GC_SESSION_ID is unset") //nolint:errcheck // best-effort stderr + return 1 + } + store, code := openCityStore(stderr, "gc session close") if store == nil { return code @@ -1782,7 +1801,7 @@ func cmdSessionClose(args []string, stdout, stderr io.Writer, jsonOutput ...bool // (unclaimWorkAssignedToRetiredSessionBead) is WORK-class and stays on the // generic store. sessStore := cliSessionStore(store, cfg, cityPath) - sessionID, err := resolveSessionIDWithConfig(cityPath, cfg, sessStore, args[0]) + sessionID, err := resolveSessionIDWithConfig(cityPath, cfg, sessStore, target) if err != nil { fmt.Fprintf(stderr, "gc session close: %v\n", err) //nolint:errcheck // best-effort stderr return 1 diff --git a/cmd/gc/cmd_session_reset_test.go b/cmd/gc/cmd_session_reset_test.go index 0b60895019..004e7bc6be 100644 --- a/cmd/gc/cmd_session_reset_test.go +++ b/cmd/gc/cmd_session_reset_test.go @@ -81,6 +81,7 @@ func TestCmdSessionReset_ClearsCircuitBreaker(t *testing.T) { func() {}, nil, nil, + nil, make(chan reloadRequest), make(chan convergenceRequest, 1), make(chan struct{}, 1), @@ -147,6 +148,7 @@ func TestCmdSessionReset_ProviderConstructionFailureReturnsError(t *testing.T) { func() {}, nil, nil, + nil, make(chan reloadRequest), make(chan convergenceRequest, 1), make(chan struct{}, 1), @@ -244,6 +246,7 @@ func TestCmdSessionKill_ClearsCircuitBreaker(t *testing.T) { func() {}, nil, nil, + nil, make(chan reloadRequest), make(chan convergenceRequest, 1), make(chan struct{}, 1), @@ -339,6 +342,7 @@ func TestCmdSessionKill_SyncsBeadToAsleep(t *testing.T) { func() {}, nil, nil, + nil, make(chan reloadRequest), make(chan convergenceRequest, 1), make(chan struct{}, 1), @@ -421,6 +425,7 @@ func TestCmdSessionKill_ClearsCircuitBreakerForAsleepNamedSession(t *testing.T) func() {}, nil, nil, + nil, make(chan reloadRequest), make(chan convergenceRequest, 1), make(chan struct{}, 1), diff --git a/cmd/gc/cmd_sling.go b/cmd/gc/cmd_sling.go index af261083d5..75653bc61c 100644 --- a/cmd/gc/cmd_sling.go +++ b/cmd/gc/cmd_sling.go @@ -168,7 +168,8 @@ Examples: type slingOpts = sling.SlingOpts var ( - slingPokeController = pokeController + // slingPokeController wakes the controller after sling routes work. + slingPokeController = pokeControllerDemand slingPokeControlDispatcher = pokeControlDispatch slingOpenCityStore = openCityStoreAt ) @@ -676,6 +677,17 @@ func (r cliBeadRouter) Route(_ context.Context, req sling.RouteRequest) error { if err := r.deps.Store.SetMetadata(req.BeadID, beadmeta.RoutedToMetadataKey, routedTo); err != nil { return fmt.Errorf("setting gc.routed_to on %s: %w", req.BeadID, err) } + // Singleton targets also get a direct assignee stamp. Without this the + // target's own Tier 1 work_query (assignee match) misses routed work + // because Tiers 2-3 short-circuit for named-origin sessions. Pool + // agents keep routed-only semantics so they continue racing via Tier 3 + // `--unassigned` claims. See gastownhall/gascity#yb5uhi. + if req.Singleton { + target := req.Target + if err := r.deps.Store.Update(req.BeadID, beads.UpdateOpts{Assignee: &target}); err != nil { + return fmt.Errorf("setting assignee on %s: %w", req.BeadID, err) + } + } return nil } @@ -1446,8 +1458,8 @@ func doSlingNudge(a *config.Agent, cityName, cityPath string, cfg *config.City, } } } - // No running config session — poke controller for immediate wake. - if err := pokeController(cityPath); err != nil { + // No running session for the target — poke the controller to wake the pool. + if err := pokeControllerDemand(cityPath); err != nil { fmt.Fprintf(stderr, "No running sessions for %q; poke failed: %v\n", a.QualifiedName(), err) //nolint:errcheck // best-effort } else { fmt.Fprintf(stdout, "No running sessions for %q — poked controller for wake\n", a.QualifiedName()) //nolint:errcheck // best-effort @@ -1477,6 +1489,19 @@ func pokeController(cityPath string) error { return pokeSupervisor() } +// pokeControllerDemand is pokeController plus a demand-rebuild signal, for +// callers that change pool demand without mutating a session bead (e.g. gc sling +// routing work to a sleeping pool) — a change the demand snapshot's fingerprint +// cannot otherwise observe. The supervisor fallback ("reload") also rebuilds demand. +func pokeControllerDemand(cityPath string) error { + _, err := sendControllerCommand(cityPath, "poke-demand") + if err == nil { + return nil + } + // Fall back to supervisor reload, which rebuilds demand via configChanged. + return pokeSupervisor() +} + // reloadControllerConfig asks the controller to reload config immediately. // If the per-city controller socket doesn't exist (supervisor model), falls // back to sending "reload" to the supervisor socket. diff --git a/cmd/gc/cmd_sling_test.go b/cmd/gc/cmd_sling_test.go index 6ec1e61e22..eab3e762fc 100644 --- a/cmd/gc/cmd_sling_test.go +++ b/cmd/gc/cmd_sling_test.go @@ -473,6 +473,9 @@ func TestDoSlingBeadToFixedAgent(t *testing.T) { if bead.Metadata["gc.routed_to"] != "mayor" { t.Errorf("gc.routed_to = %q, want mayor", bead.Metadata["gc.routed_to"]) } + if bead.Assignee != "mayor" { + t.Errorf("assignee = %q, want mayor (singleton routing must surface via Tier 1 hook query)", bead.Assignee) + } if !strings.Contains(stdout.String(), "Slung BL-42") { t.Errorf("stdout = %q, want to contain 'Slung BL-42'", stdout.String()) } @@ -505,11 +508,79 @@ func TestDoSlingPinnedDefaultSlingQueryUsesBuiltInRouting(t *testing.T) { if bead.Metadata["gc.routed_to"] != "mayor" { t.Errorf("gc.routed_to = %q, want mayor", bead.Metadata["gc.routed_to"]) } + if bead.Assignee != "mayor" { + t.Errorf("assignee = %q, want mayor (pinned-default singletons must also receive direct assignment)", bead.Assignee) + } if !strings.Contains(stdout.String(), "Slung BL-42") { t.Errorf("stdout = %q, want to contain 'Slung BL-42'", stdout.String()) } } +// TestDoSlingBeadToBindingSingletonSetsAssignee covers gastownhall/gascity#yb5uhi: +// routing to a binding-named singleton (e.g. gc-toolkit.mechanik) must stamp the +// bead's assignee so the singleton's own Tier 1 work_query surfaces routed work. +// Before the fix, only gc.routed_to was set and the singleton's hook query +// missed the bead because Tier 2/3 short-circuit for named-origin sessions. +func TestDoSlingBeadToBindingSingletonSetsAssignee(t *testing.T) { + runner := newFakeRunner() + sp := runtime.NewFake() + cfg := &config.City{Workspace: config.Workspace{Name: "test-city"}} + a := config.Agent{ + Name: "mechanik", + BindingName: "gc-toolkit", + MaxActiveSessions: intPtr(1), + } + + deps, stdout, stderr := testDeps(cfg, sp, runner.run) + opts := testOpts(a, "BL-42") + code := doSling(opts, deps, nil, stdout, stderr) + + if code != 0 { + t.Fatalf("doSling returned %d, want 0; stderr: %s", code, stderr.String()) + } + bead, err := deps.Store.Get("BL-42") + if err != nil { + t.Fatalf("store.Get(BL-42): %v", err) + } + want := "gc-toolkit.mechanik" + if bead.Metadata["gc.routed_to"] != want { + t.Errorf("gc.routed_to = %q, want %q", bead.Metadata["gc.routed_to"], want) + } + if bead.Assignee != want { + t.Errorf("assignee = %q, want %q (binding singleton routing must stamp assignee for Tier 1 hook query)", bead.Assignee, want) + } +} + +// TestDoSlingBeadToSingletonOverwritesExistingAssignee verifies that a stale +// human or other agent assignee is replaced when routing to a singleton. The +// sling operation is an explicit "give this bead to " command, so the +// target must own the bead afterward — otherwise it would remain invisible to +// the target's Tier 1 work_query. +func TestDoSlingBeadToSingletonOverwritesExistingAssignee(t *testing.T) { + runner := newFakeRunner() + sp := runtime.NewFake() + cfg := &config.City{Workspace: config.Workspace{Name: "test-city"}} + a := config.Agent{Name: "mayor", MaxActiveSessions: intPtr(1)} + + deps, stdout, stderr := testDeps(cfg, sp, runner.run) + // Pre-seed an existing assignee on the bead, as if a human had claimed it. + preAssign := "human-alice" + if err := deps.Store.Update("BL-42", beads.UpdateOpts{Assignee: &preAssign}); err != nil { + t.Fatalf("seed assignee: %v", err) + } + opts := testOpts(a, "BL-42") + if code := doSling(opts, deps, nil, stdout, stderr); code != 0 { + t.Fatalf("doSling returned %d, want 0; stderr: %s", code, stderr.String()) + } + bead, err := deps.Store.Get("BL-42") + if err != nil { + t.Fatalf("store.Get(BL-42): %v", err) + } + if bead.Assignee != "mayor" { + t.Errorf("assignee = %q, want mayor (sling to singleton must replace stale claim)", bead.Assignee) + } +} + func TestDoSlingEnvPassthrough(t *testing.T) { // Fixed agent (max=1): env should contain GC_SLING_TARGET with resolved session name. t.Run("fixed agent", func(t *testing.T) { @@ -654,6 +725,9 @@ func TestDoSlingBeadToPool(t *testing.T) { if bead.Metadata["gc.routed_to"] != "hello-world/polecat" { t.Errorf("gc.routed_to = %q, want hello-world/polecat", bead.Metadata["gc.routed_to"]) } + if bead.Assignee != "" { + t.Errorf("assignee = %q, want empty (pool agents race via --unassigned, not direct assignment)", bead.Assignee) + } } func TestDoSlingRefusesCrossStoreRoute(t *testing.T) { @@ -770,6 +844,158 @@ func TestCliBeadRouterAllowsCityTargetFromCityStore(t *testing.T) { } } +// seedSlingPoolRepourBead materializes a bead in the test store with the +// given status/assignee and routes it to hello-world/polecat, mirroring a +// handback or stale-claim shape ahead of a bare re-pour. +func seedSlingPoolRepourBead(t *testing.T, store beads.Store, id, status, assignee string) { + t.Helper() + if err := store.Update(id, beads.UpdateOpts{Status: &status, Assignee: &assignee}); err != nil { + t.Fatalf("seed status/assignee on %s: %v", id, err) + } + if err := store.SetMetadata(id, "gc.routed_to", "hello-world/polecat"); err != nil { + t.Fatalf("seed gc.routed_to on %s: %v", id, err) + } +} + +// TestDoSlingPoolRepourClearsStaleOpenAssignee covers gc-q40pm Problem A: +// a bead handed back to a named session (assignee=keeper, status=open, +// gc.routed_to=) is re-poured with a bare `gc sling `. +// The re-pour must clear the stale assignee so the bead matches the shared +// pool-demand predicate (bd ready --unassigned --metadata-field +// gc.routed_to=) that drives both reconciler spawn (scale_check) and +// worker claim (work_query Tier 3). Before the fix the bare re-pour only +// rewrote gc.routed_to to the same value — a supervisor no-op that stalled +// the chain silently. +func TestDoSlingPoolRepourClearsStaleOpenAssignee(t *testing.T) { + runner := newFakeRunner() + sp := runtime.NewFake() + cfg := &config.City{Workspace: config.Workspace{Name: "test-city"}} + a := config.Agent{ + Name: "polecat", + Dir: "hello-world", + MinActiveSessions: intPtr(1), MaxActiveSessions: intPtr(3), + } + + deps, stdout, stderr := testDeps(cfg, sp, runner.run) + seedSlingPoolRepourBead(t, deps.Store, "HW-8", "open", "hello-world/keeper") + + opts := testOpts(a, "HW-8") + if code := doSling(opts, deps, nil, stdout, stderr); code != 0 { + t.Fatalf("doSling returned %d, want 0; stderr: %s", code, stderr.String()) + } + if strings.Contains(stdout.String(), "idempotent") { + t.Fatalf("re-pour short-circuited as idempotent; stdout: %s", stdout.String()) + } + bead, err := deps.Store.Get("HW-8") + if err != nil { + t.Fatalf("store.Get(HW-8): %v", err) + } + if bead.Assignee != "" { + t.Errorf("assignee = %q, want empty (re-pour must restore the unassigned pool-demand shape)", bead.Assignee) + } + if bead.Status != "open" { + t.Errorf("status = %q, want open", bead.Status) + } + if bead.Metadata["gc.routed_to"] != "hello-world/polecat" { + t.Errorf("gc.routed_to = %q, want hello-world/polecat", bead.Metadata["gc.routed_to"]) + } +} + +// TestDoSlingPoolRepourClearsPoolTemplateAssignee covers the residue shape a +// hand-walked recovery leaves behind: assignee set to the pool template +// itself. Per dispatch.md, "assigning the pool template itself is not pool +// demand" — no scale_check counts it and no work_query tier claims it for a +// multi-instance pool — so a re-pour must clear it rather than short-circuit +// as idempotent (the pre-fix behavior). +func TestDoSlingPoolRepourClearsPoolTemplateAssignee(t *testing.T) { + runner := newFakeRunner() + sp := runtime.NewFake() + cfg := &config.City{Workspace: config.Workspace{Name: "test-city"}} + a := config.Agent{ + Name: "polecat", + Dir: "hello-world", + MinActiveSessions: intPtr(1), MaxActiveSessions: intPtr(3), + } + + deps, stdout, stderr := testDeps(cfg, sp, runner.run) + seedSlingPoolRepourBead(t, deps.Store, "HW-9", "open", "hello-world/polecat") + + opts := testOpts(a, "HW-9") + if code := doSling(opts, deps, nil, stdout, stderr); code != 0 { + t.Fatalf("doSling returned %d, want 0; stderr: %s", code, stderr.String()) + } + bead, err := deps.Store.Get("HW-9") + if err != nil { + t.Fatalf("store.Get(HW-9): %v", err) + } + if bead.Assignee != "" { + t.Errorf("assignee = %q, want empty (template-assigned beads are unspawnable and unclaimable)", bead.Assignee) + } +} + +// TestDoSlingPoolRepourLeavesInProgressClaim pins the safety boundary: an +// in-progress bead is a live claim (bd update --claim flips open→in_progress +// atomically), so a bare re-pour must NOT strip its assignee out from under +// the owning session. Reclaiming dead claims is the orphan-release path's +// job, not sling's. +func TestDoSlingPoolRepourLeavesInProgressClaim(t *testing.T) { + runner := newFakeRunner() + sp := runtime.NewFake() + cfg := &config.City{Workspace: config.Workspace{Name: "test-city"}} + a := config.Agent{ + Name: "polecat", + Dir: "hello-world", + MinActiveSessions: intPtr(1), MaxActiveSessions: intPtr(3), + } + + deps, stdout, stderr := testDeps(cfg, sp, runner.run) + seedSlingPoolRepourBead(t, deps.Store, "HW-10", "in_progress", "polecat-lo-session-abc") + + opts := testOpts(a, "HW-10") + if code := doSling(opts, deps, nil, stdout, stderr); code != 0 { + t.Fatalf("doSling returned %d, want 0; stderr: %s", code, stderr.String()) + } + bead, err := deps.Store.Get("HW-10") + if err != nil { + t.Fatalf("store.Get(HW-10): %v", err) + } + if bead.Assignee != "polecat-lo-session-abc" { + t.Errorf("assignee = %q, want polecat-lo-session-abc (in-progress claims must survive a re-pour)", bead.Assignee) + } + if bead.Status != "in_progress" { + t.Errorf("status = %q, want in_progress", bead.Status) + } +} + +// TestDoSlingPoolRepourDryRunLeavesAssignee pins that --dry-run previews the +// re-pour without mutating the bead. +func TestDoSlingPoolRepourDryRunLeavesAssignee(t *testing.T) { + runner := newFakeRunner() + sp := runtime.NewFake() + cfg := &config.City{Workspace: config.Workspace{Name: "test-city"}} + a := config.Agent{ + Name: "polecat", + Dir: "hello-world", + MinActiveSessions: intPtr(1), MaxActiveSessions: intPtr(3), + } + + deps, stdout, stderr := testDeps(cfg, sp, runner.run) + seedSlingPoolRepourBead(t, deps.Store, "HW-11", "open", "hello-world/keeper") + + opts := testOpts(a, "HW-11") + opts.DryRun = true + if code := doSling(opts, deps, nil, stdout, stderr); code != 0 { + t.Fatalf("doSling returned %d, want 0; stderr: %s", code, stderr.String()) + } + bead, err := deps.Store.Get("HW-11") + if err != nil { + t.Fatalf("store.Get(HW-11): %v", err) + } + if bead.Assignee != "hello-world/keeper" { + t.Errorf("assignee = %q, want hello-world/keeper (dry-run must not mutate)", bead.Assignee) + } +} + func TestDoSlingFormulaToAgent(t *testing.T) { runner := newFakeRunner() sp := runtime.NewFake() diff --git a/cmd/gc/cmd_start.go b/cmd/gc/cmd_start.go index 730d255c09..916196f5dc 100644 --- a/cmd/gc/cmd_start.go +++ b/cmd/gc/cmd_start.go @@ -1360,8 +1360,8 @@ func agentCommandDir(cityPath string, a *config.Agent, rigs []config.Rig) string } // providerProcessPassthroughEnv returns non-GC process context that provider -// sessions need to start reliably: user/home, provider auth/config, locale, -// XDG, telemetry, and Claude nesting resets. +// sessions need to start reliably: user/home, provider auth/config, ssh-agent +// forwarding, locale, XDG, telemetry, and Claude nesting resets. func providerProcessPassthroughEnv() map[string]string { return processenv.ProviderProcessPassthroughEnv() } @@ -1370,6 +1370,7 @@ func providerProcessPassthroughEnv() map[string]string { // agent sessions should inherit. Agents need PATH to find tools (including gc), // GC_BEADS/GC_DOLT so they use the same bead store as the parent, // GC_DOLT_HOST/PORT/USER/PASSWORD so agents can connect to remote Dolt servers, +// SSH_AUTH_SOCK so they can sign commits with the operator's ssh-agent, // and Claude auth/home context so managed sessions can launch reliably under // shell and supervisor-driven flows. func passthroughEnv() map[string]string { @@ -1384,16 +1385,25 @@ func passthroughEnv() map[string]string { return m } -// expandEnvMap returns a copy of m with os.ExpandEnv applied to each value. -// This allows TOML-sourced env blocks to reference the controller's environment, -// e.g. DOLTHUB_TOKEN = "$DOLTHUB_TOKEN". -func expandEnvMap(m map[string]string) map[string]string { +// expandEnvMap returns a copy of m with each value expanded using ${VAR}/$VAR +// syntax. References are looked up in src first, then fall back to +// os.Environ() so existing TOML blocks like DOLTHUB_TOKEN = "$DOLTHUB_TOKEN" +// still resolve from the controller's environment. src lets callers inject +// values the supervisor doesn't carry (e.g. GT_ROOT, GC_RIG, GC_RIG_ROOT +// that template_resolve.go computes into agentEnv) without leaking them +// into os.Environ. A nil src is treated as an empty map. +func expandEnvMap(src, m map[string]string) map[string]string { if m == nil { return nil } out := make(map[string]string, len(m)) for k, v := range m { - out[k] = os.ExpandEnv(v) + out[k] = os.Expand(v, func(name string) string { + if val, ok := src[name]; ok { + return val + } + return os.Getenv(name) + }) } return out } diff --git a/cmd/gc/cmd_start_test.go b/cmd/gc/cmd_start_test.go index 48cde8b456..646e94473f 100644 --- a/cmd/gc/cmd_start_test.go +++ b/cmd/gc/cmd_start_test.go @@ -751,6 +751,26 @@ func TestPassthroughEnvOmitsUnsetDoltVars(t *testing.T) { } } +func TestPassthroughEnvIncludesSSHAuthSock(t *testing.T) { + t.Setenv("SSH_AUTH_SOCK", "/tmp/ssh-agent.sock") + + got := passthroughEnv() + + if got["SSH_AUTH_SOCK"] != "/tmp/ssh-agent.sock" { + t.Errorf("passthroughEnv()[SSH_AUTH_SOCK] = %q, want %q", got["SSH_AUTH_SOCK"], "/tmp/ssh-agent.sock") + } +} + +func TestPassthroughEnvOmitsUnsetSSHAuthSock(t *testing.T) { + t.Setenv("SSH_AUTH_SOCK", "") + + got := passthroughEnv() + + if _, ok := got["SSH_AUTH_SOCK"]; ok { + t.Error("passthroughEnv() should omit empty SSH_AUTH_SOCK") + } +} + func TestPassthroughEnvIncludesClaudeAuthContext(t *testing.T) { t.Setenv("HOME", "/tmp/gc-home") t.Setenv("USER", "gcuser") diff --git a/cmd/gc/cmd_supervisor.go b/cmd/gc/cmd_supervisor.go index c519e5c088..14c9404eba 100644 --- a/cmd/gc/cmd_supervisor.go +++ b/cmd/gc/cmd_supervisor.go @@ -2048,6 +2048,7 @@ func reconcileCities( configRev := config.Revision(fsys.OSFS{}, prov, cfg, path) pokeCh := make(chan struct{}, 1) configDirty := &atomic.Bool{} + demandDirty := &atomic.Bool{} forceShutdown := &atomic.Bool{} reloadReqCh := make(chan reloadRequest) cityCtx, cityCancel := context.WithCancel(context.Background()) @@ -2066,6 +2067,7 @@ func reconcileCities( WatchTargets: watchTargets, ConfigRev: configRev, ConfigDirty: configDirty, + DemandDirty: demandDirty, Cfg: cfg, SP: sp, Publication: publication, @@ -2116,6 +2118,7 @@ func reconcileCities( cs.ct = cityRuntime.crashTrack() cs.pokeCh = pokeCh cs.configDirty = configDirty + cs.demandDirty = demandDirty cs.services = cityRuntime.svc cityRuntime.setControllerState(cs) cs.startBeadEventWatcher(cityCtx) @@ -2186,7 +2189,7 @@ func reconcileCities( // Start controller socket AFTER the alreadyRunning check so we // never destroy a live city's socket or leak a listener. sockPath := filepath.Join(path, ".gc", "controller.sock") - lis, lisErr := startControllerSocket(path, cityCancel, forceShutdown, configDirty, reloadReqCh, convergenceReqCh, pokeCh, controlDispatcherCh) + lis, lisErr := startControllerSocket(path, cityCancel, forceShutdown, configDirty, demandDirty, reloadReqCh, convergenceReqCh, pokeCh, controlDispatcherCh) if lisErr != nil { fmt.Fprintf(stderr, "gc supervisor: city '%s': controller socket: %v\n", cityName, lisErr) //nolint:errcheck lock.Close() //nolint:errcheck // no socket to race with diff --git a/cmd/gc/cmd_trace_test.go b/cmd/gc/cmd_trace_test.go index 8c996c533f..43dea34c12 100644 --- a/cmd/gc/cmd_trace_test.go +++ b/cmd/gc/cmd_trace_test.go @@ -235,7 +235,7 @@ func TestTraceControllerSocketInvalidRequestDoesNotPoke(t *testing.T) { done := make(chan struct{}) go func() { - handleControllerConn(server, cityDir, func() {}, nil, nil, nil, convergenceReqCh, pokeCh, controlDispatcherCh) + handleControllerConn(server, cityDir, func() {}, nil, nil, nil, nil, convergenceReqCh, pokeCh, controlDispatcherCh) close(done) }() @@ -396,7 +396,7 @@ func sendTraceSocketCommand(t *testing.T, cityDir, command string, req traceCont done := make(chan struct{}) go func() { - handleControllerConn(server, cityDir, func() {}, nil, nil, nil, convergenceReqCh, pokeCh, controlDispatcherCh) + handleControllerConn(server, cityDir, func() {}, nil, nil, nil, nil, convergenceReqCh, pokeCh, controlDispatcherCh) close(done) }() @@ -427,7 +427,7 @@ func sendTraceStatusSocketCommand(t *testing.T, cityDir string, pokeCh chan stru done := make(chan struct{}) go func() { - handleControllerConn(server, cityDir, func() {}, nil, nil, nil, convergenceReqCh, pokeCh, controlDispatcherCh) + handleControllerConn(server, cityDir, func() {}, nil, nil, nil, nil, convergenceReqCh, pokeCh, controlDispatcherCh) close(done) }() diff --git a/cmd/gc/compute_awake_set.go b/cmd/gc/compute_awake_set.go index f23b99d90d..956e8ef3ee 100644 --- a/cmd/gc/compute_awake_set.go +++ b/cmd/gc/compute_awake_set.go @@ -195,7 +195,16 @@ func ComputeAwakeSet(input AwakeInput) map[string]AwakeDecision { } if sn := resolveNamedSessionBeadName(input.SessionBeads, ns); sn != "" { bead := findBeadBySessionName(input.SessionBeads, sn) - if bead != nil && !bead.DependencyOnly && !bead.Drained && bead.State != "closed" { + // Admit even a drained bead: control reaches here only when a + // demand reason was set above (the switch continues otherwise), + // so genuine work exists. Gating on !bead.Drained here deadlocks + // an on_demand session that drained via drain-ack and then had + // work assigned — nothing clears Drained for the new work, so it + // stays asleep forever despite recognized demand (gc-lqzwu, + // gascity gc-155rj stranded ~6.5h). This mirrors the always-mode + // path above, which has no drained gate. A drained session with + // NO demand still stays asleep via the switch default. + if bead != nil && !bead.DependencyOnly && bead.State != "closed" { desired[sn] = reason } } else { diff --git a/cmd/gc/compute_awake_set_test.go b/cmd/gc/compute_awake_set_test.go index 3988f3fea9..1c8e7a77ed 100644 --- a/cmd/gc/compute_awake_set_test.go +++ b/cmd/gc/compute_awake_set_test.go @@ -446,6 +446,35 @@ func TestNamedOnDemand_NamedSessionDemandWakesIdleAsleepSession(t *testing.T) { assertReason(t, result, "hello-world--refinery", "named-demand") } +func TestNamedOnDemand_DrainedWithNamedSessionDemandWakes(t *testing.T) { + // Regression for gc-lqzwu: an on_demand named session that has drained + // (e.g. a refinery that called drain-ack) must re-wake when work is + // assigned afterward. The demand-driven admit previously dropped the bead + // via a !Drained gate, so the session stayed drained forever despite + // recognized NamedSessionDemand — a hard deadlock (gascity gc-155rj + // stranded ~6.5h). This mirrors the always-mode path, which already wakes + // drained beads (see TestNamedAlways_DrainedCompatibilityStateStillWakes). + // A drained bead carries no detached_at, so IdleSince is zero and the + // downstream idle-sleep block does not re-suppress the wake. + result := ComputeAwakeSet(AwakeInput{ + Agents: []AwakeAgent{{QualifiedName: "hello-world/refinery"}}, + NamedSessions: []AwakeNamedSession{{Identity: "hello-world/refinery", Template: "hello-world/refinery", Mode: "on_demand"}}, + SessionBeads: []AwakeSessionBead{{ + ID: "mc-1", + SessionName: "hello-world--refinery", + Template: "hello-world/refinery", + State: "asleep", + SleepReason: "drained", + NamedIdentity: "hello-world/refinery", + Drained: true, + }}, + NamedSessionDemand: map[string]bool{"hello-world/refinery": true}, + Now: now, + }) + assertAwake(t, result, "hello-world--refinery") + assertReason(t, result, "hello-world--refinery", "named-demand") +} + // TestNamedOnDemand_WorkQueryWakesIdleAsleepSession is the work-query variant // of #3413: a named session whose backing template's work-query found pending // work (NamedSessionWorkQ) must also wake despite being idle past timeout. @@ -462,6 +491,29 @@ func TestNamedOnDemand_WorkQueryWakesIdleAsleepSession(t *testing.T) { assertReason(t, result, "hello-world--refinery", "work-query") } +func TestNamedOnDemand_DrainedWithoutDemandStaysAsleep(t *testing.T) { + // Teardown guarantee paired with the regression above: relaxing the + // !Drained gate must NOT keep a drained on_demand session awake when there + // is no demand. With NamedSessionDemand and NamedSessionWorkQ both absent, + // the on_demand branch hits its switch default (continue) before reaching + // the bead admit, so the drained session stays asleep. + result := ComputeAwakeSet(AwakeInput{ + Agents: []AwakeAgent{{QualifiedName: "hello-world/refinery"}}, + NamedSessions: []AwakeNamedSession{{Identity: "hello-world/refinery", Template: "hello-world/refinery", Mode: "on_demand"}}, + SessionBeads: []AwakeSessionBead{{ + ID: "mc-1", + SessionName: "hello-world--refinery", + Template: "hello-world/refinery", + State: "asleep", + SleepReason: "drained", + NamedIdentity: "hello-world/refinery", + Drained: true, + }}, + Now: now, + }) + assertAsleep(t, result, "hello-world--refinery") +} + func TestNamedOnDemand_PendingCreateWakesWithoutDemand(t *testing.T) { result := ComputeAwakeSet(AwakeInput{ Agents: []AwakeAgent{{QualifiedName: "hello-world/refinery"}}, diff --git a/cmd/gc/context_inject.go b/cmd/gc/context_inject.go index c4c30be7b7..fa5a14260b 100644 --- a/cmd/gc/context_inject.go +++ b/cmd/gc/context_inject.go @@ -7,6 +7,8 @@ import ( "os" "strconv" "strings" + + "github.com/gastownhall/gascity/internal/modelwindow" ) // Context-usage injection — the context-pressure sibling of clock_inject.go. @@ -123,11 +125,14 @@ func lastTranscriptUsage(path string) (tokens int, models []string, ok bool) { } // contextWindowTokens resolves the session's context window as the MAX window -// of any model it ran (they share one context), so a 200k-window sidecar or -// compaction call (e.g. a bare claude-opus-4-8 entry inside a Fable session) -// can't flip a 1M session to the 200k default and fire the urgent tier at -// ~20% of real usage. GC_CONTEXT_WINDOW_TOKENS overrides — gc-managed -// deployments that know the launch model should pin it for determinism. +// of any model it ran (they share one context), so a smaller-window sidecar or +// compaction call (e.g. a 200k-window Haiku entry inside a 1M Fable session) +// can't flip the session to the 200k default and fire the urgent tier at ~20% +// of real usage. Per-model windows come from the shared modelwindow package so +// this agrees with the API/session-log path; an unrecognized model (window 0) +// floors to the conservative default. GC_CONTEXT_WINDOW_TOKENS overrides — +// gc-managed deployments that know the launch model should pin it for +// determinism. func contextWindowTokens(models []string) int { if v := strings.TrimSpace(os.Getenv("GC_CONTEXT_WINDOW_TOKENS")); v != "" { if n, err := strconv.Atoi(v); err == nil && n > 0 { @@ -136,32 +141,16 @@ func contextWindowTokens(models []string) int { } best := 0 for _, m := range models { - if w := classifyWindow(m); w > best { + if w := modelwindow.Window(m); w > best { best = w } } if best == 0 { - return 200_000 + return modelwindow.Default } return best } -// classifyWindow maps one model string to its context window. 1M families: -// Opus 4.6/4.7/4.8, Sonnet 4.6, Fable, Mythos, and an explicit [1m] launch -// suffix; everything else (Haiku, older models, unrecognized) is a -// conservative 200k. Kept simple/substring rather than a strict table so a -// dated-suffix variant still matches; pin GC_CONTEXT_WINDOW_TOKENS when a new -// model's window isn't yet recognized here. -func classifyWindow(model string) int { - ml := strings.ToLower(model) - for _, s := range []string{"[1m]", "fable", "mythos", "opus-4-6", "opus-4-7", "opus-4-8", "sonnet-4-6"} { - if strings.Contains(ml, s) { - return 1_000_000 - } - } - return 200_000 -} - // contextUsageMessage renders the guidance line for tokens used of window, or // "" below the advisory threshold. func contextUsageMessage(tokens, window int) string { diff --git a/cmd/gc/controller.go b/cmd/gc/controller.go index b73ee6fb77..a1df7390f1 100644 --- a/cmd/gc/controller.go +++ b/cmd/gc/controller.go @@ -126,6 +126,7 @@ func startControllerSocket( cancelFn context.CancelFunc, forceShutdown *atomic.Bool, dirty *atomic.Bool, + demandDirty *atomic.Bool, reloadReqCh chan reloadRequest, convergenceReqCh chan convergenceRequest, pokeCh chan struct{}, @@ -147,7 +148,7 @@ func startControllerSocket( if err != nil { return // listener closed } - go handleControllerConn(conn, cityPath, cancelFn, forceShutdown, dirty, reloadReqCh, convergenceReqCh, pokeCh, controlDispatcherCh) + go handleControllerConn(conn, cityPath, cancelFn, forceShutdown, dirty, demandDirty, reloadReqCh, convergenceReqCh, pokeCh, controlDispatcherCh) } }() return lis, nil @@ -163,6 +164,7 @@ func handleControllerConn( cancelFn context.CancelFunc, forceShutdown *atomic.Bool, dirty *atomic.Bool, + demandDirty *atomic.Bool, reloadReqCh chan reloadRequest, convergenceReqCh chan convergenceRequest, pokeCh chan struct{}, @@ -195,6 +197,18 @@ func handleControllerConn( default: // poke already pending } conn.Write([]byte("ok\n")) //nolint:errcheck // best-effort ack + case line == "poke-demand": + // Demand-aware wake: mark demand dirty so the tick rebuilds pool + // demand, then trigger it. Plain "poke" stays demand-snapshot- + // preserving for the high-frequency session-lifecycle path. + if demandDirty != nil { + demandDirty.Store(true) + } + select { + case pokeCh <- struct{}{}: + default: // poke already pending + } + conn.Write([]byte("ok\n")) //nolint:errcheck // best-effort ack case line == "reload": if dirty != nil { dirty.Store(true) @@ -1270,10 +1284,11 @@ func runController( pokeCh := make(chan struct{}, 1) controlDispatcherCh := make(chan struct{}, 1) configDirty := &atomic.Bool{} + demandDirty := &atomic.Bool{} sockPath := controllerSocketPath(cityPath) forceShutdown := &atomic.Bool{} - lis, err := startControllerSocket(cityPath, cancel, forceShutdown, configDirty, reloadReqCh, convergenceReqCh, pokeCh, controlDispatcherCh) + lis, err := startControllerSocket(cityPath, cancel, forceShutdown, configDirty, demandDirty, reloadReqCh, convergenceReqCh, pokeCh, controlDispatcherCh) if err != nil { fmt.Fprintf(stderr, "gc start: %v\n", err) //nolint:errcheck // best-effort stderr return 1 @@ -1311,6 +1326,7 @@ func runController( WatchTargets: initialWatchTargets, ConfigRev: configRev, ConfigDirty: configDirty, + DemandDirty: demandDirty, Cfg: cfg, SP: sp, Publication: supervisor.PublicationConfig{}, @@ -1336,6 +1352,7 @@ func runController( cs.ct = cr.crashTrack() cs.pokeCh = pokeCh cs.configDirty = configDirty + cs.demandDirty = demandDirty cs.services = cr.svc cs.emergencyCh = make(chan emergency.Record, 64) cr.setControllerState(cs) diff --git a/cmd/gc/controller_test.go b/cmd/gc/controller_test.go index c584a08597..025ddfe384 100644 --- a/cmd/gc/controller_test.go +++ b/cmd/gc/controller_test.go @@ -245,7 +245,8 @@ func TestControllerSocketFallbackUsesShortPathForLongCityPath(t *testing.T) { pokeCh := make(chan struct{}, 1) controlDispatcherCh := make(chan struct{}, 1) configDirty := &atomic.Bool{} - lis, err := startControllerSocket(cityPath, cancel, nil, configDirty, nil, convergenceReqCh, pokeCh, controlDispatcherCh) + demandDirty := &atomic.Bool{} + lis, err := startControllerSocket(cityPath, cancel, nil, configDirty, demandDirty, nil, convergenceReqCh, pokeCh, controlDispatcherCh) if err != nil { t.Fatalf("startControllerSocket: %v", err) } @@ -276,6 +277,25 @@ func TestControllerSocketFallbackUsesShortPathForLongCityPath(t *testing.T) { default: t.Fatal("reload did not enqueue poke") } + + // poke-demand marks the demand snapshot dirty (so a sleeping pool's demand + // rebuilds this tick) in addition to enqueuing a poke (gc-lskvo). + resp, err = sendControllerCommand(cityPath, "poke-demand") + if err != nil { + t.Fatalf("sendControllerCommand(poke-demand): %v", err) + } + if strings.TrimSpace(string(resp)) != "ok" { + t.Fatalf("poke-demand response = %q, want ok", resp) + } + if !demandDirty.Load() { + t.Fatal("demandDirty = false, want poke-demand to mark demand dirty") + } + select { + case <-pokeCh: + default: + t.Fatal("poke-demand did not enqueue poke") + } + if !tryStopController(cityPath, &bytes.Buffer{}) { t.Fatal("tryStopController returned false, want true via fallback socket") } @@ -1301,7 +1321,7 @@ func TestHandleControllerConnControlDispatcher(t *testing.T) { done := make(chan struct{}) go func() { - handleControllerConn(server, cityPath, func() {}, nil, nil, nil, convergenceReqCh, pokeCh, controlDispatcherCh) + handleControllerConn(server, cityPath, func() {}, nil, nil, nil, nil, convergenceReqCh, pokeCh, controlDispatcherCh) close(done) }() diff --git a/cmd/gc/doctor_session_model.go b/cmd/gc/doctor_session_model.go index 7edaac8c49..61ead3cc05 100644 --- a/cmd/gc/doctor_session_model.go +++ b/cmd/gc/doctor_session_model.go @@ -42,6 +42,8 @@ func (c *sessionModelDoctorCheck) Run(_ *doctor.CheckContext) *doctor.CheckResul return r } + knownPrefixes := knownBeadPrefixes(c.cfg) + sessionByID := make(map[string]beads.Bead) openSessionAlias := make(map[string][]beads.Bead) openSessionAliasHistory := make(map[string][]beads.Bead) @@ -78,7 +80,7 @@ func (c *sessionModelDoctorCheck) Run(_ *doctor.CheckContext) *doctor.CheckResul } else if isRetiredSessionModelOwner(owner) { findings = append(findings, fmt.Sprintf("retired-bead-owner: %s is assigned to retired session bead %s", b.ID, assignee)) } - } else if looksLikeSessionBeadID(assignee) { + } else if looksLikeSessionBeadID(assignee, knownPrefixes) { findings = append(findings, fmt.Sprintf("missing-bead-owner: %s is assigned to missing session bead %s", b.ID, assignee)) } else { matches := legacySessionTokenMatches(assignee, openSessionAlias, openSessionName) @@ -189,8 +191,50 @@ func isRetiredSessionModelOwner(b beads.Bead) bool { return session.LifecycleIdentityReleased(b.Status, b.Metadata) } -func looksLikeSessionBeadID(s string) bool { - return strings.HasPrefix(s, "gc-") || strings.HasPrefix(s, "bd-") || strings.HasPrefix(s, "mc-") +// looksLikeSessionBeadID reports whether s is shaped like a bead ID we +// should resolve through sessionByID. Classification is closed-set: only +// strings of the form "-" where appears in +// knownPrefixes are treated as bead IDs. Strings containing "/" or "." +// are rejected up front as session-name shapes that no bead ID would +// ever exhibit. This guards against false-positive "missing-bead-owner" +// findings for rig-qualified session names like +// "gc-toolkit/gastown.witness" or "gc-toolkit.mechanik" even when the +// leading segment matches a historical bead-ID prefix. +func looksLikeSessionBeadID(s string, knownPrefixes map[string]bool) bool { + if s == "" { + return false + } + if strings.ContainsRune(s, '/') || strings.ContainsRune(s, '.') { + return false + } + for prefix := range knownPrefixes { + if prefix == "" { + continue + } + if strings.HasPrefix(s, prefix+"-") { + return true + } + } + return false +} + +// knownBeadPrefixes returns the set of bead-ID prefixes the city and its +// rigs use, suitable for closed-set classification by +// looksLikeSessionBeadID. Empty prefixes are skipped. +func knownBeadPrefixes(cfg *config.City) map[string]bool { + prefixes := make(map[string]bool) + if hq := strings.TrimSpace(config.EffectiveHQPrefix(cfg)); hq != "" { + prefixes[hq] = true + } + if cfg == nil { + return prefixes + } + for i := range cfg.Rigs { + if p := strings.TrimSpace(cfg.Rigs[i].EffectivePrefix()); p != "" { + prefixes[p] = true + } + } + return prefixes } func legacySessionTokenMatches(token string, byAlias, bySessionName map[string][]beads.Bead) []beads.Bead { diff --git a/cmd/gc/doctor_session_model_test.go b/cmd/gc/doctor_session_model_test.go index 9503990a0c..0fd27561f3 100644 --- a/cmd/gc/doctor_session_model_test.go +++ b/cmd/gc/doctor_session_model_test.go @@ -1,9 +1,12 @@ package main import ( + "bytes" + "strings" "testing" "github.com/gastownhall/gascity/internal/beads" + "github.com/gastownhall/gascity/internal/config" "github.com/gastownhall/gascity/internal/session" ) @@ -75,3 +78,122 @@ func (s *doctorListSpyStore) List(query beads.ListQuery) ([]beads.Bead, error) { } return s.MemStore.List(query) } + +func TestLooksLikeSessionBeadIDRejectsSessionNames(t *testing.T) { + defaultPrefixes := map[string]bool{"gc": true, "bd": true, "mc": true, "lx": true} + cases := []struct { + name string + in string + prefixes map[string]bool + want bool + }{ + {"plain bead id with known prefix", "gc-119r", defaultPrefixes, true}, + {"workspace prefix bead id", "lx-v2yp1", defaultPrefixes, true}, + {"bd-prefixed bead id with known prefix", "bd-abc12", defaultPrefixes, true}, + {"mc-prefixed bead id with known prefix", "mc-abc12", defaultPrefixes, true}, + {"prefix not in known set", "lx-v2yp1", map[string]bool{"gc": true}, false}, + {"unknown prefix entirely", "qq-abc12", defaultPrefixes, false}, + {"rig-qualified session name", "gc-toolkit/gastown.witness", defaultPrefixes, false}, + {"role-qualified session name", "mayor/refinery", defaultPrefixes, false}, + {"plain rig-prefixed session", "gc-foo/bar", defaultPrefixes, false}, + {"dot-separated rig-qualified", "gc-toolkit.mechanik", defaultPrefixes, false}, + {"double-hyphen alias", "gascity--control-dispatcher", defaultPrefixes, false}, + {"bare alias", "mechanik", defaultPrefixes, false}, + {"hyphen alias", "control-dispatcher", defaultPrefixes, false}, + {"unrelated prefix", "agent-diagnostics-h1", defaultPrefixes, false}, + {"empty string", "", defaultPrefixes, false}, + {"empty prefix set", "gc-119r", map[string]bool{}, false}, + {"nil prefix set", "gc-119r", nil, false}, + {"empty string entry in set is skipped", "gc-119r", map[string]bool{"": true}, false}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + if got := looksLikeSessionBeadID(tt.in, tt.prefixes); got != tt.want { + t.Errorf("looksLikeSessionBeadID(%q, %v) = %v, want %v", tt.in, tt.prefixes, got, tt.want) + } + }) + } +} + +func TestKnownBeadPrefixesIncludesHQAndRigPrefixes(t *testing.T) { + cfg := &config.City{ + Workspace: config.Workspace{Name: "test-city", Prefix: "tc"}, + Rigs: []config.Rig{ + {Name: "gc-toolkit", Prefix: "tk"}, // explicit prefix wins + {Name: "gascity"}, // derived: "ga" + }, + } + got := knownBeadPrefixes(cfg) + wantKeys := []string{"tc", "tk", "ga"} + for _, k := range wantKeys { + if !got[k] { + t.Errorf("knownBeadPrefixes() missing %q (got %v)", k, got) + } + } + if got["gc-toolkit"] { + t.Errorf("knownBeadPrefixes() should not contain rig name %q (got %v)", "gc-toolkit", got) + } +} + +func TestKnownBeadPrefixesNilConfig(t *testing.T) { + got := knownBeadPrefixes(nil) + if len(got) != 0 { + t.Errorf("knownBeadPrefixes(nil) = %v, want empty", got) + } +} + +// TestPhase0DoctorDoesNotFalsePositiveOnRigQualifiedSessionName ensures that +// when an assignee is a rig-qualified session name (e.g. +// "gc-toolkit/gastown.witness" or "gc-toolkit.mechanik") whose live session +// bead exists in the store, the session-model check does not emit a +// "missing-bead-owner" finding for it. +// +// Reproduces gc-119r / gc-8ylt3: looksLikeSessionBeadID previously returned +// true for any string starting with a hardcoded prefix ("gc-", "bd-", "mc-"), +// causing rig names with those prefixes to be misclassified as session bead +// IDs. The fix in gc-119r added a "/" reject; gc-8ylt3 generalizes this to a +// closed-set classifier that also rejects "." separators. +func TestPhase0DoctorDoesNotFalsePositiveOnRigQualifiedSessionName(t *testing.T) { + cases := []struct { + name string + sessionName string + }{ + {"slash separator", "gc-toolkit/gastown.witness"}, + {"dot separator", "gc-toolkit.mechanik"}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + cityPath, store := newPhase0DoctorCity(t) + + witness, err := store.Create(beads.Bead{ + Type: session.BeadType, + Labels: []string{session.LabelSession}, + Metadata: map[string]string{ + "session_name": tt.sessionName, + "alias": tt.sessionName, + "template": "worker", + }, + }) + if err != nil { + t.Fatalf("create witness session bead: %v", err) + } + if _, err := store.Create(beads.Bead{ + Type: "task", + Status: "open", + Title: "wisp routed to live witness", + Assignee: tt.sessionName, + }); err != nil { + t.Fatalf("create wisp bead: %v", err) + } + + t.Setenv("GC_CITY", cityPath) + var stdout, stderr bytes.Buffer + _ = doDoctor(false, true, false, false, &stdout, &stderr) + + out := stdout.String() + stderr.String() + if strings.Contains(out, "missing-bead-owner") { + t.Fatalf("doctor falsely reported missing-bead-owner for live session %s:\n%s", witness.ID, out) + } + }) + } +} diff --git a/cmd/gc/doctor_stuck_creating.go b/cmd/gc/doctor_stuck_creating.go new file mode 100644 index 0000000000..4ab11602fc --- /dev/null +++ b/cmd/gc/doctor_stuck_creating.go @@ -0,0 +1,236 @@ +package main + +import ( + "fmt" + "strings" + "time" + + "github.com/gastownhall/gascity/internal/beads" + "github.com/gastownhall/gascity/internal/config" + "github.com/gastownhall/gascity/internal/doctor" + "github.com/gastownhall/gascity/internal/session" +) + +// stuckCreatingWarnAfter is how long a session may sit in state=creating +// before gc doctor warns. Healthy provider starts complete well under this +// bound, and the reconciler's stale-creating recovery normally clears wedged +// creates within about a minute — a session still creating after 3 minutes +// has already outlived auto-recovery. +const stuckCreatingWarnAfter = 3 * time.Minute + +// stuckCreatingFailAfter (2× the warn threshold) is when the check fails +// instead of warns: the session is unambiguously stuck and needs an operator. +const stuckCreatingFailAfter = 2 * stuckCreatingWarnAfter + +// stuckCreatingMessageNameCap bounds how many stuck identities the one-line +// summary message names before collapsing the rest into "+N more". The full +// list always appears in Details (verbose output). +const stuckCreatingMessageNameCap = 5 + +// stuckCreatingDoctorCheck reports sessions wedged in state=creating. The +// reconciler auto-recovers the common cases (reapStaleSessionBeads); this +// check is the visibility net for the variants that don't auto-recover, so +// an operator or monitoring agent sees the stuck template without hand-rolling +// `gc session list --json` queries. +type stuckCreatingDoctorCheck struct { + cfg *config.City + cityPath string + newStore func(string) (beads.Store, error) + // now overrides the clock for tests; nil means time.Now. + now func() time.Time +} + +func (c *stuckCreatingDoctorCheck) Name() string { return "session-stuck-creating" } + +func (c *stuckCreatingDoctorCheck) CanFix() bool { return false } + +func (c *stuckCreatingDoctorCheck) Fix(_ *doctor.CheckContext) error { return nil } + +// stuckCreatingFinding captures one session bead wedged in state=creating. +type stuckCreatingFinding struct { + id string + identity string + age time.Duration + // ageKnown is false when the bead has neither a parseable + // pending_create_started_at marker nor a CreatedAt. + ageKnown bool + started time.Time +} + +func (f stuckCreatingFinding) detail() string { + if !f.ageKnown { + return fmt.Sprintf("%s (%s) in creating with no usable start timestamp (no pending_create_started_at, zero created_at); treated as stuck", f.id, f.identity) + } + return fmt.Sprintf("%s (%s) in creating for %s (started %s)", f.id, f.identity, f.age.Truncate(time.Second), f.started.Format(time.RFC3339)) +} + +func (c *stuckCreatingDoctorCheck) Run(_ *doctor.CheckContext) *doctor.CheckResult { + r := &doctor.CheckResult{Name: c.Name(), Status: doctor.StatusOK, Message: "no sessions stuck in creating state"} + if c == nil || c.newStore == nil { + return r + } + store, err := c.newStore(c.cityPath) + if err != nil { + r.Status = doctor.StatusWarning + r.Message = fmt.Sprintf("stuck-creating diagnostics skipped: %v", err) + return r + } + sessions, err := session.ListAllSessionBeads(store, beads.ListQuery{Sort: beads.SortCreatedAsc}) + if err != nil { + r.Status = doctor.StatusWarning + r.Message = fmt.Sprintf("stuck-creating diagnostics skipped: %v", err) + return r + } + now := time.Now().UTC() + if c.now != nil { + now = c.now() + } + + // Respect a configured startup_timeout larger than the default bands: the + // reconciler still treats a start within that window as valid, so neither + // warn nor fail until it has elapsed. Without this a city with a long + // startup_timeout (e.g. 12m) would see gc doctor block-fail healthy slow + // starts at the fixed 6m threshold. + warnAfter, failAfter := stuckCreatingWarnAfter, stuckCreatingFailAfter + if c.cfg != nil { + if st := c.cfg.Session.StartupTimeoutDuration(); st > warnAfter { + warnAfter = st + failAfter = st + stuckCreatingWarnAfter + } + } + + var failed, warned []stuckCreatingFinding + var allowlisted []string + for _, b := range sessions { + if b.Status == "closed" { + continue + } + if strings.TrimSpace(b.Metadata["state"]) != string(session.StateCreating) { + continue + } + f := stuckCreatingFinding{id: b.ID, identity: stuckCreatingIdentity(b)} + if started, ok := stuckCreatingStartedAt(b); ok { + f.started = started + f.age = now.Sub(started) + f.ageKnown = true + } + if stuckCreatingPreStartAllowlisted(c.cfg, b) { + // Templates with pre_start commands legitimately create slowly; + // excluded per spec, but surfaced in verbose output so a + // genuinely wedged pre_start session is still discoverable. + allowlisted = append(allowlisted, "allowlisted (pre_start configured): "+f.detail()) + continue + } + switch { + case !f.ageKnown: + // Nothing can ever age this bead out; mirror the reconciler's + // zero-CreatedAt handling and treat it as stuck now. + failed = append(failed, f) + case f.age >= failAfter: + failed = append(failed, f) + case f.age >= warnAfter: + warned = append(warned, f) + } + } + + // Most severe first so verbose output leads with the actionable lines; + // allowlisted notes trail as context. + var details []string + for _, f := range failed { + details = append(details, f.detail()) + } + for _, f := range warned { + details = append(details, f.detail()) + } + details = append(details, allowlisted...) + r.Details = details + + switch { + case len(failed) > 0: + r.Status = doctor.StatusError + r.Message = fmt.Sprintf("%d session(s) stuck in creating > %s: %s", len(failed), failAfter, stuckCreatingNameList(failed)) + if len(warned) > 0 { + r.Message += fmt.Sprintf("; %d more > %s: %s", len(warned), warnAfter, stuckCreatingNameList(warned)) + } + case len(warned) > 0: + r.Status = doctor.StatusWarning + r.Message = fmt.Sprintf("%d session(s) in creating > %s: %s", len(warned), warnAfter, stuckCreatingNameList(warned)) + } + if r.Status != doctor.StatusOK { + r.FixHint = "inspect stuck sessions with `gc session list --json`; see engdocs/contributors/reconciler-debugging.md for the reconciler diagnosis workflow" + } + return r +} + +// stuckCreatingStartedAt returns the timestamp anchoring how long the session +// has been in its current create attempt. It prefers the per-attempt +// pending_create_started_at marker over bead CreatedAt — the same preference +// the reconciler's staleness logic (isStaleCreating) uses — so doctor and +// reconciler agree about age. ok=false means neither anchor is usable. +func stuckCreatingStartedAt(b beads.Bead) (time.Time, bool) { + if t, ok := parseRFC3339Metadata(b.Metadata["pending_create_started_at"]); ok { + return t, true + } + if !b.CreatedAt.IsZero() { + return b.CreatedAt, true + } + return time.Time{}, false +} + +// stuckCreatingIdentity names the session for operator-facing output. The +// template metadata is the canonical config identity; alias and session_name +// are progressively weaker fallbacks for beads predating template stamping. +func stuckCreatingIdentity(b beads.Bead) string { + for _, v := range []string{ + strings.TrimSpace(b.Metadata["template"]), + strings.TrimSpace(b.Metadata["alias"]), + strings.TrimSpace(b.Metadata["session_name"]), + } { + if v != "" { + return v + } + } + return "unknown template" +} + +// stuckCreatingPreStartAllowlisted reports whether the session's backing +// agent template configures pre_start commands. Such templates legitimately +// spend long stretches in state=creating (heavy warmups run before the +// provider start completes), so the spec excludes them from stuck findings. +// The first identity that resolves to an agent decides; named-session +// identities resolve through their backing template. +func stuckCreatingPreStartAllowlisted(cfg *config.City, b beads.Bead) bool { + if cfg == nil { + return false + } + for _, identity := range []string{ + strings.TrimSpace(b.Metadata["template"]), + strings.TrimSpace(b.Metadata["alias"]), + } { + if identity == "" { + continue + } + if a := config.FindAgent(cfg, identity); a != nil { + return len(a.PreStart) > 0 + } + if ns := config.FindNamedSession(cfg, identity); ns != nil { + if a := config.FindAgent(cfg, ns.TemplateQualifiedName()); a != nil { + return len(a.PreStart) > 0 + } + } + } + return false +} + +// stuckCreatingNameList joins finding identities for the one-line summary, +// capped at stuckCreatingMessageNameCap names so a mass wedge stays readable. +func stuckCreatingNameList(findings []stuckCreatingFinding) string { + names := make([]string, 0, len(findings)) + for _, f := range findings { + names = append(names, f.identity) + } + if len(names) > stuckCreatingMessageNameCap { + return strings.Join(names[:stuckCreatingMessageNameCap], ", ") + fmt.Sprintf(", +%d more", len(names)-stuckCreatingMessageNameCap) + } + return strings.Join(names, ", ") +} diff --git a/cmd/gc/doctor_stuck_creating_test.go b/cmd/gc/doctor_stuck_creating_test.go new file mode 100644 index 0000000000..eef677f22a --- /dev/null +++ b/cmd/gc/doctor_stuck_creating_test.go @@ -0,0 +1,399 @@ +package main + +import ( + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/gastownhall/gascity/internal/beads" + "github.com/gastownhall/gascity/internal/config" + "github.com/gastownhall/gascity/internal/doctor" + "github.com/gastownhall/gascity/internal/session" +) + +func newStuckCreatingCheck(store beads.Store, cfg *config.City, now time.Time) *stuckCreatingDoctorCheck { + return &stuckCreatingDoctorCheck{ + cfg: cfg, + cityPath: "unused-city-path", + newStore: func(string) (beads.Store, error) { return store, nil }, + now: func() time.Time { return now }, + } +} + +func createStuckCreatingSessionBead(t *testing.T, store beads.Store, meta map[string]string) beads.Bead { + t.Helper() + merged := map[string]string{"state": string(session.StateCreating)} + for k, v := range meta { + merged[k] = v + } + b, err := store.Create(beads.Bead{ + Title: "session under test", + Type: session.BeadType, + Labels: []string{session.LabelSession}, + Metadata: merged, + }) + if err != nil { + t.Fatalf("Create(session bead): %v", err) + } + return b +} + +func TestStuckCreatingCheckOKWithNoStuckSessions(t *testing.T) { + now := time.Now().UTC() + cases := []struct { + name string + seed func(t *testing.T, store beads.Store) + }{ + {"no session beads", func(_ *testing.T, _ beads.Store) {}}, + {"active session", func(t *testing.T, store beads.Store) { + createStuckCreatingSessionBead(t, store, map[string]string{"state": "active"}) + }}, + {"creating under warn threshold", func(t *testing.T, store beads.Store) { + createStuckCreatingSessionBead(t, store, map[string]string{ + "pending_create_started_at": now.Add(-time.Minute).Format(time.RFC3339), + }) + }}, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + store := beads.NewMemStore() + tt.seed(t, store) + r := newStuckCreatingCheck(store, nil, now).Run(nil) + if r.Status != doctor.StatusOK { + t.Fatalf("Run() status = %v, want StatusOK (message %q, details %v)", r.Status, r.Message, r.Details) + } + }) + } +} + +func TestStuckCreatingCheckWarnsBetweenThresholds(t *testing.T) { + now := time.Now().UTC() + store := beads.NewMemStore() + b := createStuckCreatingSessionBead(t, store, map[string]string{ + "template": "gascity/worker", + "pending_create_started_at": now.Add(-4 * time.Minute).Format(time.RFC3339), + }) + + r := newStuckCreatingCheck(store, nil, now).Run(nil) + + if r.Status != doctor.StatusWarning { + t.Fatalf("Run() status = %v, want StatusWarning (message %q)", r.Status, r.Message) + } + if !strings.Contains(r.Message, "gascity/worker") { + t.Errorf("message %q does not name the stuck template", r.Message) + } + if !strings.Contains(r.Message, stuckCreatingWarnAfter.String()) { + t.Errorf("message %q does not mention warn threshold %s", r.Message, stuckCreatingWarnAfter) + } + if len(r.Details) == 0 || !strings.Contains(strings.Join(r.Details, "\n"), b.ID) { + t.Errorf("details %v do not reference stuck session bead %s", r.Details, b.ID) + } +} + +func TestStuckCreatingCheckFailsPastTwiceThreshold(t *testing.T) { + now := time.Now().UTC() + store := beads.NewMemStore() + b := createStuckCreatingSessionBead(t, store, map[string]string{ + "template": "gascity/worker", + "pending_create_started_at": now.Add(-7 * time.Minute).Format(time.RFC3339), + }) + + r := newStuckCreatingCheck(store, nil, now).Run(nil) + + if r.Status != doctor.StatusError { + t.Fatalf("Run() status = %v, want StatusError (message %q)", r.Status, r.Message) + } + if r.Severity != doctor.SeverityBlocking { + t.Errorf("Run() severity = %v, want SeverityBlocking", r.Severity) + } + if !strings.Contains(r.Message, "gascity/worker") { + t.Errorf("message %q does not name the stuck template", r.Message) + } + if !strings.Contains(r.Message, stuckCreatingFailAfter.String()) { + t.Errorf("message %q does not mention fail threshold %s", r.Message, stuckCreatingFailAfter) + } + if len(r.Details) == 0 || !strings.Contains(strings.Join(r.Details, "\n"), b.ID) { + t.Errorf("details %v do not reference stuck session bead %s", r.Details, b.ID) + } + if r.FixHint == "" { + t.Error("FixHint is empty; operators need a next step") + } +} + +// TestStuckCreatingCheckRespectsConfiguredStartupTimeout verifies that a slow +// start still within a configured [session].startup_timeout is not flagged +// (gc-c1rpx review P2): the fixed 3m/6m bands shift to begin at the timeout so +// the reconciler's valid-create window is honored before warn/fail fire. +func TestStuckCreatingCheckRespectsConfiguredStartupTimeout(t *testing.T) { + now := time.Now().UTC() + cfg := &config.City{Session: config.SessionConfig{StartupTimeout: "12m"}} + + t.Run("within startup_timeout is not flagged", func(t *testing.T) { + store := beads.NewMemStore() + // 7m would fail at the fixed 6m band, but the 12m startup_timeout means + // the reconciler still considers this start valid. + createStuckCreatingSessionBead(t, store, map[string]string{ + "template": "gascity/worker", + "pending_create_started_at": now.Add(-7 * time.Minute).Format(time.RFC3339), + }) + r := newStuckCreatingCheck(store, cfg, now).Run(nil) + if r.Status != doctor.StatusOK { + t.Fatalf("Run() status = %v, want StatusOK (start within 12m startup_timeout); message %q", r.Status, r.Message) + } + }) + + t.Run("past startup_timeout plus grace fails", func(t *testing.T) { + store := beads.NewMemStore() + // 16m exceeds startup_timeout (12m) + the warn grace (3m) = 15m fail band. + createStuckCreatingSessionBead(t, store, map[string]string{ + "template": "gascity/worker", + "pending_create_started_at": now.Add(-16 * time.Minute).Format(time.RFC3339), + }) + r := newStuckCreatingCheck(store, cfg, now).Run(nil) + if r.Status != doctor.StatusError { + t.Fatalf("Run() status = %v, want StatusError (past 12m+3m); message %q", r.Status, r.Message) + } + }) +} + +// TestStuckCreatingCheckPrefersPendingCreateMarker pins the age anchor to the +// per-attempt pending_create_started_at marker, not bead CreatedAt — the same +// preference the reconciler's staleness logic uses. A bead created long ago +// whose latest create attempt is recent must not be flagged. +func TestStuckCreatingCheckPrefersPendingCreateMarker(t *testing.T) { + base := time.Now().UTC() + store := beads.NewMemStore() + // CreatedAt is stamped ≈base by the store; the marker says the current + // attempt began 9 minutes later. At now=base+10m the attempt is only + // 1 minute old even though the bead itself is 10 minutes old. + createStuckCreatingSessionBead(t, store, map[string]string{ + "pending_create_started_at": base.Add(9 * time.Minute).Format(time.RFC3339), + }) + + r := newStuckCreatingCheck(store, nil, base.Add(10*time.Minute)).Run(nil) + + if r.Status != doctor.StatusOK { + t.Fatalf("Run() status = %v, want StatusOK; anchor must prefer pending_create_started_at (message %q)", r.Status, r.Message) + } +} + +func TestStuckCreatingCheckFallsBackToCreatedAt(t *testing.T) { + base := time.Now().UTC() + store := beads.NewMemStore() + b := createStuckCreatingSessionBead(t, store, nil) // no marker; CreatedAt ≈ base + + r := newStuckCreatingCheck(store, nil, base.Add(7*time.Minute)).Run(nil) + + if r.Status != doctor.StatusError { + t.Fatalf("Run() status = %v, want StatusError via CreatedAt fallback (message %q, details %v)", r.Status, r.Message, r.Details) + } + if !strings.Contains(strings.Join(r.Details, "\n"), b.ID) { + t.Errorf("details %v do not reference stuck session bead %s", r.Details, b.ID) + } +} + +// TestStuckCreatingCheckTreatsUnknownAnchorAsStuck covers corrupt beads with +// neither a parseable pending_create_started_at nor a CreatedAt: nothing can +// ever age them out, so the check reports them stuck — mirroring +// isStaleCreating's zero-CreatedAt handling. +func TestStuckCreatingCheckTreatsUnknownAnchorAsStuck(t *testing.T) { + store := beads.NewMemStoreFrom(1, []beads.Bead{{ + ID: "gc-corrupt", + Status: "open", + Type: session.BeadType, + Metadata: map[string]string{ + "state": string(session.StateCreating), + "pending_create_started_at": "not-a-timestamp", + }, + }}, nil) + + r := newStuckCreatingCheck(store, nil, time.Now().UTC()).Run(nil) + + if r.Status != doctor.StatusError { + t.Fatalf("Run() status = %v, want StatusError for unknown anchor (message %q)", r.Status, r.Message) + } + if !strings.Contains(strings.Join(r.Details, "\n"), "gc-corrupt") { + t.Errorf("details %v do not reference corrupt session bead", r.Details) + } +} + +func TestStuckCreatingCheckAllowlistsPreStartTemplates(t *testing.T) { + now := time.Now().UTC() + cfgWithPreStart := &config.City{ + Workspace: config.Workspace{Name: "demo"}, + Agents: []config.Agent{{ + Name: "worker", + Dir: "gascity", + PreStart: []string{"./slow-warmup.sh"}, + }}, + } + + t.Run("template with pre_start is excluded", func(t *testing.T) { + store := beads.NewMemStore() + b := createStuckCreatingSessionBead(t, store, map[string]string{ + "template": "gascity/worker", + "pending_create_started_at": now.Add(-30 * time.Minute).Format(time.RFC3339), + }) + + r := newStuckCreatingCheck(store, cfgWithPreStart, now).Run(nil) + + if r.Status != doctor.StatusOK { + t.Fatalf("Run() status = %v, want StatusOK for allowlisted template (message %q)", r.Status, r.Message) + } + if !strings.Contains(strings.Join(r.Details, "\n"), b.ID) { + t.Errorf("details %v do not surface allowlisted session %s for verbose visibility", r.Details, b.ID) + } + }) + + t.Run("alias resolves the template when template metadata is absent", func(t *testing.T) { + store := beads.NewMemStore() + createStuckCreatingSessionBead(t, store, map[string]string{ + "alias": "gascity/worker", + "pending_create_started_at": now.Add(-30 * time.Minute).Format(time.RFC3339), + }) + + r := newStuckCreatingCheck(store, cfgWithPreStart, now).Run(nil) + + if r.Status != doctor.StatusOK { + t.Fatalf("Run() status = %v, want StatusOK for allowlisted alias (message %q)", r.Status, r.Message) + } + }) + + t.Run("template without pre_start is still flagged", func(t *testing.T) { + cfg := &config.City{ + Workspace: config.Workspace{Name: "demo"}, + Agents: []config.Agent{{Name: "worker", Dir: "gascity"}}, + } + store := beads.NewMemStore() + createStuckCreatingSessionBead(t, store, map[string]string{ + "template": "gascity/worker", + "pending_create_started_at": now.Add(-7 * time.Minute).Format(time.RFC3339), + }) + + r := newStuckCreatingCheck(store, cfg, now).Run(nil) + + if r.Status != doctor.StatusError { + t.Fatalf("Run() status = %v, want StatusError for template without pre_start (message %q)", r.Status, r.Message) + } + }) +} + +func TestStuckCreatingCheckIgnoresClosedAndForeignStates(t *testing.T) { + now := time.Now().UTC() + store := beads.NewMemStore() + stale := now.Add(-time.Hour).Format(time.RFC3339) + + closed := createStuckCreatingSessionBead(t, store, map[string]string{ + "pending_create_started_at": stale, + }) + if err := store.Close(closed.ID); err != nil { + t.Fatalf("Close(%s): %v", closed.ID, err) + } + createStuckCreatingSessionBead(t, store, map[string]string{ + "state": "active", + "pending_create_started_at": stale, + }) + createStuckCreatingSessionBead(t, store, map[string]string{ + "state": "start_pending", + "pending_create_started_at": stale, + }) + + r := newStuckCreatingCheck(store, nil, now).Run(nil) + + if r.Status != doctor.StatusOK { + t.Fatalf("Run() status = %v, want StatusOK; closed/active/start_pending beads must be ignored (message %q, details %v)", r.Status, r.Message, r.Details) + } +} + +func TestStuckCreatingCheckSkipsWhenStoreUnavailable(t *testing.T) { + check := &stuckCreatingDoctorCheck{ + cityPath: "unused-city-path", + newStore: func(string) (beads.Store, error) { return nil, errors.New("dolt offline") }, + } + + r := check.Run(nil) + + if r.Status != doctor.StatusWarning { + t.Fatalf("Run() status = %v, want StatusWarning when store unavailable (message %q)", r.Status, r.Message) + } + if !strings.Contains(r.Message, "skipped") { + t.Errorf("message %q should say diagnostics were skipped", r.Message) + } +} + +func TestStuckCreatingCheckMessageCapsIdentityList(t *testing.T) { + now := time.Now().UTC() + store := beads.NewMemStore() + for i := 0; i < 7; i++ { + createStuckCreatingSessionBead(t, store, map[string]string{ + "template": fmt.Sprintf("gascity/worker-%d", i), + "pending_create_started_at": now.Add(-10 * time.Minute).Format(time.RFC3339), + }) + } + + r := newStuckCreatingCheck(store, nil, now).Run(nil) + + if r.Status != doctor.StatusError { + t.Fatalf("Run() status = %v, want StatusError (message %q)", r.Status, r.Message) + } + if !strings.Contains(r.Message, "+2 more") { + t.Errorf("message %q should cap the identity list at 5 names and summarize the rest", r.Message) + } + if len(r.Details) < 7 { + t.Errorf("details should list all 7 stuck sessions, got %d: %v", len(r.Details), r.Details) + } +} + +func TestStuckCreatingStartedAtAnchors(t *testing.T) { + created := time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC) + marker := time.Date(2026, 6, 1, 12, 30, 0, 0, time.UTC) + cases := []struct { + name string + bead beads.Bead + want time.Time + wantOK bool + }{ + { + name: "marker preferred over CreatedAt", + bead: beads.Bead{ + CreatedAt: created, + Metadata: map[string]string{"pending_create_started_at": marker.Format(time.RFC3339)}, + }, + want: marker, + wantOK: true, + }, + { + name: "CreatedAt fallback when marker missing", + bead: beads.Bead{CreatedAt: created}, + want: created, + wantOK: true, + }, + { + name: "CreatedAt fallback when marker unparseable", + bead: beads.Bead{ + CreatedAt: created, + Metadata: map[string]string{"pending_create_started_at": "garbage"}, + }, + want: created, + wantOK: true, + }, + { + name: "no anchor available", + bead: beads.Bead{}, + wantOK: false, + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + got, ok := stuckCreatingStartedAt(tt.bead) + if ok != tt.wantOK { + t.Fatalf("stuckCreatingStartedAt() ok = %v, want %v", ok, tt.wantOK) + } + if ok && !got.Equal(tt.want) { + t.Errorf("stuckCreatingStartedAt() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/gc/doctor_tmp_scratch.go b/cmd/gc/doctor_tmp_scratch.go new file mode 100644 index 0000000000..71d6e3ec9e --- /dev/null +++ b/cmd/gc/doctor_tmp_scratch.go @@ -0,0 +1,121 @@ +package main + +import ( + "fmt" + "os" + "strconv" + "strings" + + "github.com/gastownhall/gascity/internal/doctor" +) + +// tmpScratchSpaceCheck reports free space on the filesystem backing the +// build/test scratch temp dir ($TMPDIR, default /tmp). +// +// On this class of Gas City host /tmp is a size-capped tmpfs, and +// `make check` (`go test -p=4 ./...`) links several CGO/ICU test binaries in +// parallel; the linker scratch for each lands under $TMPDIR. The peak can +// exhaust a 16 GiB tmpfs in a burst and surface as spurious "No space left on +// device" test failures that are green in isolation — costing an operator real +// time to distinguish from genuine failures (gc-yiqil). +// +// The existing free-space guards (managed-Dolt startup preflight, +// store-maintenance DOLT_GC guard) watch only the Dolt data dir, so this +// exhaustion class is otherwise invisible to routine health checks. This check +// closes that monitoring gap. +// +// Pure observability (SeverityAdvisory): it stats the temp filesystem only, +// never mutates anything, and never gates dispatch. On platforms without +// statfs (the Windows stub) or on any probe error it reports OK and skips — +// a missing free-space reading must never turn into a health failure. +type tmpScratchSpaceCheck struct { + // tempDir returns the scratch dir to probe. Injectable for tests; + // production uses os.TempDir (honors $TMPDIR, defaults to /tmp). + tempDir func() string + // freeBytes returns the bytes available to an unprivileged process in the + // filesystem containing path. Injectable for tests; production reuses the + // managed-Dolt preflight's build-tagged statfs reader (containerFreeBytes). + freeBytes func(path string) (int64, error) + // warnFreeBytes is the free-byte floor at or below which the check warns. + // Zero disables the check. + warnFreeBytes int64 +} + +const ( + // defaultTmpScratchWarnFreeBytes is the soft floor (2 GiB). The parallel + // CGO/ICU link phase of `make check` can consume multiple GiB of scratch in + // a burst, so a temp filesystem below this is at real risk of a link-phase + // ENOSPC. Matches the managed-Dolt preflight warn floor for consistency. + defaultTmpScratchWarnFreeBytes = int64(2) << 30 // 2 GiB + + tmpScratchGiB = float64(1 << 30) +) + +func newTmpScratchSpaceCheck() *tmpScratchSpaceCheck { + return &tmpScratchSpaceCheck{ + tempDir: os.TempDir, + freeBytes: containerFreeBytes, + warnFreeBytes: tmpScratchWarnFreeBytes(), + } +} + +func (c *tmpScratchSpaceCheck) Name() string { return "tmp-scratch-space" } +func (c *tmpScratchSpaceCheck) CanFix() bool { return false } +func (c *tmpScratchSpaceCheck) Fix(_ *doctor.CheckContext) error { return nil } +func (c *tmpScratchSpaceCheck) WarmupEligible() bool { return false } + +// tmpScratchWarnFreeBytes reads the warn floor from GC_TMP_WARN_FREE_BYTES, +// defaulting to 2 GiB. A value of 0 disables the check; a negative or +// unparseable value falls back to the default. +func tmpScratchWarnFreeBytes() int64 { + if v := strings.TrimSpace(os.Getenv("GC_TMP_WARN_FREE_BYTES")); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n >= 0 { + return n + } + } + return defaultTmpScratchWarnFreeBytes +} + +func (c *tmpScratchSpaceCheck) Run(_ *doctor.CheckContext) *doctor.CheckResult { + res := &doctor.CheckResult{Name: c.Name(), Severity: doctor.SeverityAdvisory} + + if c.warnFreeBytes == 0 { + res.Status = doctor.StatusOK + res.Message = "tmp-scratch-space: check disabled (GC_TMP_WARN_FREE_BYTES=0)" + return res + } + + dir := c.tempDir() + free, err := c.freeBytes(dir) + if err != nil { + // Fail open: an unsupported platform (Windows stub returns + // errDiskPreflightUnsupported) or a transient probe error must never + // become a health failure. + res.Status = doctor.StatusOK + res.Message = fmt.Sprintf("tmp-scratch-space: free-space probe unavailable for %s — skipped", dir) + return res + } + + if free < c.warnFreeBytes { + res.Status = doctor.StatusWarning + res.Message = fmt.Sprintf( + "low scratch space: %.1f GiB free on %s (warn < %.1f GiB) — parallel `make check` CGO/ICU linking can ENOSPC here", + float64(free)/tmpScratchGiB, dir, float64(c.warnFreeBytes)/tmpScratchGiB) + res.Details = []string{ + "On a size-capped /tmp tmpfs the `go test -p=4 ./...` link phase can exhaust the", + "filesystem in a burst and surface as spurious 'No space left on device' test failures", + "that pass in isolation. Routine free-space guards (managed-Dolt preflight,", + "store-maintenance) watch only the Dolt data dir, so this class is otherwise invisible.", + "Reclaim leaked build scratch (safe; only stale dirs):", + fmt.Sprintf(" find %s -maxdepth 1 \\( -name 'go-build*' -o -name 'go-link*' \\) -mmin +30 -type d -exec rm -rf {} +", dir), + "Durable fix: point the build/test TMPDIR at a non-tmpfs path (gc-v2z1p), or enlarge /tmp.", + "Tune or disable this floor via GC_TMP_WARN_FREE_BYTES (bytes; 0 disables).", + } + res.FixHint = "clear stale go-build/go-link dirs from $TMPDIR, or set TMPDIR to a non-tmpfs path" + return res + } + + res.Status = doctor.StatusOK + res.Message = fmt.Sprintf("tmp-scratch-space: %.1f GiB free on %s", float64(free)/tmpScratchGiB, dir) + return res +} diff --git a/cmd/gc/doctor_tmp_scratch_test.go b/cmd/gc/doctor_tmp_scratch_test.go new file mode 100644 index 0000000000..21dd7c2539 --- /dev/null +++ b/cmd/gc/doctor_tmp_scratch_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "errors" + "strings" + "testing" + + "github.com/gastownhall/gascity/internal/doctor" +) + +func TestTmpScratchSpaceCheckWarnsBelowFloor(t *testing.T) { + c := &tmpScratchSpaceCheck{ + tempDir: func() string { return "/scratch" }, + freeBytes: func(string) (int64, error) { return 1 << 30, nil }, // 1 GiB free + warnFreeBytes: 2 << 30, // 2 GiB floor + } + res := c.Run(nil) + if res.Status != doctor.StatusWarning { + t.Fatalf("status = %v, want Warning; msg=%q", res.Status, res.Message) + } + // Advisory so the check reports the exhaustion risk without ever gating + // city dispatch — /tmp scratch is a build-host concern, not a runtime gate. + if res.Severity != doctor.SeverityAdvisory { + t.Fatalf("severity = %v, want Advisory", res.Severity) + } + if !strings.Contains(res.Message, "/scratch") { + t.Fatalf("message should name the probed dir: %q", res.Message) + } + if len(res.Details) == 0 || res.FixHint == "" { + t.Fatalf("warning should carry remediation details + fix hint; details=%v hint=%q", res.Details, res.FixHint) + } +} + +func TestTmpScratchSpaceCheckOKAboveFloor(t *testing.T) { + c := &tmpScratchSpaceCheck{ + tempDir: func() string { return "/scratch" }, + freeBytes: func(string) (int64, error) { return 8 << 30, nil }, // 8 GiB free + warnFreeBytes: 2 << 30, + } + res := c.Run(nil) + if res.Status != doctor.StatusOK { + t.Fatalf("status = %v, want OK; msg=%q", res.Status, res.Message) + } +} + +func TestTmpScratchSpaceCheckFailsOpenOnProbeError(t *testing.T) { + c := &tmpScratchSpaceCheck{ + tempDir: func() string { return "/scratch" }, + freeBytes: func(string) (int64, error) { return -1, errors.New("statfs boom") }, + warnFreeBytes: 2 << 30, + } + res := c.Run(nil) + if res.Status != doctor.StatusOK { + t.Fatalf("probe error must fail open to OK, got %v (%q)", res.Status, res.Message) + } +} + +func TestTmpScratchSpaceCheckDisabledWhenFloorZero(t *testing.T) { + c := &tmpScratchSpaceCheck{ + tempDir: func() string { return "/scratch" }, + freeBytes: func(string) (int64, error) { + t.Fatal("freeBytes must not be probed when the check is disabled") + return 0, nil + }, + warnFreeBytes: 0, + } + res := c.Run(nil) + if res.Status != doctor.StatusOK { + t.Fatalf("disabled check must report OK, got %v", res.Status) + } +} + +func TestTmpScratchWarnFreeBytesEnvOverride(t *testing.T) { + t.Setenv("GC_TMP_WARN_FREE_BYTES", "1048576") // 1 MiB + if got := tmpScratchWarnFreeBytes(); got != 1<<20 { + t.Fatalf("override = %d, want %d", got, 1<<20) + } + t.Setenv("GC_TMP_WARN_FREE_BYTES", "not-a-number") + if got := tmpScratchWarnFreeBytes(); got != defaultTmpScratchWarnFreeBytes { + t.Fatalf("invalid override = %d, want default %d", got, defaultTmpScratchWarnFreeBytes) + } + t.Setenv("GC_TMP_WARN_FREE_BYTES", "-1") + if got := tmpScratchWarnFreeBytes(); got != defaultTmpScratchWarnFreeBytes { + t.Fatalf("negative override = %d, want default %d", got, defaultTmpScratchWarnFreeBytes) + } + t.Setenv("GC_TMP_WARN_FREE_BYTES", "0") + if got := tmpScratchWarnFreeBytes(); got != 0 { + t.Fatalf("explicit 0 = %d, want 0 (disabled)", got) + } +} + +func TestNewTmpScratchSpaceCheckDefaults(t *testing.T) { + c := newTmpScratchSpaceCheck() + if c.tempDir == nil || c.freeBytes == nil { + t.Fatal("constructor left a nil dependency") + } + if c.Name() != "tmp-scratch-space" { + t.Fatalf("name = %q, want tmp-scratch-space", c.Name()) + } + if c.CanFix() { + t.Fatal("CanFix should be false (no safe automatic remediation)") + } + if err := c.Fix(nil); err != nil { + t.Fatalf("Fix should be a no-op, got %v", err) + } + if c.WarmupEligible() { + t.Fatal("WarmupEligible should be false") + } +} diff --git a/cmd/gc/doctor_warmup_eligible.go b/cmd/gc/doctor_warmup_eligible.go index 8468b945b8..7233f586e8 100644 --- a/cmd/gc/doctor_warmup_eligible.go +++ b/cmd/gc/doctor_warmup_eligible.go @@ -36,6 +36,10 @@ func (*mcpSharedTargetDoctorCheck) WarmupEligible() bool { return false } // `gc start` warm-up scan. func (c *sessionModelDoctorCheck) WarmupEligible() bool { return false } +// WarmupEligible returns false; this check is not part of the +// `gc start` warm-up scan. +func (c *stuckCreatingDoctorCheck) WarmupEligible() bool { return false } + // WarmupEligible returns false; this check is not part of the // `gc start` warm-up scan. func (c *v2RoutedToNamespaceCheck) WarmupEligible() bool { return false } diff --git a/cmd/gc/dolt_maintenance_ops.go b/cmd/gc/dolt_maintenance_ops.go new file mode 100644 index 0000000000..e603ddf7f5 --- /dev/null +++ b/cmd/gc/dolt_maintenance_ops.go @@ -0,0 +1,160 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" +) + +// doltMaintenanceClient is the minimal SQL surface the multi-database +// store-maintenance ops need against the managed Dolt server. Production is +// sqlDoltMaintenanceClient (wraps *sql.DB); tests inject a fake. This mirrors +// CleanupDoltClient so the per-database iteration in managedDoltMaintenanceOps +// stays unit-testable without a live Dolt server. +type doltMaintenanceClient interface { + // ListUserDatabases returns the managed user databases (system databases + // such as information_schema/mysql/dolt are excluded). + ListUserDatabases(ctx context.Context) ([]string, error) + // GCDatabase runs CALL DOLT_GC() against the named database. + GCDatabase(ctx context.Context, name string) error + // CountIssues returns SELECT COUNT(*) FROM issues for the named database — + // the post-gc readability smoke test. + CountIssues(ctx context.Context, name string) (int, error) + // Close releases the underlying connection pool. + Close() error +} + +// managedDoltMaintenanceOps implements supervisor.DoltOps for a managed Dolt +// server that hosts multiple databases (the fork's gc/tk/sl/lx/su beads +// ledgers). CALL DOLT_GC() compacts only the session's selected database, so +// reclaiming disk across the whole server requires selecting and compacting +// each managed database in turn — a single call would leave every database +// but the default at its peak on-disk size. +type managedDoltMaintenanceOps struct { + client doltMaintenanceClient +} + +// newManagedDoltMaintenanceOps opens a maintenance-tuned connection to the +// managed Dolt server and wraps it for the store-maintenance loop. +func newManagedDoltMaintenanceOps(host, port, user string) (*managedDoltMaintenanceOps, error) { + db, err := openManagedDoltMaintenanceDB(host, port, user) + if err != nil { + return nil, err + } + return &managedDoltMaintenanceOps{client: &sqlDoltMaintenanceClient{db: db}}, nil +} + +// ExecGC runs CALL DOLT_GC() against every managed user database. It fails +// fast, naming the database, so the maintenance loop can classify and alert on +// the specific database that could not be compacted; databases compacted +// before the failure keep their reclaimed space. +func (o *managedDoltMaintenanceOps) ExecGC(ctx context.Context) error { + dbs, err := o.client.ListUserDatabases(ctx) + if err != nil { + return fmt.Errorf("list managed databases: %w", err) + } + if len(dbs) == 0 { + return errors.New("no managed user databases found for DOLT_GC") + } + for _, name := range dbs { + if err := o.client.GCDatabase(ctx, name); err != nil { + return fmt.Errorf("DOLT_GC on database %q: %w", name, err) + } + } + return nil +} + +// SmokeCount sums SELECT COUNT(*) FROM issues across every managed user +// database. A per-database query error (e.g. a table the gc corrupted) +// propagates so the loop records a smoke-test failure; the supervisor treats a +// zero total as unhealthy. Summing tolerates an individual database that +// legitimately holds zero issues as long as some database is populated. +func (o *managedDoltMaintenanceOps) SmokeCount(ctx context.Context) (int, error) { + dbs, err := o.client.ListUserDatabases(ctx) + if err != nil { + return 0, fmt.Errorf("list managed databases: %w", err) + } + total := 0 + for _, name := range dbs { + n, err := o.client.CountIssues(ctx, name) + if err != nil { + return 0, fmt.Errorf("smoke count on database %q: %w", name, err) + } + total += n + } + return total, nil +} + +// Close releases the underlying connection pool. +func (o *managedDoltMaintenanceOps) Close() error { + return o.client.Close() +} + +// sqlDoltMaintenanceClient is the production doltMaintenanceClient: it runs each +// operation against the managed Dolt server over a *sql.DB pool. Each +// database-scoped call takes a connection, selects the database with USE, then +// runs the statement. CALL DOLT_GC() invalidates every connection open across +// it (Error 1105 "...please reconnect."), so a poisoned connection must never be +// reused: the pool is built by openManagedDoltMaintenanceDB → +// tuneManagedDoltMaintenancePool, which retains no idle connection and thereby +// reconnects fresh for every operation. +type sqlDoltMaintenanceClient struct { + db *sql.DB +} + +func (c *sqlDoltMaintenanceClient) ListUserDatabases(ctx context.Context) ([]string, error) { + conn, err := c.db.Conn(ctx) + if err != nil { + return nil, err + } + defer conn.Close() //nolint:errcheck // best-effort; pool owns lifecycle + return managedDoltSelectUserDatabasesFromConn(ctx, conn) +} + +func (c *sqlDoltMaintenanceClient) GCDatabase(ctx context.Context, name string) error { + conn, err := c.db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() //nolint:errcheck // best-effort; pool owns lifecycle + if err := useManagedDoltDatabase(ctx, conn, name); err != nil { + return err + } + _, err = conn.ExecContext(ctx, "CALL DOLT_GC()") + return err +} + +func (c *sqlDoltMaintenanceClient) CountIssues(ctx context.Context, name string) (int, error) { + conn, err := c.db.Conn(ctx) + if err != nil { + return 0, err + } + defer conn.Close() //nolint:errcheck // best-effort; pool owns lifecycle + if err := useManagedDoltDatabase(ctx, conn, name); err != nil { + return 0, err + } + var n int + // The bd schema names the work-unit table "issues" in every managed + // ledger; internal/supervisor.maintenanceSmokeTable is the same literal. + if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM `issues`").Scan(&n); err != nil { + return 0, err + } + return n, nil +} + +func (c *sqlDoltMaintenanceClient) Close() error { + return c.db.Close() +} + +// useManagedDoltDatabase selects name on conn, backtick-escaping the +// identifier. The names come from the server's own SHOW DATABASES catalog, but +// escaping matches the managed-Dolt convention (see PurgeDroppedDatabases). +func useManagedDoltDatabase(ctx context.Context, conn *sql.Conn, name string) error { + safe := strings.ReplaceAll(name, "`", "``") + if _, err := conn.ExecContext(ctx, fmt.Sprintf("USE `%s`", safe)); err != nil { //nolint:gosec // G201: identifier-escaped + return fmt.Errorf("USE %q: %w", name, err) + } + return nil +} diff --git a/cmd/gc/dolt_maintenance_ops_test.go b/cmd/gc/dolt_maintenance_ops_test.go new file mode 100644 index 0000000000..336ad2b96d --- /dev/null +++ b/cmd/gc/dolt_maintenance_ops_test.go @@ -0,0 +1,434 @@ +package main + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "io" + "strings" + "sync" + "testing" + "time" +) + +// fakeDoltMaintenanceClient is an injectable doltMaintenanceClient that records +// calls so tests can assert on iteration order and error propagation without a +// live Dolt server. +type fakeDoltMaintenanceClient struct { + databases []string + listErr error + gcErrs map[string]error // per-database GCDatabase errors + countByDB map[string]int // per-database CountIssues results + countErrs map[string]error // per-database CountIssues errors + gcCalls []string // databases passed to GCDatabase, in order + countCalls []string // databases passed to CountIssues, in order + closed bool + closeErr error +} + +func (f *fakeDoltMaintenanceClient) ListUserDatabases(_ context.Context) ([]string, error) { + if f.listErr != nil { + return nil, f.listErr + } + return f.databases, nil +} + +func (f *fakeDoltMaintenanceClient) GCDatabase(_ context.Context, name string) error { + f.gcCalls = append(f.gcCalls, name) + if f.gcErrs != nil { + return f.gcErrs[name] + } + return nil +} + +func (f *fakeDoltMaintenanceClient) CountIssues(_ context.Context, name string) (int, error) { + f.countCalls = append(f.countCalls, name) + if f.countErrs != nil { + if err := f.countErrs[name]; err != nil { + return 0, err + } + } + if f.countByDB != nil { + return f.countByDB[name], nil + } + return 0, nil +} + +func (f *fakeDoltMaintenanceClient) Close() error { + f.closed = true + return f.closeErr +} + +func TestManagedDoltMaintenanceOpsExecGCRunsEveryDatabaseInOrder(t *testing.T) { + fake := &fakeDoltMaintenanceClient{databases: []string{"gc", "tk", "sl", "lx", "su"}} + ops := &managedDoltMaintenanceOps{client: fake} + + if err := ops.ExecGC(context.Background()); err != nil { + t.Fatalf("ExecGC: unexpected error: %v", err) + } + want := []string{"gc", "tk", "sl", "lx", "su"} + if strings.Join(fake.gcCalls, ",") != strings.Join(want, ",") { + t.Fatalf("GCDatabase calls = %v, want %v (every managed database, in order)", fake.gcCalls, want) + } +} + +func TestManagedDoltMaintenanceOpsExecGCFailsFastNamingDatabase(t *testing.T) { + boom := errors.New("dolt gc exploded") + fake := &fakeDoltMaintenanceClient{ + databases: []string{"gc", "tk", "sl"}, + gcErrs: map[string]error{"tk": boom}, + } + ops := &managedDoltMaintenanceOps{client: fake} + + err := ops.ExecGC(context.Background()) + if err == nil { + t.Fatal("ExecGC: expected error when a database fails to gc") + } + if !errors.Is(err, boom) { + t.Fatalf("ExecGC error = %v, want wrapped %v", err, boom) + } + if !strings.Contains(err.Error(), "tk") { + t.Fatalf("ExecGC error = %q, want it to name the failing database %q", err.Error(), "tk") + } + // Fail-fast: gc on the database after the failure must not run; gc on the + // database before it must (its reclaimed space is preserved). + if got := strings.Join(fake.gcCalls, ","); got != "gc,tk" { + t.Fatalf("GCDatabase calls = %q, want %q (stop at the failure)", got, "gc,tk") + } +} + +func TestManagedDoltMaintenanceOpsExecGCErrorsWhenNoUserDatabases(t *testing.T) { + fake := &fakeDoltMaintenanceClient{databases: nil} + ops := &managedDoltMaintenanceOps{client: fake} + + if err := ops.ExecGC(context.Background()); err == nil { + t.Fatal("ExecGC: expected error when the server has no managed user databases") + } + if len(fake.gcCalls) != 0 { + t.Fatalf("GCDatabase called %v with no databases; expected none", fake.gcCalls) + } +} + +func TestManagedDoltMaintenanceOpsExecGCPropagatesListError(t *testing.T) { + boom := errors.New("show databases failed") + fake := &fakeDoltMaintenanceClient{listErr: boom} + ops := &managedDoltMaintenanceOps{client: fake} + + err := ops.ExecGC(context.Background()) + if !errors.Is(err, boom) { + t.Fatalf("ExecGC error = %v, want wrapped %v", err, boom) + } +} + +func TestManagedDoltMaintenanceOpsSmokeCountSumsAcrossDatabases(t *testing.T) { + fake := &fakeDoltMaintenanceClient{ + databases: []string{"gc", "tk", "sl"}, + countByDB: map[string]int{"gc": 12, "tk": 0, "sl": 7}, + } + ops := &managedDoltMaintenanceOps{client: fake} + + got, err := ops.SmokeCount(context.Background()) + if err != nil { + t.Fatalf("SmokeCount: unexpected error: %v", err) + } + if got != 19 { + t.Fatalf("SmokeCount = %d, want 19 (sum across databases; an empty database is tolerated)", got) + } + want := []string{"gc", "tk", "sl"} + if strings.Join(fake.countCalls, ",") != strings.Join(want, ",") { + t.Fatalf("CountIssues calls = %v, want %v", fake.countCalls, want) + } +} + +func TestManagedDoltMaintenanceOpsSmokeCountPropagatesErrorNamingDatabase(t *testing.T) { + boom := errors.New("issues table missing") + fake := &fakeDoltMaintenanceClient{ + databases: []string{"gc", "tk"}, + countErrs: map[string]error{"tk": boom}, + } + ops := &managedDoltMaintenanceOps{client: fake} + + _, err := ops.SmokeCount(context.Background()) + if !errors.Is(err, boom) { + t.Fatalf("SmokeCount error = %v, want wrapped %v", err, boom) + } + if !strings.Contains(err.Error(), "tk") { + t.Fatalf("SmokeCount error = %q, want it to name the failing database %q", err.Error(), "tk") + } +} + +func TestManagedDoltMaintenanceOpsSmokeCountPropagatesListError(t *testing.T) { + boom := errors.New("show databases failed") + fake := &fakeDoltMaintenanceClient{listErr: boom} + ops := &managedDoltMaintenanceOps{client: fake} + + if _, err := ops.SmokeCount(context.Background()); !errors.Is(err, boom) { + t.Fatalf("SmokeCount error = %v, want wrapped %v", err, boom) + } +} + +func TestManagedDoltMaintenanceOpsCloseClosesClient(t *testing.T) { + fake := &fakeDoltMaintenanceClient{} + ops := &managedDoltMaintenanceOps{client: fake} + + if err := ops.Close(); err != nil { + t.Fatalf("Close: unexpected error: %v", err) + } + if !fake.closed { + t.Fatal("Close did not close the underlying client") + } +} + +func TestManagedDoltBaseConfigOmitsReadDeadlineForLongOperations(t *testing.T) { + cfg, err := managedDoltBaseConfig("", "3307", "") + if err != nil { + t.Fatalf("managedDoltBaseConfig: unexpected error: %v", err) + } + // The shared base config must NOT carry a per-read/write socket deadline: + // CALL DOLT_GC() can run for minutes and is bounded by the caller's + // context (gc_timeout) instead. managedDoltOpenDB layers the 5 s probe + // deadlines on top for quick health checks; the maintenance opener relies + // on the base as-is, so a short read deadline here would silently kill a + // long gc after 5 s. + if cfg.ReadTimeout != 0 { + t.Errorf("ReadTimeout = %v, want 0 (no short read deadline on the shared base)", cfg.ReadTimeout) + } + if cfg.WriteTimeout != 0 { + t.Errorf("WriteTimeout = %v, want 0", cfg.WriteTimeout) + } + if cfg.Timeout != 5*time.Second { + t.Errorf("dial Timeout = %v, want 5s (fast-fail on an unreachable server)", cfg.Timeout) + } + if cfg.User != "root" { + t.Errorf("User = %q, want defaulted to root", cfg.User) + } +} + +func TestManagedDoltBaseConfigRequiresPort(t *testing.T) { + if _, err := managedDoltBaseConfig("host", "", "root"); err == nil { + t.Fatal("managedDoltBaseConfig: expected error for empty port") + } +} + +func TestOpenManagedDoltMaintenanceDBRequiresPort(t *testing.T) { + if _, err := openManagedDoltMaintenanceDB("host", "", "root"); err == nil { + t.Fatal("openManagedDoltMaintenanceDB: expected error for empty port") + } +} + +// TestManagedDoltMaintenanceOpsReconnectsAcrossOnlineGC is the regression for +// the online-GC connection-invalidation path (Error 1105). The managed Dolt +// server invalidates every connection that was open across an online +// CALL DOLT_GC(): the next statement on such a connection fails with +// "...this connection can no longer be used. please reconnect." database/sql's +// Conn.Close() returns a connection to the pool for reuse, so without pool +// tuning a GC-poisoned connection is handed straight back out — the next +// database's USE aborts the whole multi-database run (observed in production as +// lx failing at 0.3s). tuneManagedDoltMaintenancePool must make the pool retain +// no idle connection so every operation gets a fresh physical connection (a +// reconnect) after each GC. +// +// The doltGCPoisonServer fake models that invalidation at the driver level: a +// connection opened in an earlier GC generation fails its next statement, while +// a freshly-opened connection adopts the current generation and succeeds. +func TestManagedDoltMaintenanceOpsReconnectsAcrossOnlineGC(t *testing.T) { + cases := []struct { + name string + tune bool + wantErr bool + }{ + // Documents the bug: the default pool reuses the poisoned connection. + {name: "untuned_pool_reuses_poisoned_connection", tune: false, wantErr: true}, + // Verifies the fix: the tuned pool reconnects after every GC. + {name: "tuned_pool_reconnects_after_each_gc", tune: true, wantErr: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + server := &doltGCPoisonServer{databases: []string{"gc", "lx"}} + dsn := registerDoltGCPoisonServer(t, server) + + db, err := sql.Open(doltGCPoisonDriverName, dsn) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + defer db.Close() //nolint:errcheck + if tc.tune { + tuneManagedDoltMaintenancePool(db) + } + + ops := &managedDoltMaintenanceOps{client: &sqlDoltMaintenanceClient{db: db}} + err = ops.ExecGC(context.Background()) + + if tc.wantErr { + if err == nil { + t.Fatal("ExecGC: want a reconnect error from the reused poisoned connection, got nil") + } + if !strings.Contains(err.Error(), "lx") || !strings.Contains(strings.ToLower(err.Error()), "reconnect") { + t.Fatalf("ExecGC error = %v, want it to name the second database (lx) and mention reconnect", err) + } + return + } + + if err != nil { + t.Fatalf("ExecGC: unexpected error after pool tuning: %v", err) + } + if got := server.gcRuns(); got != 2 { + t.Fatalf("CALL DOLT_GC ran %d times, want 2 (once per managed database — proves the second db was reached after a reconnect)", got) + } + // The post-gc smoke count runs on connections opened after the final + // GC; a fresh-connection pool keeps it readable too. + total, err := ops.SmokeCount(context.Background()) + if err != nil { + t.Fatalf("SmokeCount after GC: unexpected error: %v", err) + } + if total == 0 { + t.Fatal("SmokeCount = 0 after GC, want the summed issue counts") + } + }) + } +} + +// --- doltGCPoisonServer: a fake database/sql driver that models Dolt's +// online-GC connection invalidation, for the regression test above. --- + +const doltGCPoisonDriverName = "gc_dolt_gc_poison" + +// errDoltGCReconnect is the driver-level analog of Dolt's Error 1105 ("this +// connection was established when this server performed an online garbage +// collection ... please reconnect."). The code under test does not inspect the +// error code, only propagates it, so a plain error faithfully reproduces the +// failure that reusing a poisoned connection triggers. Kept lowercase and +// unpunctuated per Go error-string convention; the test matches on "reconnect". +var errDoltGCReconnect = errors.New("dolt error 1105: connection established across an online garbage collection can no longer be used; please reconnect") + +// doltGCPoisonServers maps a per-test DSN to its server so subtests stay +// isolated while sharing one registered driver (sql.Register panics on a +// duplicate name). +var doltGCPoisonServers = struct { + sync.Mutex + byDSN map[string]*doltGCPoisonServer +}{byDSN: map[string]*doltGCPoisonServer{}} + +func init() { + sql.Register(doltGCPoisonDriverName, doltGCPoisonDriver{}) +} + +// registerDoltGCPoisonServer associates server with a unique DSN derived from +// the test name and returns that DSN for sql.Open. +func registerDoltGCPoisonServer(t *testing.T, server *doltGCPoisonServer) string { + t.Helper() + dsn := t.Name() + doltGCPoisonServers.Lock() + doltGCPoisonServers.byDSN[dsn] = server + doltGCPoisonServers.Unlock() + t.Cleanup(func() { + doltGCPoisonServers.Lock() + delete(doltGCPoisonServers.byDSN, dsn) + doltGCPoisonServers.Unlock() + }) + return dsn +} + +// doltGCPoisonServer is the shared state behind every connection a test opens. +// generation advances each time a connection runs CALL DOLT_GC(); a connection +// opened in an earlier generation is "poisoned" and fails its next statement. +type doltGCPoisonServer struct { + mu sync.Mutex + databases []string + generation int + gcCount int +} + +func (s *doltGCPoisonServer) open() *doltGCPoisonConn { + s.mu.Lock() + defer s.mu.Unlock() + return &doltGCPoisonConn{server: s, generation: s.generation} +} + +func (s *doltGCPoisonServer) gcRuns() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.gcCount +} + +type doltGCPoisonDriver struct{} + +func (doltGCPoisonDriver) Open(dsn string) (driver.Conn, error) { + doltGCPoisonServers.Lock() + server := doltGCPoisonServers.byDSN[dsn] + doltGCPoisonServers.Unlock() + if server == nil { + return nil, fmt.Errorf("no doltGCPoisonServer registered for dsn %q", dsn) + } + return server.open(), nil +} + +type doltGCPoisonConn struct { + server *doltGCPoisonServer + generation int +} + +func (c *doltGCPoisonConn) Prepare(string) (driver.Stmt, error) { + return nil, errors.New("prepare unsupported") +} +func (c *doltGCPoisonConn) Close() error { return nil } +func (c *doltGCPoisonConn) Begin() (driver.Tx, error) { return nil, errors.New("tx unsupported") } + +// staleLocked reports whether this connection predates the latest online GC. +// Callers hold server.mu. +func (c *doltGCPoisonConn) staleLocked() bool { + return c.generation < c.server.generation +} + +func (c *doltGCPoisonConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) { + c.server.mu.Lock() + defer c.server.mu.Unlock() + if c.staleLocked() { + return nil, errDoltGCReconnect + } + if strings.HasPrefix(query, "CALL DOLT_GC()") { + c.server.gcCount++ + // Online GC invalidates every currently-open connection, including + // this one — its next statement would now be stale. + c.server.generation++ + } + return driver.RowsAffected(0), nil +} + +func (c *doltGCPoisonConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) { + c.server.mu.Lock() + defer c.server.mu.Unlock() + if c.staleLocked() { + return nil, errDoltGCReconnect + } + switch { + case strings.HasPrefix(query, "SHOW DATABASES"): + rows := make([][]driver.Value, 0, len(c.server.databases)) + for _, name := range c.server.databases { + rows = append(rows, []driver.Value{name}) + } + return &doltGCPoisonRows{cols: []string{"Database"}, data: rows}, nil + case strings.Contains(query, "COUNT(*)"): + return &doltGCPoisonRows{cols: []string{"count"}, data: [][]driver.Value{{int64(7)}}}, nil + } + return nil, fmt.Errorf("unexpected query %q", query) +} + +type doltGCPoisonRows struct { + cols []string + data [][]driver.Value + pos int +} + +func (r *doltGCPoisonRows) Columns() []string { return r.cols } +func (r *doltGCPoisonRows) Close() error { return nil } +func (r *doltGCPoisonRows) Next(dest []driver.Value) error { + if r.pos >= len(r.data) { + return io.EOF + } + copy(dest, r.data[r.pos]) + r.pos++ + return nil +} diff --git a/cmd/gc/dolt_sql_health.go b/cmd/gc/dolt_sql_health.go index c73e9e26f2..acbc467157 100644 --- a/cmd/gc/dolt_sql_health.go +++ b/cmd/gc/dolt_sql_health.go @@ -263,7 +263,13 @@ func managedDoltPassword() string { return strings.TrimSpace(os.Getenv("GC_DOLT_PASSWORD")) } -func managedDoltOpenDB(host, port, user string) (*sql.DB, error) { +// managedDoltBaseConfig builds the shared go-sql-driver config for the +// managed Dolt server (host/port/user normalization, password, native +// auth, 5 s dial timeout). Callers layer per-operation socket deadlines +// on top: managedDoltOpenDB adds 5 s read/write deadlines for quick +// probes, while openManagedDoltMaintenanceDB leaves them unset for +// long-running maintenance. +func managedDoltBaseConfig(host, port, user string) (*mysql.Config, error) { host = managedDoltConnectHost(host) port = strings.TrimSpace(port) if port == "" { @@ -279,12 +285,59 @@ func managedDoltOpenDB(host, port, user string) (*sql.DB, error) { cfg.Net = "tcp" cfg.Addr = host + ":" + port cfg.Timeout = 5 * time.Second + cfg.AllowNativePasswords = true + return cfg, nil +} + +func managedDoltOpenDB(host, port, user string) (*sql.DB, error) { + cfg, err := managedDoltBaseConfig(host, port, user) + if err != nil { + return nil, err + } cfg.ReadTimeout = 5 * time.Second cfg.WriteTimeout = 5 * time.Second - cfg.AllowNativePasswords = true return sql.Open("mysql", cfg.FormatDSN()) } +// openManagedDoltMaintenanceDB opens a pooled connection to the managed +// Dolt server tuned for long-running maintenance. Unlike managedDoltOpenDB +// (5 s read/write deadlines suited to quick health probes), it leaves the +// per-read/write socket deadlines unset so a multi-minute CALL DOLT_GC() +// is bounded by the caller's context (the maintenance loop's gc_timeout) +// rather than killed after 5 s. The dial timeout stays short so an +// unreachable server fails fast. +func openManagedDoltMaintenanceDB(host, port, user string) (*sql.DB, error) { + cfg, err := managedDoltBaseConfig(host, port, user) + if err != nil { + return nil, err + } + db, err := sql.Open("mysql", cfg.FormatDSN()) + if err != nil { + return nil, err + } + tuneManagedDoltMaintenancePool(db) + return db, nil +} + +// tuneManagedDoltMaintenancePool configures db so a connection is never reused +// after CALL DOLT_GC(). Dolt's online GC invalidates every connection that was +// open across it: the next statement on such a connection fails with Error 1105 +// ("...this connection can no longer be used. please reconnect."). Because +// database/sql's Conn.Close() returns a connection to the idle pool for reuse, +// an untuned pool hands the GC-poisoned connection straight back out, and the +// next database's USE — or the post-gc smoke count — aborts the whole cycle. +// +// Retaining zero idle connections forces a fresh physical connection (i.e. a +// reconnect) for every operation, which is exactly the remedy the 1105 error +// asks for. Capping open connections at one matches the maintenance loop's +// strictly sequential access (it GCs one database at a time and never holds two +// connections at once) and guarantees the server never sees a second, +// mid-GC-poisoned maintenance connection. +func tuneManagedDoltMaintenancePool(db *sql.DB) { + db.SetMaxIdleConns(0) + db.SetMaxOpenConns(1) +} + func managedDoltQueryProbeDirect(host, port, user string) error { db, err := managedDoltOpenDB(host, port, user) if err != nil { diff --git a/cmd/gc/graph_dispatch_mem_test.go b/cmd/gc/graph_dispatch_mem_test.go index 35539466f4..fa76cd2fcb 100644 --- a/cmd/gc/graph_dispatch_mem_test.go +++ b/cmd/gc/graph_dispatch_mem_test.go @@ -87,7 +87,7 @@ func testControlDispatcherAgentTOML(dir string) string { b.WriteString("\n[[agent]]\n") b.WriteString("name = \"control-dispatcher\"\n") if dir != "" { - b.WriteString(fmt.Sprintf("dir = %q\n", dir)) + fmt.Fprintf(&b, "dir = %q\n", dir) } b.WriteString("start_command = \"gc convoy control --serve --follow {{.Agent}}\"\n") b.WriteString("prompt_mode = \"none\"\n") diff --git a/cmd/gc/main_test.go b/cmd/gc/main_test.go index acc6f302a4..5574c05044 100644 --- a/cmd/gc/main_test.go +++ b/cmd/gc/main_test.go @@ -24,6 +24,7 @@ import ( "github.com/gastownhall/gascity/internal/runtime" sessionpkg "github.com/gastownhall/gascity/internal/session" "github.com/gastownhall/gascity/internal/supervisor" + "github.com/gastownhall/gascity/internal/testutil" "github.com/gastownhall/gascity/test/tmuxtest" "github.com/rogpeppe/go-internal/testscript" ) @@ -235,6 +236,18 @@ func TestMain(m *testing.M) { panic(err) } testTempRootAliveSentinel = sentinel + // Written under testTempRoot (swept with the binary's temp state), so this + // must come after testTempRoot is created. See gc-j1gi1. + gitConfigPath, err := testutil.WriteIsolatedGitConfig(testTempRoot) + if err != nil { + panic(err) + } + if err := os.Setenv("GIT_CONFIG_GLOBAL", gitConfigPath); err != nil { + panic(err) + } + if err := os.Setenv("GIT_CONFIG_SYSTEM", os.DevNull); err != nil { + panic(err) + } if err := os.Setenv("TMPDIR", testTempRoot); err != nil { panic(err) } diff --git a/cmd/gc/order_dispatch.go b/cmd/gc/order_dispatch.go index 568c54ef95..4414c6993d 100644 --- a/cmd/gc/order_dispatch.go +++ b/cmd/gc/order_dispatch.go @@ -17,6 +17,7 @@ import ( "sync" "syscall" "time" + "unicode/utf8" "github.com/gastownhall/gascity/internal/beadmeta" "github.com/gastownhall/gascity/internal/beads" @@ -1356,7 +1357,16 @@ func (m *memoryOrderDispatcher) dispatchExec(ctx context.Context, front *orders. outcome = orders.RunOutcomeExecFailed logDispatchError(m.stderr, "gc: order exec %s failed: %s", scoped, execErrMsg) if len(output) > 0 { - logDispatchError(m.stderr, "gc: order exec %s output: %s", scoped, execenv.RedactText(string(output), redactionEnv)) + redactedOutput := execenv.RedactText(string(output), redactionEnv) + logDispatchError(m.stderr, "gc: order exec %s output: %s", scoped, redactedOutput) + // Fold a bounded tail of the command's own output into the + // failure message so the order.failed event records the + // actionable cause, not just the bare "exit status N". Exec + // orders (e.g. `gc dolt compact`) print the real reason and + // then exit non-zero; without this the event is undiagnosable. + if excerpt := orderExecOutputExcerpt(redactedOutput); excerpt != "" { + execErrMsg = execErrMsg + "; output: " + excerpt + } } } } @@ -1396,6 +1406,50 @@ func (m *memoryOrderDispatcher) dispatchExec(ctx context.Context, front *orders. }) } +// orderExecOutputExcerptMaxLines and orderExecOutputExcerptMaxBytes bound the +// trailing excerpt of an exec order's combined output that is folded into the +// order.failed event message. The tail is what matters for a failed exec — the +// failing command's own error is the last thing it prints before exiting — so +// the excerpt keeps the final lines and caps total size to keep event payloads +// and tracking-bead history small. +const ( + orderExecOutputExcerptMaxLines = 20 + orderExecOutputExcerptMaxBytes = 2000 +) + +// orderExecOutputExcerpt returns a bounded, trailing excerpt of an exec order's +// (already redacted) combined output, suitable for embedding in failure +// diagnostics. It keeps at most the last orderExecOutputExcerptMaxLines lines +// and caps the result at orderExecOutputExcerptMaxBytes bytes, always keeping +// the tail. A leading "…" marks content dropped by either bound. The byte cap +// never splits a multibyte rune, so the result stays valid UTF-8 for JSON event +// encoding. Returns "" when the trimmed output is empty. +func orderExecOutputExcerpt(output string) string { + trimmed := strings.TrimRight(output, "\r\n \t") + if trimmed == "" { + return "" + } + truncated := false + lines := strings.Split(trimmed, "\n") + if len(lines) > orderExecOutputExcerptMaxLines { + lines = lines[len(lines)-orderExecOutputExcerptMaxLines:] + truncated = true + } + excerpt := strings.Join(lines, "\n") + if len(excerpt) > orderExecOutputExcerptMaxBytes { + excerpt = excerpt[len(excerpt)-orderExecOutputExcerptMaxBytes:] + // Advance past any partial leading rune so the excerpt stays valid UTF-8. + for len(excerpt) > 0 && !utf8.RuneStart(excerpt[0]) { + excerpt = excerpt[1:] + } + truncated = true + } + if truncated { + excerpt = "…" + excerpt + } + return excerpt +} + func prepareOrderWispRecipe(ctx context.Context, store beads.Store, a orders.Order, searchPaths []string, vars map[string]string) (*formula.Recipe, error) { inv, err := graphv2.PrepareInvocation(ctx, store, a.Formula, searchPaths, "", vars) if err != nil { diff --git a/cmd/gc/order_dispatch_test.go b/cmd/gc/order_dispatch_test.go index 96041b2c15..e162ed669f 100644 --- a/cmd/gc/order_dispatch_test.go +++ b/cmd/gc/order_dispatch_test.go @@ -14,6 +14,7 @@ import ( "sync" "testing" "time" + "unicode/utf8" "github.com/gastownhall/gascity/internal/beadmeta" "github.com/gastownhall/gascity/internal/beads" @@ -743,6 +744,104 @@ func TestOrderDispatchEventExecFailureAdvancesCursor(t *testing.T) { } } +func TestOrderDispatchExecFailureIncludesOutputInFailedEvent(t *testing.T) { + store := beads.NewMemStore() + eventLog := events.NewFake() + eventLog.Record(events.Event{Type: events.BeadClosed, Actor: "test"}) + var rec memRecorder + + // Real exec orders (e.g. `gc dolt compact`) print the actionable cause to + // stdout/stderr and then exit non-zero. The bare exec error ("exit status + // 1") is useless on its own; the order.failed event must carry the captured + // output so operators can diagnose the failure from `gc` tooling alone. + output := "compact: db=sl flatten ok\ncompact: db=sl table=mail value hash changed after flatten — quarantine and investigate before GC" + execRun := func(context.Context, string, string, []string) ([]byte, error) { + return []byte(output + "\n"), fmt.Errorf("exit status 1") + } + + ad := buildOrderDispatcherFromListExec([]orders.Order{{ + Name: "dolt-compact", + Trigger: "event", + On: events.BeadClosed, + Exec: "scripts/compact.sh", + }}, store, eventLog, execRun, &rec) + if ad == nil { + t.Fatal("expected non-nil dispatcher") + } + + ad.dispatch(context.Background(), t.TempDir(), time.Now()) + ad.drain(context.Background()) + + msg := recordedOrderFailedMessage(t, &rec, "dolt-compact") + if !strings.Contains(msg, "exit status 1") { + t.Fatalf("order.failed message = %q, want it to retain the exec error", msg) + } + if !strings.Contains(msg, "value hash changed after flatten") { + t.Fatalf("order.failed message = %q, want it to include the captured command output", msg) + } +} + +func TestOrderExecOutputExcerpt(t *testing.T) { + t.Run("empty input yields empty excerpt", func(t *testing.T) { + if got := orderExecOutputExcerpt(""); got != "" { + t.Fatalf("orderExecOutputExcerpt(%q) = %q, want empty", "", got) + } + if got := orderExecOutputExcerpt("\n\n \n"); got != "" { + t.Fatalf("orderExecOutputExcerpt(blank) = %q, want empty", got) + } + }) + + t.Run("short output passes through trimmed", func(t *testing.T) { + got := orderExecOutputExcerpt("compact: db=gc failed\n") + if got != "compact: db=gc failed" { + t.Fatalf("orderExecOutputExcerpt = %q, want trailing newline trimmed and no marker", got) + } + }) + + t.Run("keeps trailing lines when line bound exceeded", func(t *testing.T) { + var sb strings.Builder + for i := 0; i < orderExecOutputExcerptMaxLines+10; i++ { + fmt.Fprintf(&sb, "line %d\n", i) + } + got := orderExecOutputExcerpt(sb.String()) + if !strings.HasPrefix(got, "…") { + t.Fatalf("orderExecOutputExcerpt = %q, want a leading truncation marker", got) + } + lastLine := fmt.Sprintf("line %d", orderExecOutputExcerptMaxLines+9) + if !strings.HasSuffix(got, lastLine) { + t.Fatalf("orderExecOutputExcerpt = %q, want it to end with the final line %q", got, lastLine) + } + if strings.Contains(got, "line 0\n") { + t.Fatalf("orderExecOutputExcerpt = %q, want early lines dropped", got) + } + }) + + t.Run("byte bound keeps tail and stays valid utf8", func(t *testing.T) { + // A single oversized line full of multibyte runes — the byte cap must + // not split a rune (the order.failed message is JSON-encoded). + got := orderExecOutputExcerpt(strings.Repeat("—", orderExecOutputExcerptMaxBytes)) + if len(got) > orderExecOutputExcerptMaxBytes+len("…") { + t.Fatalf("excerpt length = %d, want <= %d", len(got), orderExecOutputExcerptMaxBytes+len("…")) + } + if !utf8.ValidString(got) { + t.Fatalf("excerpt is not valid UTF-8: %q", got) + } + }) +} + +func recordedOrderFailedMessage(t *testing.T, rec *memRecorder, subject string) string { + t.Helper() + rec.mu.Lock() + defer rec.mu.Unlock() + for _, e := range rec.events { + if e.Type == events.OrderFailed && e.Subject == subject { + return e.Message + } + } + t.Fatalf("no order.failed event for subject %q among %d recorded events", subject, len(rec.events)) + return "" +} + func TestOrderDispatchEventExecLatestSeqErrorDoesNotRunExec(t *testing.T) { store := beads.NewMemStore() tracking, err := store.Create(beads.Bead{ diff --git a/cmd/gc/order_store.go b/cmd/gc/order_store.go index a0c4622634..0cedf11094 100644 --- a/cmd/gc/order_store.go +++ b/cmd/gc/order_store.go @@ -573,7 +573,15 @@ func cachedOrderStoresResolver(cityPath string, cfg *config.City) orderStoresRes } } -func orderTrackingSweepTargetsForConfig(cityPath string, cfg *config.City) []orderTrackingSweepTarget { +// orderTrackingSweepTargetsForConfig enumerates the city and configured rig +// scopes the order-tracking sweep visits. Rig paths that do not exist on +// disk (or are not directories) are skipped with a stderr note instead of +// becoming sweep targets: every bd subprocess against a dead scope root can +// burn the full 2m command timeout — worse when bd's silent on-disk +// fallback auto-imports a stale issues.jsonl — and the controller watchdog +// re-opens sweep stores every 30s, so one stale entry repeatedly starves +// the reconciler tick (gc-q40pm Problem B). stderr may be nil. +func orderTrackingSweepTargetsForConfig(cityPath string, cfg *config.City, stderr io.Writer) []orderTrackingSweepTarget { targets := []orderTrackingSweepTarget{{ target: legacyOrderCityTarget(cityPath, cfg), label: "city", @@ -584,6 +592,12 @@ func orderTrackingSweepTargetsForConfig(cityPath string, cfg *config.City) []ord if strings.TrimSpace(rig.Path) == "" { continue } + if info, err := os.Stat(rig.Path); err != nil || !info.IsDir() { + if stderr != nil { + fmt.Fprintf(stderr, "order tracking sweep: skipping rig %q: scope root %s is not a directory on disk\n", rig.Name, rig.Path) //nolint:errcheck // best-effort stderr + } + continue + } targets = append(targets, orderTrackingSweepTarget{ target: execStoreTarget{ ScopeRoot: rig.Path, @@ -598,8 +612,8 @@ func orderTrackingSweepTargetsForConfig(cityPath string, cfg *config.City) []ord return targets } -func orderTrackingSweepStoresForConfigTargets(cityPath string, cfg *config.City, requiredTargets map[string][]string) ([]beads.Store, error) { - targets := orderTrackingSweepTargetsForConfig(cityPath, cfg) +func orderTrackingSweepStoresForConfigTargets(cityPath string, cfg *config.City, requiredTargets map[string][]string, stderr io.Writer) ([]beads.Store, error) { + targets := orderTrackingSweepTargetsForConfig(cityPath, cfg, stderr) if len(requiredTargets) > 0 { filtered := targets[:0] for _, target := range targets { diff --git a/cmd/gc/order_store_test.go b/cmd/gc/order_store_test.go new file mode 100644 index 0000000000..e6538e2acc --- /dev/null +++ b/cmd/gc/order_store_test.go @@ -0,0 +1,84 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/gastownhall/gascity/internal/config" +) + +// TestOrderTrackingSweepTargetsSkipMissingRigPaths covers gc-q40pm Problem B: +// the order-tracking sweep must not open stores for configured rigs whose +// directory no longer exists on disk. Every bd subprocess against a dead +// scope root can burn the full 2m command timeout (worse when bd's silent +// on-disk fallback auto-imports a stale issues.jsonl), and the controller +// watchdog re-opens sweep stores every 30s — so a single stale rig entry +// repeatedly starves the reconciler tick. Missing directories are skipped +// loudly instead. +func TestOrderTrackingSweepTargetsSkipMissingRigPaths(t *testing.T) { + cityDir := t.TempDir() + liveRig := filepath.Join(cityDir, "live-rig") + if err := os.MkdirAll(liveRig, 0o755); err != nil { + t.Fatalf("MkdirAll(liveRig): %v", err) + } + goneRig := filepath.Join(cityDir, "gone-rig") // never created + + cfg := &config.City{ + Workspace: config.Workspace{Name: "demo"}, + Rigs: []config.Rig{ + {Name: "live", Path: liveRig}, + {Name: "gone", Path: goneRig}, + }, + } + + var stderr bytes.Buffer + targets := orderTrackingSweepTargetsForConfig(cityDir, cfg, &stderr) + + var labels []string + for _, target := range targets { + labels = append(labels, target.label) + } + if len(targets) != 2 { + t.Fatalf("targets = %v, want city + live rig only", labels) + } + if targets[0].label != "city" { + t.Errorf("targets[0].label = %q, want city", targets[0].label) + } + if targets[1].target.RigName != "live" { + t.Errorf("targets[1].RigName = %q, want live", targets[1].target.RigName) + } + if !strings.Contains(stderr.String(), `"gone"`) || !strings.Contains(stderr.String(), goneRig) { + t.Errorf("stderr = %q, want skip log naming rig \"gone\" and path %q", stderr.String(), goneRig) + } +} + +// TestOrderTrackingSweepTargetsNonDirRigPathSkipped treats a rig path that +// exists but is not a directory like a missing one: there is no store to +// sweep underneath it, only a bd subprocess timeout to pay. +func TestOrderTrackingSweepTargetsNonDirRigPathSkipped(t *testing.T) { + cityDir := t.TempDir() + filePath := filepath.Join(cityDir, "not-a-dir") + if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + cfg := &config.City{ + Workspace: config.Workspace{Name: "demo"}, + Rigs: []config.Rig{ + {Name: "flat", Path: filePath}, + }, + } + + targets := orderTrackingSweepTargetsForConfig(cityDir, cfg, nil) // nil stderr must be safe + + if len(targets) != 1 || targets[0].label != "city" { + var labels []string + for _, target := range targets { + labels = append(labels, target.label) + } + t.Fatalf("targets = %v, want city only", labels) + } +} diff --git a/cmd/gc/phase2_reporting_test.go b/cmd/gc/phase2_reporting_test.go index af0a7bddb5..283bae181f 100644 --- a/cmd/gc/phase2_reporting_test.go +++ b/cmd/gc/phase2_reporting_test.go @@ -178,7 +178,7 @@ func initialMessageFirstStartResult(tc phase2ProviderCase, prepared *preparedSta } func initialMessageResumeResult(tc phase2ProviderCase, prepared *preparedStart) workertest.Result { - got, evidence, err := phase2PromptPayload(tc, prepared) + _, evidence, err := phase2PromptPayload(tc, prepared) if err != nil { return workertest.Fail(tc.profileID, workertest.RequirementInputInitialMessageResume, fmt.Sprintf("PromptSuffix encoding invalid: %v", err)).WithEvidence(evidence) @@ -194,21 +194,24 @@ func initialMessageResumeResult(tc phase2ProviderCase, prepared *preparedStart) case strings.TrimSpace(prepared.cfg.PromptFlag) != "": return workertest.Fail(tc.profileID, workertest.RequirementInputInitialMessageResume, fmt.Sprintf("PromptFlag = %q, want no flag startup prompt replay on resume", prepared.cfg.PromptFlag)).WithEvidence(evidence) - case got != "Base worker prompt": + case strings.Contains(prepared.cfg.Nudge, "Base worker prompt"): + // gc-7go2a: a resume-mode restart for a nudge-having agent must wake on + // the nudge alone. The base prompt is rehydrated via --resume, so folding + // it into the nudge re-injects the already-restored role on every wake. return workertest.Fail(tc.profileID, workertest.RequirementInputInitialMessageResume, - fmt.Sprintf("restart prompt payload = %q, want base worker prompt on resume", got)).WithEvidence(evidence) + fmt.Sprintf("cfg.Nudge = %q, want the configured nudge alone on resume — the base prompt is rehydrated via --resume, not folded into the nudge", prepared.cfg.Nudge)).WithEvidence(evidence) case strings.Contains(prepared.cfg.Nudge, "Do the first task."): return workertest.Fail(tc.profileID, workertest.RequirementInputInitialMessageResume, fmt.Sprintf("cfg.Nudge = %q, want no replayed initial message", prepared.cfg.Nudge)).WithEvidence(evidence) case prepared.cfg.Env[startupPromptDeliveredEnv] != "1": return workertest.Fail(tc.profileID, workertest.RequirementInputInitialMessageResume, - fmt.Sprintf("%s = %q, want 1 when restart prompt is delivered on resume", startupPromptDeliveredEnv, prepared.cfg.Env[startupPromptDeliveredEnv])).WithEvidence(evidence) + fmt.Sprintf("%s = %q, want 1 so the SessionStart hook suppresses re-injecting the resumed prompt", startupPromptDeliveredEnv, prepared.cfg.Env[startupPromptDeliveredEnv])).WithEvidence(evidence) case !startupNudgeMatches(tc, prepared.cfg.Nudge): return workertest.Fail(tc.profileID, workertest.RequirementInputInitialMessageResume, - fmt.Sprintf("cfg.Nudge = %q, want configured nudge preserved after restart prompt", prepared.cfg.Nudge)).WithEvidence(evidence) + fmt.Sprintf("cfg.Nudge = %q, want configured nudge preserved on resume", prepared.cfg.Nudge)).WithEvidence(evidence) default: return workertest.Pass(tc.profileID, workertest.RequirementInputInitialMessageResume, - "resumed sessions receive the base restart prompt without replaying initial_message").WithEvidence(evidence) + "resumed sessions wake on their configured nudge without replaying initial_message or re-folding the base prompt").WithEvidence(evidence) } } diff --git a/cmd/gc/prompt.go b/cmd/gc/prompt.go index 47da219df0..402c9ad30d 100644 --- a/cmd/gc/prompt.go +++ b/cmd/gc/prompt.go @@ -24,17 +24,28 @@ const ( // PromptContext holds template data for prompt rendering. type PromptContext struct { - CityRoot string - AgentName string // qualified: "rig/polecat-1" or "mayor" - TemplateName string // config name: "polecat" (template) or "mayor" (named backing template) - BindingName string - BindingPrefix string - RigName string - RigRoot string - WorkDir string - IssuePrefix string - Branch string - DefaultBranch string // e.g. "main" — from git symbolic-ref origin/HEAD + CityRoot string + AgentName string // qualified: "rig/polecat-1" or "mayor" + TemplateName string // config name: "polecat" (template) or "mayor" (named backing template) + BindingName string + BindingPrefix string + RigName string + RigRoot string + WorkDir string + IssuePrefix string + Branch string + DefaultBranch string // e.g. "main" — from git symbolic-ref origin/HEAD + // ConfigDir is the directory where the agent's config was defined — the + // pack source dir for pack-imported agents, the city dir for inline + // agents. Templates use {{ .ConfigDir }} to reference pack-relative + // assets (scripts, fragments, docs). Mirrors the field of the same name + // on SessionSetupContext. + ConfigDir string + // DefaultMergeStrategy is the rig's effective default merge strategy + // ("direct" or "pr"), resolved via Rig.EffectiveDefaultMergeStrategy. + // Empty string when neither the rig nor the city overrides the formula + // default; pack templates fall back to their own `[vars.X.default]`. + DefaultMergeStrategy string WorkQuery string // command to find available work (from Agent.EffectiveWorkQuery) AssignedInProgressQuery string // command to find assigned in-progress work (from Agent.EffectiveAssignedInProgressQuery) AssignedReadyQuery string // command to find pre-assigned ready work (from Agent.EffectiveAssignedReadyQuery) @@ -331,6 +342,8 @@ func buildTemplateData(ctx PromptContext) map[string]string { m["IssuePrefix"] = ctx.IssuePrefix m["Branch"] = ctx.Branch m["DefaultBranch"] = ctx.DefaultBranch + m["ConfigDir"] = ctx.ConfigDir + m["DefaultMergeStrategy"] = ctx.DefaultMergeStrategy m["WorkQuery"] = ctx.WorkQuery m["AssignedInProgressQuery"] = ctx.AssignedInProgressQuery m["AssignedReadyQuery"] = ctx.AssignedReadyQuery @@ -382,6 +395,27 @@ func defaultBranchForRig(rigName string, rigs []config.Rig, dir string) string { return defaultBranchFor(dir) } +// mergeStrategyForRig resolves the effective default merge strategy for +// templates rendered in the context of `rigName`, walking rig → city → "". +// Returns the empty string when neither the rig nor the city sets a +// default; pack templates use this to opt into city-wide PR policy without +// modifying the formula's own default. Empty `rigName` falls through to +// the city default so an HQ-only city (no rig context) still honors a +// city-level `default_merge_strategy`. `cfg` may be nil. +func mergeStrategyForRig(rigName string, rigs []config.Rig, cfg *config.City) string { + if rigName != "" { + for i := range rigs { + if rigs[i].Name == rigName { + return rigs[i].EffectiveDefaultMergeStrategy(cfg) + } + } + } + if cfg != nil { + return strings.TrimSpace(cfg.DefaultMergeStrategy) + } + return "" +} + // promptFuncMap returns template functions available in prompt templates. // sessionTemplate is the custom session naming template (empty = default). // store is used by the "session" function to look up bead-derived session diff --git a/cmd/gc/prompt_test.go b/cmd/gc/prompt_test.go index 9aa3fee745..ec1a0de774 100644 --- a/cmd/gc/prompt_test.go +++ b/cmd/gc/prompt_test.go @@ -334,6 +334,32 @@ func TestRenderPromptDefaultBranch(t *testing.T) { } } +// TestRenderPromptConfigDirResolves pins the wiring documented in commit +// 0f64ea4 (the {{ .ConfigDir }} migration): templates that reference +// {{ .ConfigDir }} must resolve to the pack source directory, not the +// empty string. Without this field, missingkey=zero silently renders +// {{ .ConfigDir }}/assets/scripts/foo.sh as /assets/scripts/foo.sh. +func TestRenderPromptConfigDirResolves(t *testing.T) { + f := fsys.NewFake() + f.Files["/city/prompts/test.template.md"] = []byte("Watcher: {{ .ConfigDir }}/assets/scripts/gc-bd-watch.sh\n") + ctx := PromptContext{ConfigDir: "/home/user/packs/gc-toolkit"} + got := renderPrompt(f, "/city", "", "prompts/test.template.md", ctx, "", io.Discard, nil, nil, nil) + want := "Watcher: /home/user/packs/gc-toolkit/assets/scripts/gc-bd-watch.sh\n" + if got != want { + t.Errorf("renderPrompt(ConfigDir) = %q, want %q", got, want) + } +} + +func TestRenderPromptDefaultMergeStrategy(t *testing.T) { + f := fsys.NewFake() + f.Files["/city/prompts/test.md.tmpl"] = []byte("Strategy: {{ .DefaultMergeStrategy }}") + ctx := PromptContext{DefaultMergeStrategy: "mr"} + got := renderPrompt(f, "/city", "", "prompts/test.md.tmpl", ctx, "", io.Discard, nil, nil, nil) + if got != "Strategy: mr" { + t.Errorf("renderPrompt(DefaultMergeStrategy) = %q, want %q", got, "Strategy: mr") + } +} + func TestDefaultBranchForRig_PrefersStoredValue(t *testing.T) { rigs := []config.Rig{ {Name: "scamper", Path: "/scamper", DefaultBranch: "master"}, @@ -363,6 +389,49 @@ func TestDefaultBranchForRig_EmptyRigName(t *testing.T) { } } +func TestMergeStrategyForRig_PrefersRigValue(t *testing.T) { + rigs := []config.Rig{ + {Name: "scamper", Path: "/scamper", DefaultMergeStrategy: "direct"}, + {Name: "other", Path: "/other"}, + } + cfg := &config.City{DefaultMergeStrategy: "mr"} + got := mergeStrategyForRig("scamper", rigs, cfg) + if got != "direct" { + t.Errorf("mergeStrategyForRig(scamper) = %q, want %q (rig value)", got, "direct") + } +} + +func TestMergeStrategyForRig_FallsBackToCity(t *testing.T) { + rigs := []config.Rig{ + {Name: "scamper", Path: "/scamper"}, + } + cfg := &config.City{DefaultMergeStrategy: "mr"} + got := mergeStrategyForRig("scamper", rigs, cfg) + if got != "mr" { + t.Errorf("mergeStrategyForRig(scamper) = %q, want %q (city fallback)", got, "mr") + } +} + +func TestMergeStrategyForRig_EmptyWhenUnset(t *testing.T) { + rigs := []config.Rig{{Name: "scamper", Path: "/scamper"}} + cfg := &config.City{} + got := mergeStrategyForRig("scamper", rigs, cfg) + if got != "" { + t.Errorf("mergeStrategyForRig() = %q, want empty", got) + } + if got := mergeStrategyForRig("", nil, nil); got != "" { + t.Errorf("mergeStrategyForRig() with empty rig, nil city = %q, want empty", got) + } +} + +func TestMergeStrategyForRig_EmptyRigNameUsesCity(t *testing.T) { + cfg := &config.City{DefaultMergeStrategy: "mr"} + got := mergeStrategyForRig("", nil, cfg) + if got != "mr" { + t.Errorf("mergeStrategyForRig() with empty rig = %q, want %q (city default)", got, "mr") + } +} + func TestRenderPromptEnvOverridePriority(t *testing.T) { f := fsys.NewFake() f.Files["/city/prompts/test.md.tmpl"] = []byte("Root: {{ .CityRoot }}") diff --git a/cmd/gc/service_runtime.go b/cmd/gc/service_runtime.go index 6f38ae15e1..be4a2c31d9 100644 --- a/cmd/gc/service_runtime.go +++ b/cmd/gc/service_runtime.go @@ -76,3 +76,12 @@ func (rt *serviceRuntime) Poke() { default: } } + +// PokeDemand forces a pool-demand rebuild on the next tick, then signals it. +// See controllerState.PokeDemand. +func (rt *serviceRuntime) PokeDemand() { + if rt.cr.demandDirty != nil { + rt.cr.demandDirty.Store(true) + } + rt.Poke() +} diff --git a/cmd/gc/session_beads.go b/cmd/gc/session_beads.go index e6209132c6..bef56538b2 100644 --- a/cmd/gc/session_beads.go +++ b/cmd/gc/session_beads.go @@ -1698,15 +1698,18 @@ func syncSessionBeadsWithSnapshotAndRigStores( if managedAlias != "" { lockFn := func() error { if err := session.EnsureAliasAvailableWithConfigForOwner(store, cfg, managedAlias, "", managedAlias); err != nil { + // Block creation when the alias is already held by a live + // session bead. Previously only configured-named sessions + // blocked here; pool sessions silently proceeded without + // the alias, leaving phantom beads (gc-53fv) that the + // witness later observed sharing alias+work_dir with the + // real session. fmt.Fprintf(stderr, "session beads: alias %q for %s unavailable: %v\n", managedAlias, agentName, err) //nolint:errcheck - if isConfiguredNamed { - createErr = err - blocked = true - return nil - } - } else { - meta["alias"] = managedAlias + createErr = err + blocked = true + return nil } + meta["alias"] = managedAlias if isConfiguredNamed { if err := session.EnsureSessionNameAvailableWithConfigForOwner(store, cfg, sn, "", managedAlias); err != nil { fmt.Fprintf(stderr, "session beads: session_name %q for %s unavailable: %v\n", sn, agentName, err) //nolint:errcheck diff --git a/cmd/gc/session_beads_test.go b/cmd/gc/session_beads_test.go index 991c6fc1d8..39d02d667c 100644 --- a/cmd/gc/session_beads_test.go +++ b/cmd/gc/session_beads_test.go @@ -8362,3 +8362,76 @@ func TestSyncTailReturnsFreshStoreLoadNotLocalSlice(t *testing.T) { t.Fatalf("returned snapshot open set %v != fresh store load %v", gotIDs, freshIDs) } } + +// TestSyncSessionBeads_DoesNotCreateDuplicatePoolAlias guards against the +// gc-53fv duplicate session registration: a second pool bead must not be +// created for an alias already held by an open pool bead. Previously the +// non-configured-named branch in syncSessionBeadsWithSnapshotAndRigStores +// silently logged the alias-unavailable error and proceeded to create a +// "ghost" bead without the alias, which the witness later observed as a +// duplicate session-registry entry sharing alias+work_dir with the live +// session. +func TestSyncSessionBeads_DoesNotCreateDuplicatePoolAlias(t *testing.T) { + store := beads.NewMemStore() + clk := &clock.Fake{Time: time.Date(2026, 5, 1, 23, 56, 0, 0, time.UTC)} + sp := runtime.NewFake() + cfg := &config.City{ + Workspace: config.Workspace{Name: "test-city"}, + Agents: []config.Agent{ + {Name: "polecat", Dir: "myrig"}, + }, + } + + owner, err := store.Create(beads.Bead{ + Title: "polecat-1", + Type: sessionBeadType, + Labels: []string{sessionBeadLabel, "agent:myrig/polecat-1"}, + Metadata: map[string]string{ + "session_name": "polecat-existing", + "agent_name": "myrig/polecat-1", + "alias": "myrig/polecat-1", + "template": "myrig/polecat", + "state": "active", + "session_origin": "ephemeral", + "pool_slot": "1", + poolManagedMetadataKey: boolMetadata(true), + }, + }) + if err != nil { + t.Fatalf("creating live owner bead: %v", err) + } + + ds := map[string]TemplateParams{ + "polecat-new": { + TemplateName: "myrig/polecat", + InstanceName: "myrig/polecat-1", + Alias: "myrig/polecat-1", + Command: "claude", + PoolSlot: 1, + }, + } + + var stderr bytes.Buffer + syncSessionBeads("", store, ds, sp, allConfiguredDS(ds), cfg, clk, &stderr, false) + + all := allSessionBeads(t, store) + if len(all) != 1 { + for i, b := range all { + t.Logf("bead[%d]: id=%s status=%q session_name=%q alias=%q agent_name=%q pool_slot=%q state=%q", + i, b.ID, b.Status, + b.Metadata["session_name"], b.Metadata["alias"], + b.Metadata["agent_name"], b.Metadata["pool_slot"], + b.Metadata["state"]) + } + t.Fatalf("expected only the live owner bead, got %d beads — duplicate pool alias accepted (gc-53fv)", len(all)) + } + if all[0].ID != owner.ID { + t.Fatalf("remaining bead = %s, want owner %s", all[0].ID, owner.ID) + } + if got := all[0].Metadata["alias"]; got != "myrig/polecat-1" { + t.Fatalf("owner alias after sync = %q, want %q", got, "myrig/polecat-1") + } + if !strings.Contains(stderr.String(), `alias "myrig/polecat-1"`) { + t.Fatalf("stderr = %q, want alias unavailable warning for duplicate pool alias", stderr.String()) + } +} diff --git a/cmd/gc/session_lifecycle_parallel.go b/cmd/gc/session_lifecycle_parallel.go index f0ccf888e8..b620412a8b 100644 --- a/cmd/gc/session_lifecycle_parallel.go +++ b/cmd/gc/session_lifecycle_parallel.go @@ -192,14 +192,36 @@ func wakeFairnessTime(c startCandidate) time.Time { return time.Time{} } -// sortCandidatesByWakeFairness orders candidates least-recently-woken first so a -// budget-limited tick rotates wakes across sessions instead of always deferring -// the same back-of-order sessions. Stable on the original order for ties. Callers -// must only sort within a dependency wave (every candidate's deps are satisfied). -func sortCandidatesByWakeFairness(candidates []startCandidate) { - sort.SliceStable(candidates, func(i, j int) bool { - return wakeFairnessTime(candidates[i]).Before(wakeFairnessTime(candidates[j])) +// sortCandidatesByWakeFairness orders a tick's wake candidates least-recently-woken +// first, then interleaves them round-robin across logical templates so one +// template's backlog can't fill the whole wake budget and starve other templates +// (gc-unpyk). A lone template keeps strict least-recently-woken order. Callers must +// only sort within a dependency wave (every candidate's deps are satisfied). +func sortCandidatesByWakeFairness(candidates []startCandidate, cfg *config.City) { + type rankedCandidate struct { + candidate startCandidate + wokeAt time.Time + round int // 0-based index within its own template, oldest first + } + ranked := make([]rankedCandidate, len(candidates)) + for i, c := range candidates { + ranked[i] = rankedCandidate{candidate: c, wokeAt: wakeFairnessTime(c)} + } + sort.SliceStable(ranked, func(i, j int) bool { + return ranked[i].wokeAt.Before(ranked[j].wokeAt) }) + perTemplate := make(map[string]int, len(ranked)) + for i := range ranked { + template := ranked[i].candidate.logicalTemplate(cfg) + ranked[i].round = perTemplate[template] + perTemplate[template]++ + } + sort.SliceStable(ranked, func(i, j int) bool { + return ranked[i].round < ranked[j].round + }) + for i := range ranked { + candidates[i] = ranked[i].candidate + } } func (c startCandidate) logicalTemplate(cfg *config.City) string { @@ -222,10 +244,12 @@ type preparedStart struct { // promptDelivery decision AND-ed with the fresh-launch condition, i.e. the // exact complement of the resume override below — so a resume that swaps in // restartPromptNudge and re-sets GC_STARTUP_PROMPT_DELIVERED for hooks stamps - // no priming marker. promptHash is the sha256 of the rendered startup template - // prompt (tp.Prompt) only — it excludes the one-shot initial_message override - // appended to the delivered payload, so the stored hash still matches a later - // re-derivation from the template (S19 re-eligibility). + // no priming marker. ACP is carved out of that override (gc-xlj7s), so an ACP + // resume does deliver — through the nudge — and does stamp. promptHash is the + // sha256 of the rendered startup template prompt (tp.Prompt) only — it excludes + // the one-shot initial_message override appended to the delivered payload, so + // the stored hash still matches a later re-derivation from the template (S19 + // re-eligibility). promptDelivered bool promptHash string } @@ -1070,7 +1094,10 @@ func buildPreparedStartWithWorkDirResolver( // launch — the exact complement of the resume override below, which swaps in // restartPromptNudge and delivers nothing. Reading the env marker instead // would mis-stamp every resume (it is re-set to "1" for hook consumption). - promptDelivered := delivery.Delivered && (firstStart || forceFresh || !hasResumeKey) + // tp.IsACP is part of that complement: ACP is carved out of the override + // (gc-xlj7s), so an ACP resume never suppresses delivery — it re-delivers the + // rendered prompt through the nudge and must still stamp the marker. + promptDelivered := delivery.Delivered && (firstStart || forceFresh || !hasResumeKey || tp.IsACP) // prompt_hash is the sha256 of the rendered startup TEMPLATE prompt (tp.Prompt) // only, computed here BEFORE the one-shot initial_message is appended to the // delivered payload below. The hash exists so a template/config change re-primes @@ -1078,10 +1105,28 @@ func buildPreparedStartWithWorkDirResolver( // replays the transient initial_message, so hashing the delivered bytes would // make the stored hash never match the re-derivation and re-prime forever. promptHash := sessionpkg.PromptHash(tp.Prompt) - if !firstStart && !forceFresh && hasResumeKey { + // !tp.IsACP mirrors the CLI-resume guard above: only providers that actually + // take a --resume/--session-id flag rehydrate the prior conversation. ACP is + // excluded from resolveSessionCommand and acp.Provider.Start always opens a + // fresh session/new, delivering the rendered role prompt via the nudge — so + // suppressing prompt replay on an ACP "resume" would drop the role prompt + // entirely (gc-xlj7s). A Claude ACP session can still mint a session_key, so + // hasResumeKey alone is not enough to tell that --resume rehydration applies. + if !firstStart && !forceFresh && hasResumeKey && !tp.IsACP { agentCfg.PromptSuffix = "" agentCfg.PromptFlag = "" - agentCfg.Nudge = restartPromptNudge(tp.Prompt, tp.Hints.Nudge) + // On resume the provider rehydrates the prior conversation (the rendered + // role prompt included) via --resume, so an agent that already carries a + // nudge only needs the nudge to wake without landing idle. Prepending the + // prompt in that case duplicates the already-restored role on every wake + // (gc-7go2a; the ~20k-token re-injection in gc-cbtfq). #2477's prompt + // replay stays as the fallback for nudge-less agents, whose only + // first-turn payload is the prompt itself. + if strings.TrimSpace(tp.Hints.Nudge) != "" { + agentCfg.Nudge = tp.Hints.Nudge + } else { + agentCfg.Nudge = restartPromptNudge(tp.Prompt, tp.Hints.Nudge) + } if agentCfg.Env != nil { delete(agentCfg.Env, startupPromptDeliveredEnv) } @@ -2597,12 +2642,9 @@ func executePlannedStartsTraced( } ready = append(ready, candidate) } - // Fairness: spend a budget-limited tick on the least-recently-woken - // candidates first. The wave order is a stable dependency topo-sort, so - // without this the same back-of-order sessions are deferred_by_wake_budget - // every tick. Sorting within the dependency wave is safe: every - // candidate here already has its dependencies satisfied. - sortCandidatesByWakeFairness(ready) + // Least-recently-woken first, then round-robin across templates so one + // template's backlog can't spend the whole tick's wake budget (gc-unpyk). + sortCandidatesByWakeFairness(ready, cfg) for offset := 0; offset < len(ready); { if wakeCount >= maxWakes { for _, candidate := range ready[offset:] { diff --git a/cmd/gc/session_lifecycle_parallel_phase2_test.go b/cmd/gc/session_lifecycle_parallel_phase2_test.go index 3b9dcb705d..66dfbe3672 100644 --- a/cmd/gc/session_lifecycle_parallel_phase2_test.go +++ b/cmd/gc/session_lifecycle_parallel_phase2_test.go @@ -252,3 +252,84 @@ func preparePhase2ResumeRestartStart(t *testing.T, tc phase2ProviderCase, overri } return prepared } + +// TestPhase2ResumeRestartNudgePreservedWithoutPromptReplay is the regression +// guard for gc-7go2a, which narrows #2477. On a resume-mode restart an agent +// that already carries a non-empty startup nudge must receive the nudge ALONE: +// the provider's --resume path rehydrates the prior conversation (the rendered +// role prompt included), so prepending that prompt to the nudge only duplicates +// the already-restored role (the ~20k-token per-wake re-injection documented in +// gc-cbtfq). The nudge-LESS branch keeps #2477's behavior — the rendered prompt +// is the only payload that stops a resumed session from landing idle — and is +// covered by TestPhase2InitialInputDelivery's ResumeRestart requirements. +func TestPhase2ResumeRestartNudgePreservedWithoutPromptReplay(t *testing.T) { + // Model a legitimate resume (keyed transcript present) so the pre-flight + // guard does not reclassify these resumes as fresh starts; mirrors the stub + // in TestPhase2InitialInputDelivery. + prevProbe := staleResumeKeyProbe + staleResumeKeyProbe = func(string, string, string) (present, probeable bool) { return true, true } + t.Cleanup(func() { staleResumeKeyProbe = prevProbe }) + + for _, tc := range selectedPhase2ProviderCases(t) { + tc := tc + t.Run(string(tc.profileID), func(t *testing.T) { + prepared, prompt, nudge := preparePhase2ResumeRestartStartWithNudge(t, tc) + + switch { + case strings.TrimSpace(nudge) == "": + t.Fatalf("setup: resume restart nudge is empty; need a non-empty startup nudge to exercise the nudge-having branch") + case strings.TrimSpace(prompt) == "": + t.Fatalf("setup: rendered prompt is empty; need a non-empty prompt to detect re-injection") + case prepared.cfg.Nudge != nudge: + t.Fatalf("cfg.Nudge = %q, want the startup nudge alone (%q): the rendered prompt must not be prepended on a resume restart", prepared.cfg.Nudge, nudge) + case strings.Contains(prepared.cfg.Nudge, prompt): + t.Fatalf("cfg.Nudge = %q, want it to omit the rendered prompt %q (already rehydrated via --resume)", prepared.cfg.Nudge, prompt) + case strings.TrimSpace(prepared.cfg.PromptSuffix) != "": + t.Fatalf("cfg.PromptSuffix = %q, want empty on a resume restart", prepared.cfg.PromptSuffix) + case strings.TrimSpace(prepared.cfg.PromptFlag) != "": + t.Fatalf("cfg.PromptFlag = %q, want empty on a resume restart", prepared.cfg.PromptFlag) + case prepared.cfg.Env[startupPromptDeliveredEnv] != "1": + // Marker stays set so the SessionStart hook suppresses prompt + // re-injection: the prompt is present via --resume, not via the + // nudge, but either way the hook must not re-add it. + t.Fatalf("%s = %q, want 1 so the SessionStart hook does not re-inject the resumed prompt", startupPromptDeliveredEnv, prepared.cfg.Env[startupPromptDeliveredEnv]) + } + }) + } +} + +// preparePhase2ResumeRestartStartWithNudge mirrors preparePhase2ResumeRestartStart +// but keeps the resolved template's non-empty startup nudge instead of clearing +// it, exercising the nudge-having resume-restart branch. It returns the prepared +// start along with the rendered prompt and the startup nudge so callers can +// assert the prompt is not folded into the nudge. The restart-nudge branch does +// not depend on assigned work, so this helper omits the work-bead machinery. +func preparePhase2ResumeRestartStartWithNudge(t *testing.T, tc phase2ProviderCase) (prepared *preparedStart, prompt, nudge string) { + t.Helper() + + store := beads.NewMemStore() + session, err := store.Create(beads.Bead{ + Title: "phase2-" + tc.family, + Type: sessionBeadType, + Labels: []string{sessionBeadLabel}, + Metadata: map[string]string{ + "session_name": "phase2-" + tc.family, + "template": "worker", + "started_config_hash": "already-started", + "session_key": "phase2-resume-key", + }, + }) + if err != nil { + t.Fatalf("Create session bead: %v", err) + } + + tp := phase2TemplateParams(t, tc, "Base worker prompt") + prepared, err = prepareStartCandidate(startCandidate{ + info: sessiontest.SeedBead(t, session), + tp: tp, + }, &config.City{}, store, &clock.Fake{Time: time.Date(2026, 4, 5, 12, 0, 0, 0, time.UTC)}) + if err != nil { + t.Fatalf("prepareStartCandidate(%s): %v", tc.profileID, err) + } + return prepared, tp.Prompt, tp.Hints.Nudge +} diff --git a/cmd/gc/session_lifecycle_parallel_test.go b/cmd/gc/session_lifecycle_parallel_test.go index 31a20ecf63..cab0d5a548 100644 --- a/cmd/gc/session_lifecycle_parallel_test.go +++ b/cmd/gc/session_lifecycle_parallel_test.go @@ -1115,6 +1115,83 @@ func TestPrepareStartCandidate_DoesNotAppendCLIResumeFlagForACP(t *testing.T) { } } +// TestPrepareStartCandidate_ACPResumeKeepsStartupPromptInNudge is the +// regression guard for the ACP prompt-loss found on PR#61 (gc-xlj7s): the +// gc-7go2a nudge-only resume branch must NOT run for ACP sessions. ACP is +// excluded from CLI resume (resolveSessionCommand, gated on !tp.IsACP), and +// acp.Provider.Start always opens a fresh session/new — so the rendered role +// prompt rides the post-handshake nudge (prependStartupPromptToNudge), never a +// --resume rehydration. A Claude ACP session can still mint a session_key, so +// without the !tp.IsACP guard the resume branch would overwrite cfg.Nudge +// (prompt+nudge) with the bare hint while keeping GC_STARTUP_PROMPT_DELIVERED=1, +// leaving the restarted ACP agent with no role prompt at all. +func TestPrepareStartCandidate_ACPResumeKeepsStartupPromptInNudge(t *testing.T) { + store := beads.NewMemStore() + session, err := store.Create(beads.Bead{ + Title: "mayor", + Type: sessionBeadType, + Labels: []string{sessionBeadLabel, "agent:mayor"}, + Metadata: map[string]string{ + "template": "mayor", + "session_name": "mayor", + // A Claude ACP session mints a session_key, so the resume key is + // present... + "session_key": "claude-acp-session", + // ...and this is a restart, not a first start, so the resume branch + // is reachable. + "started_config_hash": "previous-start", + }, + }) + if err != nil { + t.Fatal(err) + } + + const prompt = "You are the mayor. Coordinate the city." + const nudgeHint = "Check your hook." + prepared, err := prepareStartCandidate(startCandidate{ + info: sessiontest.SeedBead(t, session), + tp: TemplateParams{ + TemplateName: "mayor", + SessionName: "mayor", + Command: "claude acp", + IsACP: true, + Prompt: prompt, + Hints: agent.StartupHints{Nudge: nudgeHint}, + ResolvedProvider: &config.ResolvedProvider{ + Name: "claude", + SessionIDFlag: "--session-id", + }, + }, + order: 0, + }, &config.City{}, store, &clock.Fake{Time: time.Date(2026, 5, 3, 10, 0, 0, 0, time.UTC)}) + if err != nil { + t.Fatalf("prepareStartCandidate: %v", err) + } + + // ACP delivers the rendered role prompt via the post-handshake nudge. The + // resume branch must not have stripped it down to the bare hint. + if !strings.Contains(prepared.cfg.Nudge, prompt) { + t.Fatalf("cfg.Nudge = %q, want it to still carry the rendered role prompt %q — ACP delivers the prompt via the nudge, not --resume rehydration", prepared.cfg.Nudge, prompt) + } + if !strings.Contains(prepared.cfg.Nudge, nudgeHint) { + t.Fatalf("cfg.Nudge = %q, want it to still carry the configured nudge hint %q", prepared.cfg.Nudge, nudgeHint) + } + // PromptSuffix is always empty for ACP (the prompt rides the nudge), but the + // delivered marker must stay set so the SessionStart hook does not ALSO + // inject the prompt the nudge already carries. + if strings.TrimSpace(prepared.cfg.PromptSuffix) != "" { + t.Fatalf("cfg.PromptSuffix = %q, want empty for ACP", prepared.cfg.PromptSuffix) + } + if prepared.cfg.Env[startupPromptDeliveredEnv] != "1" { + t.Fatalf("%s = %q, want 1 so the SessionStart hook does not double-inject the prompt delivered via the nudge", startupPromptDeliveredEnv, prepared.cfg.Env[startupPromptDeliveredEnv]) + } + // ACP is carved out of the resume override, so this incarnation genuinely + // re-delivers the rendered prompt and must stamp the S19 priming pair. + if !prepared.promptDelivered { + t.Fatalf("promptDelivered = false, want true: an ACP resume delivers the rendered prompt via the nudge and must stamp the priming pair") + } +} + func TestReconcileSessionBeads_BlockedCandidatesDoNotConsumeWakeBudget(t *testing.T) { env := newReconcilerTestEnv() env.cfg = &config.City{ @@ -1314,6 +1391,231 @@ func TestExecutePlannedStarts_WakeBudgetPrioritizesLeastRecentlyWoken(t *testing } } +// TestSortCandidatesByWakeFairness_RoundRobin proves the sort orders candidates +// least-recently-woken first, then interleaves templates round-robin so one +// template's backlog cannot occupy the front of the queue — while a lone template +// keeps strict least-recently-woken order. +func TestSortCandidatesByWakeFairness_RoundRobin(t *testing.T) { + cfg := &config.City{} + mk := func(name, template, lastWoke string) startCandidate { + return startCandidate{ + info: sessiontest.InfoFromMeta(t, map[string]string{ + "session_name": name, + "last_woke_at": lastWoke, + }), + tp: TemplateParams{SessionName: name, TemplateName: template}, + } + } + names := func(cs []startCandidate) []string { + out := make([]string, len(cs)) + for i, c := range cs { + out[i] = c.tp.SessionName + } + return out + } + + // Template A's backlog is older than B's lone candidate, so a pure fairness + // sort would front-load a1,a2,a3 and bury b1. Round-robin lifts b1 into the + // second slot, behind A's oldest but ahead of A's overflow. + mixed := []startCandidate{ + mk("a1", "A", "2020-01-01T00:00:00Z"), + mk("a2", "A", "2020-01-01T00:00:01Z"), + mk("a3", "A", "2020-01-01T00:00:02Z"), + mk("b1", "B", "2021-01-01T00:00:00Z"), + } + sortCandidatesByWakeFairness(mixed, cfg) + if got, want := names(mixed), []string{"a1", "b1", "a2", "a3"}; !reflect.DeepEqual(got, want) { + t.Fatalf("round-robin order = %v, want %v", got, want) + } + + // A single template has nothing to interleave with, so the order is exactly + // least-recently-woken first. + lone := []startCandidate{ + mk("p3", "pool", "2020-01-01T00:00:02Z"), + mk("p1", "pool", "2020-01-01T00:00:00Z"), + mk("p2", "pool", "2020-01-01T00:00:01Z"), + } + sortCandidatesByWakeFairness(lone, cfg) + if got, want := names(lone), []string{"p1", "p2", "p3"}; !reflect.DeepEqual(got, want) { + t.Fatalf("lone-template order = %v, want %v", got, want) + } +} + +// TestExecutePlannedStarts_RoundRobinProtectsForeignTemplate proves the +// round-robin interleave (gc-unpyk defense-in-depth). When one logical template +// has more ready candidates than the budget, it must not consume the ENTIRE +// per-tick budget and starve a different template's start: even though the greedy +// template's candidates are all older, round-robin lifts the foreign template's +// candidate into a slot. +func TestExecutePlannedStarts_RoundRobinProtectsForeignTemplate(t *testing.T) { + sp := runtime.NewFake() + store := beads.NewMemStore() + clk := &clock.Fake{Time: time.Date(2026, 6, 12, 20, 0, 0, 0, time.UTC)} + budget := 3 // round-robin order is greedy-1, victim-1, greedy-2; greedy-3 overflows + cfg := &config.City{Daemon: config.DaemonConfig{MaxWakesPerTick: &budget}} + + mkTP := func(sessionName, template string) TemplateParams { + return TemplateParams{ + Command: "claude", + SessionName: sessionName, + TemplateName: template, + ResolvedProvider: &config.ResolvedProvider{ + Name: "claude", + Command: "claude", + PromptMode: "arg", + ResumeFlag: "--resume", + ResumeStyle: "flag", + SessionIDFlag: "--session-id", + }, + } + } + + // Three "greedy" sessions (one template) woken long ago would, under a pure + // fairness sort, take all three slots and starve the single "victim" session + // of a different template. Round-robin interleaves the victim ahead of + // greedy's overflow so it still wins a slot. + specs := []struct{ name, template, lastWoke string }{ + {"greedy-1", "greedy", "2020-01-01T00:00:00Z"}, + {"greedy-2", "greedy", "2020-01-01T00:00:00Z"}, + {"greedy-3", "greedy", "2020-01-01T00:00:00Z"}, + {"victim-1", "victim", "2024-01-01T00:00:00Z"}, + } + desired := map[string]TemplateParams{} + var candidates []startCandidate + for i, s := range specs { + sess, err := store.Create(beads.Bead{ + Title: s.name, + Type: sessionBeadType, + Labels: []string{sessionBeadLabel}, + Metadata: map[string]string{ + "session_name": s.name, + "agent_name": s.name, + "template": s.template, + "state": "creating", + "generation": "1", + "instance_token": "test-token", + "live_hash": runtime.LiveFingerprint(runtime.Config{Command: "test-cmd"}), + "last_woke_at": s.lastWoke, + }, + }) + if err != nil { + t.Fatalf("Create(%s): %v", s.name, err) + } + sCopy := sess + tp := mkTP(s.name, s.template) + desired[s.name] = tp + candidates = append(candidates, startCandidate{info: sessiontest.SeedBead(t, sCopy), tp: tp, order: i}) + } + + woken := executePlannedStarts( + context.Background(), + candidates, + cfg, + desired, + sp, + store, + "", + clk, + events.Discard, + 5*time.Second, + ioDiscard{}, + ioDiscard{}, + ) + if woken != budget { + t.Fatalf("woken = %d, want %d", woken, budget) + } + if !sp.IsRunning("victim-1") { + t.Fatal("foreign template 'victim-1' was starved by the greedy template consuming the entire wake budget — round-robin not applied") + } + if sp.IsRunning("greedy-3") { + t.Fatal("greedy-3 started ahead of the foreign template; round-robin did not defer greedy's overflow") + } + for _, n := range []string{"greedy-1", "greedy-2"} { + if !sp.IsRunning(n) { + t.Fatalf("%s should have started", n) + } + } +} + +// TestExecutePlannedStarts_SingleTemplateUsesFullBudget proves round-robin never +// wastes a slot: with only one template ready (nothing to interleave with), the +// interleave reduces to least-recently-woken order and the lone template may use +// the FULL budget. +func TestExecutePlannedStarts_SingleTemplateUsesFullBudget(t *testing.T) { + sp := runtime.NewFake() + store := beads.NewMemStore() + clk := &clock.Fake{Time: time.Date(2026, 6, 12, 20, 0, 0, 0, time.UTC)} + budget := 3 // one template, so round-robin keeps all three in order + cfg := &config.City{Daemon: config.DaemonConfig{MaxWakesPerTick: &budget}} + + mkTP := func(sessionName string) TemplateParams { + return TemplateParams{ + Command: "claude", + SessionName: sessionName, + TemplateName: "pool", + ResolvedProvider: &config.ResolvedProvider{ + Name: "claude", + Command: "claude", + PromptMode: "arg", + ResumeFlag: "--resume", + ResumeStyle: "flag", + SessionIDFlag: "--session-id", + }, + } + } + + names := []string{"pool-1", "pool-2", "pool-3"} + desired := map[string]TemplateParams{} + var candidates []startCandidate + for i, name := range names { + sess, err := store.Create(beads.Bead{ + Title: name, + Type: sessionBeadType, + Labels: []string{sessionBeadLabel}, + Metadata: map[string]string{ + "session_name": name, + "agent_name": name, + "template": "pool", + "state": "creating", + "generation": "1", + "instance_token": "test-token", + "live_hash": runtime.LiveFingerprint(runtime.Config{Command: "test-cmd"}), + "last_woke_at": "2020-01-01T00:00:00Z", + }, + }) + if err != nil { + t.Fatalf("Create(%s): %v", name, err) + } + sCopy := sess + tp := mkTP(name) + desired[name] = tp + candidates = append(candidates, startCandidate{info: sessiontest.SeedBead(t, sCopy), tp: tp, order: i}) + } + + woken := executePlannedStarts( + context.Background(), + candidates, + cfg, + desired, + sp, + store, + "", + clk, + events.Discard, + 5*time.Second, + ioDiscard{}, + ioDiscard{}, + ) + if woken != budget { + t.Fatalf("woken = %d, want %d", woken, budget) + } + for _, n := range names { + if !sp.IsRunning(n) { + t.Fatalf("%s should have started — a single-template burst must use the full budget when nothing else contends", n) + } + } +} + func TestPrepareStartCandidate_NoneModeInitialMessageStaysInNudge(t *testing.T) { store := beads.NewMemStore() bead, err := store.Create(beads.Bead{ diff --git a/cmd/gc/session_model_phase0_cli_surface_spec_test.go b/cmd/gc/session_model_phase0_cli_surface_spec_test.go index bf1cf584ab..4095453936 100644 --- a/cmd/gc/session_model_phase0_cli_surface_spec_test.go +++ b/cmd/gc/session_model_phase0_cli_surface_spec_test.go @@ -342,6 +342,108 @@ mode = "always" } } +// TestPhase0CLISessionClose_DefaultsToGCSessionID verifies that +// `gc session close` with no positional arg falls back to +// $GC_SESSION_ID — the canonical way for an agent to self-close +// from inside its own runtime. +func TestPhase0CLISessionClose_DefaultsToGCSessionID(t *testing.T) { + cityDir := t.TempDir() + writePhase0InterfaceCity(t, cityDir, `[workspace] +name = "test-city" + +[beads] +provider = "file" + +[[agent]] +name = "worker" +start_command = "true" +max_active_sessions = 1 + +[[named_session]] +template = "worker" +mode = "always" +`) + t.Setenv("GC_CITY", cityDir) + t.Setenv("GC_DIR", t.TempDir()) + t.Setenv("GC_BEADS", "file") + t.Setenv("GC_SESSION", "fake") + + store, err := openCityStoreAt(cityDir) + if err != nil { + t.Fatalf("openCityStoreAt: %v", err) + } + bead, err := store.Create(beads.Bead{ + Title: "worker", + Type: session.BeadType, + Labels: []string{session.LabelSession}, + Metadata: map[string]string{ + "session_name": "test-city--worker", + "alias": "worker", + "template": "worker", + "configured_named_session": "true", + "configured_named_identity": "worker", + "configured_named_mode": "always", + "state": "suspended", + "continuity_eligible": "true", + }, + }) + if err != nil { + t.Fatalf("Create(named session): %v", err) + } + + t.Setenv("GC_SESSION_ID", bead.ID) + + var stdout, stderr bytes.Buffer + code := cmdSessionClose(nil, &stdout, &stderr) + if code != 0 { + t.Fatalf("cmdSessionClose(nil) = %d, want 0; stdout=%s stderr=%s", code, stdout.String(), stderr.String()) + } + reopened, err := openCityStoreAt(cityDir) + if err != nil { + t.Fatalf("reopen city store: %v", err) + } + got, err := reopened.Get(bead.ID) + if err != nil { + t.Fatalf("Get(%s): %v", bead.ID, err) + } + if got.Status != "closed" { + t.Fatalf("status = %q, want closed", got.Status) + } +} + +// TestPhase0CLISessionClose_NoArgsRequiresGCSessionID verifies +// that with no positional arg and $GC_SESSION_ID empty/unset the +// command exits non-zero with a clear stderr message instead of +// panicking on an empty argument list. +func TestPhase0CLISessionClose_NoArgsRequiresGCSessionID(t *testing.T) { + cityDir := t.TempDir() + writePhase0InterfaceCity(t, cityDir, `[workspace] +name = "test-city" + +[beads] +provider = "file" + +[[agent]] +name = "worker" +start_command = "true" +max_active_sessions = 1 +`) + t.Setenv("GC_CITY", cityDir) + t.Setenv("GC_DIR", t.TempDir()) + t.Setenv("GC_BEADS", "file") + t.Setenv("GC_SESSION", "fake") + t.Setenv("GC_SESSION_ID", "") + + var stdout, stderr bytes.Buffer + code := cmdSessionClose(nil, &stdout, &stderr) + if code == 0 { + t.Fatalf("cmdSessionClose(nil) with empty $GC_SESSION_ID succeeded; want failure. stdout=%s stderr=%s", stdout.String(), stderr.String()) + } + if msg := stderr.String(); !strings.Contains(msg, "GC_SESSION_ID") { + t.Fatalf("stderr = %q, want mention of GC_SESSION_ID", msg) + } +} + func TestPhase0MailRecipientIdentity_RejectsTemplateFactoryTarget(t *testing.T) { t.Setenv("GC_SESSION", "fake") diff --git a/cmd/gc/session_model_phase0_doctor_spec_test.go b/cmd/gc/session_model_phase0_doctor_spec_test.go index ebbd54697b..c932eecf07 100644 --- a/cmd/gc/session_model_phase0_doctor_spec_test.go +++ b/cmd/gc/session_model_phase0_doctor_spec_test.go @@ -82,11 +82,14 @@ func TestPhase0DoctorReportsStaleRoutedConfig(t *testing.T) { func TestPhase0DoctorReportsMissingBeadOwner(t *testing.T) { cityPath, store := newPhase0DoctorCity(t) + // Default test city derives prefix "tc" from "test-city". Use that + // prefix so the assignee is classified as a bead ID under the + // config-driven contract enforced by looksLikeSessionBeadID. if _, err := store.Create(beads.Bead{ Type: "task", Status: "open", Title: "missing owner", - Assignee: "gc-missing-session", + Assignee: "tc-missing-session", }); err != nil { t.Fatalf("create work bead: %v", err) } diff --git a/cmd/gc/session_model_phase0_rare_state_spec_test.go b/cmd/gc/session_model_phase0_rare_state_spec_test.go index 68d3be18e7..4f03ced312 100644 --- a/cmd/gc/session_model_phase0_rare_state_spec_test.go +++ b/cmd/gc/session_model_phase0_rare_state_spec_test.go @@ -709,6 +709,185 @@ func TestConfigDrift_AttachedSessionPersistsAcrossCycles(t *testing.T) { } } +func TestConfigDrift_ManualSessionPersistsAcrossCycles(t *testing.T) { + // An operator-owned manual shadow has no standing wake reason, so a + // config-drift drain would be an unrecoverable kill (named sessions + // restart-in-place; manual shadows can't). It must be exempted from + // config-drift drains — drift accepted — the same as the pool sweep + // already exempts manual sessions. + env := newReconcilerTestEnv() + env.cfg = &config.City{ + Workspace: config.Workspace{Name: "test-city"}, + Agents: []config.Agent{{ + Name: "worker", + StartCommand: "new-cmd", + MaxActiveSessions: intPtr(4), + }}, + } + + sessionName := "worker-thread-adhoc-abc123" + env.desiredState[sessionName] = TemplateParams{ + TemplateName: "worker", + InstanceName: sessionName, + Alias: sessionName, + Command: "new-cmd", + } + + oldRuntime := runtime.Config{Command: "old-cmd"} + oldStartedHash := runtime.CoreFingerprint(oldRuntime) + if err := env.sp.Start(context.Background(), sessionName, oldRuntime); err != nil { + t.Fatalf("Start(old runtime): %v", err) + } + // Deliberately NOT attached: a shadow the operator has detached from. + + session := env.createSessionBead(sessionName, "worker") + env.markSessionActive(&session) + env.setSessionMetadata(&session, map[string]string{ + "session_origin": "manual", + "session_key": "old-provider-conversation", + "started_config_hash": oldStartedHash, + "started_live_hash": runtime.LiveFingerprint(oldRuntime), + }) + + // Multiple reconcile cycles — the shadow must survive all of them. + for i := 0; i < 5; i++ { + env.clk.Time = env.clk.Now().Add(10 * time.Second) + got, err := env.store.Get(session.ID) + if err != nil { + t.Fatalf("cycle %d: Get(%s): %v", i, session.ID, err) + } + env.reconcile([]beads.Bead{got}) + + if !env.sp.IsRunning(sessionName) { + t.Fatalf("cycle %d: manual shadow was stopped during config-drift", i) + } + got, err = env.store.Get(session.ID) + if err != nil { + t.Fatalf("cycle %d: Get after reconcile: %v", i, err) + } + if got.Metadata["state"] == "creating" { + t.Fatalf("cycle %d: state = creating; want drift accepted (no restart)", i) + } + if got.Metadata["session_key"] != "old-provider-conversation" { + t.Fatalf("cycle %d: session_key = %q; want conversation preserved", i, got.Metadata["session_key"]) + } + } + + // The drift was accepted (hash rebaselined), not left to re-drain. + got, err := env.store.Get(session.ID) + if err != nil { + t.Fatalf("final Get: %v", err) + } + if got.Metadata["started_config_hash"] == oldStartedHash { + t.Fatal("started_config_hash unchanged; want rebaselined to current (drift accepted)") + } + if got.Metadata["started_config_hash"] == "" { + t.Fatal("started_config_hash cleared; want rebaselined, not cleared") + } +} + +func TestConfigDrift_ManualSessionAppliesLiveDriftWithoutRestart(t *testing.T) { + // A manual shadow whose config edit changed BOTH a core field and + // session_live must keep running (core drift accepted, no drain) AND still + // get the live change applied via RunLive. Accepting core drift by stamping + // every fingerprint field — started_live_hash included — would make the next + // tick believe session_live was already applied when RunLive never ran, so + // the live commands would be silently dropped while the metadata claims they + // landed. Regression for the codex review on PR#25: rebaseline only the core + // hash so the live-drift path re-applies session_live on the next tick. + env := newReconcilerTestEnv() + env.cfg = &config.City{ + Workspace: config.Workspace{Name: "test-city"}, + Agents: []config.Agent{{ + Name: "worker", + StartCommand: "new-cmd", + MaxActiveSessions: intPtr(4), + }}, + } + + sessionName := "worker-thread-adhoc-live1" + // Desired state drifts on BOTH core (command) and live (session_live). + tp := TemplateParams{ + TemplateName: "worker", + InstanceName: sessionName, + Alias: sessionName, + Command: "new-cmd", + } + tp.Hints.SessionLive = []string{"echo new-live"} + env.desiredState[sessionName] = tp + + oldRuntime := runtime.Config{Command: "old-cmd"} + oldStartedHash := runtime.CoreFingerprint(oldRuntime) + oldLiveHash := runtime.LiveFingerprint(oldRuntime) + if err := env.sp.Start(context.Background(), sessionName, oldRuntime); err != nil { + t.Fatalf("Start(old runtime): %v", err) + } + // Deliberately NOT attached: a shadow the operator has detached from. + + session := env.createSessionBead(sessionName, "worker") + env.markSessionActive(&session) + env.setSessionMetadata(&session, map[string]string{ + "session_origin": "manual", + "session_key": "old-provider-conversation", + "started_config_hash": oldStartedHash, + "started_live_hash": oldLiveHash, + }) + + // The live hash the running shadow must converge to once session_live is + // re-applied. Guard the setup: the test is meaningless without live drift. + wantLive := runtime.LiveFingerprint(templateParamsToConfig(tp)) + if wantLive == oldLiveHash { + t.Fatal("test setup: desired live hash matches old; no live drift to exercise") + } + + // Multiple reconcile cycles — the shadow survives all of them and the live + // change reaches it via RunLive (no restart, conversation preserved). + for i := 0; i < 5; i++ { + env.clk.Time = env.clk.Now().Add(10 * time.Second) + got, err := env.store.Get(session.ID) + if err != nil { + t.Fatalf("cycle %d: Get(%s): %v", i, session.ID, err) + } + env.reconcile([]beads.Bead{got}) + + if !env.sp.IsRunning(sessionName) { + t.Fatalf("cycle %d: manual shadow was stopped during config-drift", i) + } + got, err = env.store.Get(session.ID) + if err != nil { + t.Fatalf("cycle %d: Get after reconcile: %v", i, err) + } + if got.Metadata["state"] == "creating" { + t.Fatalf("cycle %d: state = creating; want drift accepted (no restart)", i) + } + if got.Metadata["session_key"] != "old-provider-conversation" { + t.Fatalf("cycle %d: session_key = %q; want conversation preserved", i, got.Metadata["session_key"]) + } + } + + // THE regression guard: the live change must have been applied to the + // running shadow. Without the core-only rebaseline, started_live_hash is + // stamped to current on the same tick the drift is accepted, so the + // live-drift path never fires and RunLive is never called (count 0). + if n := env.sp.CountCalls("RunLive", sessionName); n == 0 { + t.Fatal("RunLive never called: session_live silently dropped (manual core-drift rebaseline masked the concurrent live drift)") + } + + got, err := env.store.Get(session.ID) + if err != nil { + t.Fatalf("final Get: %v", err) + } + // Core drift was accepted (rebaselined), not left to re-drain. + if got.Metadata["started_config_hash"] == oldStartedHash { + t.Fatal("started_config_hash unchanged; want rebaselined to current (core drift accepted)") + } + // Live baseline converged to the desired config — the claim that the live + // config is applied is now true because RunLive actually ran. + if got.Metadata["started_live_hash"] != wantLive { + t.Fatalf("started_live_hash = %q; want %q (session_live applied)", got.Metadata["started_live_hash"], wantLive) + } +} + func TestConfigDrift_AttachedSessionSurvivesTransientFalseNegative(t *testing.T) { env := newReconcilerTestEnv() env.cfg = &config.City{ diff --git a/cmd/gc/session_reconciler.go b/cmd/gc/session_reconciler.go index 25e11f9ba7..fbb8f7c60f 100644 --- a/cmd/gc/session_reconciler.go +++ b/cmd/gc/session_reconciler.go @@ -2712,6 +2712,33 @@ func reconcileSessionBeadsTracedWithNamedDemand( } continue } + if isManualSessionInfo(infoByID[id]) { + // Operator-owned shadow: no standing wake reason, so a + // config-drift drain is an unrecoverable kill, not a + // restart-in-place. Accept the core drift like the pool + // sweep and leave the shadow running. + // + // Rebaseline ONLY the core fingerprint, never the live + // fingerprint. If this same config edit also changed + // session_live, stamping started_live_hash here would + // make the next tick believe the live config was already + // applied when RunLive never ran — silently dropping the + // live change. Leaving the live hash stale lets the + // live-drift path below re-apply session_live via RunLive + // on the next tick (live changes need no restart). + rebaseBatch, rebaseErr := silentRebaselineSessionCoreHash(id, sessFront, agentCfg) + if rebaseErr != nil { + fmt.Fprintf(stderr, "session reconciler: rebaselining manual-session config-drift hash for %s: %v\n", name, rebaseErr) //nolint:errcheck + } + tick.apply(id, rebaseBatch) + cancelSessionConfigDriftDrainInfo(infoByID[id], sp, dt) + if trace != nil { + trace.RecordDecision(TraceSiteReconcilerConfigDrift, TraceReasonConfigDrift, TraceOutcomeDeferredActive, tp.TemplateName, name, configDriftTracePayload(storedHash, currentHash, driftedFields, traceRecordPayload{ + "active_reason": "manual_session", + })) + } + continue + } if isNamedSessionInfo(infoByID[id]) { // Defer config-drift restart for named sessions // that are actively in use (pending interaction, @@ -5367,6 +5394,35 @@ func silentRebaselineSessionHashes(id string, sessFront *sessionpkg.Store, agent return patch, nil } +// silentRebaselineSessionCoreHash advances ONLY the core fingerprint baseline +// (started_config_hash + core_hash_breakdown), leaving the live fingerprint +// fields (started_live_hash, live_hash) stale. The reconciler uses this to +// accept core config drift for a session it must not restart (an operator-owned +// manual shadow) without masking a concurrent session_live change: stamping the +// live half here would make the next tick believe session_live was already +// applied when RunLive never ran. Leaving the live hash stale lets the +// live-drift path re-apply session_live via RunLive on the next tick. +func silentRebaselineSessionCoreHash(id string, sessFront *sessionpkg.Store, agentCfg runtime.Config) (map[string]string, error) { + if id == "" || sessFront == nil { + return nil, nil + } + breakdownJSON, err := json.Marshal(runtime.CoreFingerprintBreakdown(agentCfg)) + if err != nil { + return nil, fmt.Errorf("marshaling core_hash_breakdown: %w", err) + } + patch := map[string]string{ + "started_config_hash": runtime.CoreFingerprint(agentCfg), + "core_hash_breakdown": string(breakdownJSON), + } + if err := sessFront.ApplyPatch(id, patch); err != nil { + return nil, fmt.Errorf("rebaselining core hash: %w", err) + } + // The caller folds the returned patch onto the typed snapshot (tick.apply); + // the raw session.Metadata mirror was retired (Step 5c) — the rebaselined + // hash is never read off the raw bead this tick. + return patch, nil +} + // relaunchAgentForLaunchDrift handles a launch-only config-drift (B2.3): the // LaunchFingerprint moved while the ProvisionFingerprint held, so the agent can // be re-launched in the existing warm box instead of a full re-provision diff --git a/cmd/gc/session_wake_fairness_pin_test.go b/cmd/gc/session_wake_fairness_pin_test.go index 9227b376a4..3472fb4d06 100644 --- a/cmd/gc/session_wake_fairness_pin_test.go +++ b/cmd/gc/session_wake_fairness_pin_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/gastownhall/gascity/internal/beads" + "github.com/gastownhall/gascity/internal/config" "github.com/gastownhall/gascity/internal/session" "github.com/gastownhall/gascity/internal/session/sessiontest" ) @@ -144,7 +145,7 @@ func TestWakeFairnessInfoTwinCharacterization(t *testing.T) { })) cands := []startCandidate{recentlyWoken, newSlept, oldSlept} - sortCandidatesByWakeFairness(cands) + sortCandidatesByWakeFairness(cands, &config.City{}) gotOrder := []string{cands[0].info.ID, cands[1].info.ID, cands[2].info.ID} wantOrder := []string{"ga-old", "ga-new", "ga-recent"} for i := range wantOrder { diff --git a/cmd/gc/skill_integration.go b/cmd/gc/skill_integration.go index 4814deef03..22d85e9664 100644 --- a/cmd/gc/skill_integration.go +++ b/cmd/gc/skill_integration.go @@ -6,7 +6,10 @@ import ( "os" "path/filepath" "strings" + "sync" + "github.com/gastownhall/gascity/internal/builtinpacks" + "github.com/gastownhall/gascity/internal/citylayout" "github.com/gastownhall/gascity/internal/config" "github.com/gastownhall/gascity/internal/materialize" "github.com/gastownhall/gascity/internal/runtime" @@ -249,15 +252,19 @@ func effectiveSkillsForAgent(city *materialize.CityCatalog, agent *config.Agent, } // mergeSkillFingerprintEntries adds one "skills:" → content-hash -// entry to fpExtra for each desired skill. Hashes use -// runtime.HashPathContent so any byte-level change to a skill's source -// directory triggers a config-fingerprint drift and drains the agent. +// entry to fpExtra for each desired skill so a byte-level change to a +// skill's source triggers a config-fingerprint drift and drains the agent. +// +// The hash source depends on the skill's origin (see skillFingerprintHash): +// builtin system-pack skills hash the running binary's embedded bytes (stable +// across foreign restaging in a self-hosting city); rig/agent/user skills hash +// the on-disk source directory so live edits still reload the agent. // // Nil-map safe: allocates fpExtra if the caller passed nil. Returns // the (possibly new) map. The "skills:" prefix partitions the key // space so entries cannot collide with other fpExtra keys // (pool.min/pool.max/wake_mode/etc.). -func mergeSkillFingerprintEntries(fpExtra map[string]string, desired []materialize.SkillEntry) map[string]string { +func mergeSkillFingerprintEntries(cityPath string, fpExtra map[string]string, desired []materialize.SkillEntry) map[string]string { if len(desired) == 0 { return fpExtra } @@ -265,11 +272,92 @@ func mergeSkillFingerprintEntries(fpExtra map[string]string, desired []materiali fpExtra = make(map[string]string, len(desired)) } for _, e := range desired { - fpExtra["skills:"+e.Name] = runtime.HashPathContent(e.Source) + fpExtra["skills:"+e.Name] = skillFingerprintHash(cityPath, e) } return fpExtra } +// builtinSkillFingerprintCache memoizes embedded builtin-pack skill hashes by +// "\x00". The embedded bytes are constant for the process +// lifetime, so the hash never changes; caching avoids re-walking the embedded +// FS for every skill on every reconciler tick. +var builtinSkillFingerprintCache sync.Map + +// builtinSystemPackSkillHash returns a deterministic content hash for a skill +// whose source is a materialized builtin system pack +// (/.gc/system/packs//...), derived from the running binary's +// EMBEDDED pack bytes rather than the on-disk copy. Returns ok=false when the +// source is not a builtin system-pack skill (rig/agent/user skills, an +// unrecognized pack, or a skill absent from the embedded pack), so the caller +// falls back to hashing the on-disk source. +// +// Rationale: every gc config-load restages the embedded builtin packs into the +// shared /.gc/system/packs path (see MaterializeBuiltinPacks). In a +// self-hosting city, agents run gc binaries built from divergent worktrees, so +// they overwrite that shared path with different SKILL.md bytes between the +// supervisor's reconciler ticks. Hashing the on-disk copy therefore flaps the +// CoreFingerprint and spins config-drift restarts until the wake budget +// starves. The supervisor's embedded bytes are constant for its process +// lifetime and are exactly what it will re-stage on the agent's next launch, so +// they are the correct, stable fingerprint input. Both started_config_hash and +// the per-tick drift hash are computed by the supervisor process, so the +// embedded hash stays internally consistent across start and drift checks; a +// genuine binary upgrade restarts the supervisor and yields one correct drift. +func builtinSystemPackSkillHash(cityPath, source string) (string, bool) { + if cityPath == "" || source == "" { + return "", false + } + systemRoot, err := filepath.Abs(filepath.Join(cityPath, citylayout.SystemPacksRoot)) + if err != nil { + return "", false + } + absSource, err := filepath.Abs(source) + if err != nil { + return "", false + } + rel, err := filepath.Rel(systemRoot, absSource) + if err != nil { + return "", false + } + rel = filepath.ToSlash(rel) + if rel == "." || rel == ".." || strings.HasPrefix(rel, "../") { + return "", false + } + // Need /; the pack root itself is not a skill source. + parts := strings.SplitN(rel, "/", 2) + if len(parts) < 2 || parts[0] == "" || parts[1] == "" { + return "", false + } + packName, skillRel := parts[0], parts[1] + bp, ok := builtinpacks.ByName(packName) + if !ok { + return "", false + } + cacheKey := packName + "\x00" + skillRel + if v, ok := builtinSkillFingerprintCache.Load(cacheKey); ok { + return v.(string), true + } + h := runtime.HashFSContent(bp.FS, skillRel) + if h == "" { + // Embedded subtree missing — fall back to disk hashing rather than + // poisoning the fingerprint with the HASH_UNAVAILABLE sentinel. + return "", false + } + builtinSkillFingerprintCache.Store(cacheKey, h) + return h, true +} + +// skillFingerprintHash returns the fingerprint content hash for one desired +// skill entry. Builtin system-pack skills hash the binary's embedded bytes +// (stable across foreign restaging); rig/agent/user skills hash the on-disk +// source content so live edits to those still drain and restart the agent. +func skillFingerprintHash(cityPath string, e materialize.SkillEntry) string { + if h, ok := builtinSystemPackSkillHash(cityPath, e.Source); ok { + return h + } + return runtime.HashPathContent(e.Source) +} + // effectiveInjectAssignedSkills resolves the agent's prompt-appendix // preference. Returns true by default (nil pointer → inject) so the // feature is opt-out rather than opt-in. An explicit agent-level diff --git a/cmd/gc/skill_integration_test.go b/cmd/gc/skill_integration_test.go index 113beef002..126447583d 100644 --- a/cmd/gc/skill_integration_test.go +++ b/cmd/gc/skill_integration_test.go @@ -8,8 +8,10 @@ import ( "testing" "time" + "github.com/gastownhall/gascity/internal/builtinpacks" "github.com/gastownhall/gascity/internal/config" "github.com/gastownhall/gascity/internal/materialize" + "github.com/gastownhall/gascity/internal/runtime" ) func TestIsStage2EligibleSession(t *testing.T) { @@ -340,7 +342,7 @@ func TestMergeSkillFingerprintEntries(t *testing.T) { } // Nil fpExtra: allocates and populates. - got := mergeSkillFingerprintEntries(nil, desired) + got := mergeSkillFingerprintEntries("", nil, desired) if len(got) != 2 { t.Fatalf("len = %d, want 2; %+v", len(got), got) } @@ -352,7 +354,7 @@ func TestMergeSkillFingerprintEntries(t *testing.T) { // Non-nil fpExtra: preserves existing keys. base := map[string]string{"pool.max": "3"} - got = mergeSkillFingerprintEntries(base, desired) + got = mergeSkillFingerprintEntries("", base, desired) if got["pool.max"] != "3" { t.Errorf("existing key dropped: %+v", got) } @@ -362,7 +364,7 @@ func TestMergeSkillFingerprintEntries(t *testing.T) { // Empty desired: returns input unchanged. orig := map[string]string{"x": "y"} - got = mergeSkillFingerprintEntries(orig, nil) + got = mergeSkillFingerprintEntries("", orig, nil) if !reflect.DeepEqual(got, orig) { t.Errorf("empty desired modified map: got %+v, want %+v", got, orig) } @@ -377,7 +379,7 @@ func TestMergeSkillFingerprintEntriesPrefixPartitioning(t *testing.T) { mustCreateSkill(t, filepath.Join(tmp, "x")) desired := []materialize.SkillEntry{{Name: "x", Source: filepath.Join(tmp, "x")}} - got := mergeSkillFingerprintEntries(nil, desired) + got := mergeSkillFingerprintEntries("", nil, desired) for k := range got { if !strings.HasPrefix(k, "skills:") { t.Errorf("non-prefix key present: %q", k) @@ -385,6 +387,73 @@ func TestMergeSkillFingerprintEntriesPrefixPartitioning(t *testing.T) { } } +// TestSkillFingerprintHashBuiltinIgnoresOnDiskStomp is the gc-155rj regression: +// a skill materialized under /.gc/system/packs//skills/ +// must fingerprint the binary's EMBEDDED bytes, not the on-disk copy. In a +// self-hosting city, foreign gc binaries restage that shared path with +// divergent SKILL.md content between reconciler ticks; hashing disk flapped the +// CoreFingerprint and spun config-drift restarts until the wake budget starved. +func TestSkillFingerprintHashBuiltinIgnoresOnDiskStomp(t *testing.T) { + const pack, skill = "core", "gc-dispatch" + bp, ok := builtinpacks.ByName(pack) + if !ok { + t.Skipf("builtin pack %q not present in this binary", pack) + } + embedded := runtime.HashFSContent(bp.FS, "skills/"+skill) + if embedded == "" { + t.Skipf("embedded pack %q ships no skills/%s", pack, skill) + } + + cityPath := t.TempDir() + skillDir := filepath.Join(cityPath, ".gc", "system", "packs", pack, "skills", skill) + if err := os.MkdirAll(skillDir, 0o755); err != nil { + t.Fatal(err) + } + skillMD := filepath.Join(skillDir, "SKILL.md") + writeStomp := func(content string) { + if err := os.WriteFile(skillMD, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + writeStomp("STOMPED-BY-FOREIGN-BINARY-v1") + + e := materialize.SkillEntry{Name: pack + "." + skill, Source: skillDir} + // Must equal the embedded hash, never the on-disk hash. + if got := skillFingerprintHash(cityPath, e); got != embedded { + t.Fatalf("hash = %q, want embedded %q (must ignore on-disk content)", got, embedded) + } + if onDisk := runtime.HashPathContent(skillDir); onDisk == embedded { + t.Fatal("test precondition broken: on-disk content must differ from embedded") + } + // Re-stomp with different content: the fingerprint must stay constant. + writeStomp("STOMPED-BY-FOREIGN-BINARY-v2-DIFFERENT") + if got := skillFingerprintHash(cityPath, e); got != embedded { + t.Fatalf("fingerprint flapped under on-disk mutation: %q != embedded %q", got, embedded) + } +} + +// TestSkillFingerprintHashNonBuiltinUsesDisk asserts rig/agent/user skills +// (not under .gc/system/packs) keep on-disk hashing, so a live edit still +// drains and reloads the agent. +func TestSkillFingerprintHashNonBuiltinUsesDisk(t *testing.T) { + cityPath := t.TempDir() + skillDir := filepath.Join(cityPath, "rigs", "myrig", "packs", "p", "skills", "custom") + mustCreateSkill(t, skillDir) + e := materialize.SkillEntry{Name: "custom", Source: skillDir} + + got := skillFingerprintHash(cityPath, e) + if want := runtime.HashPathContent(skillDir); got != want { + t.Fatalf("non-builtin skill hash = %q, want on-disk %q", got, want) + } + // A live edit changes the fingerprint. + if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("edited-body"), 0o644); err != nil { + t.Fatal(err) + } + if got2 := skillFingerprintHash(cityPath, e); got2 == got { + t.Fatal("live edit to a non-builtin skill must change the fingerprint") + } +} + func TestEffectiveInjectAssignedSkills(t *testing.T) { t.Parallel() yes, no := true, false diff --git a/cmd/gc/telemetry_lifecycle_metrics_test.go b/cmd/gc/telemetry_lifecycle_metrics_test.go index fafb6c8192..c6a68392c2 100644 --- a/cmd/gc/telemetry_lifecycle_metrics_test.go +++ b/cmd/gc/telemetry_lifecycle_metrics_test.go @@ -563,6 +563,7 @@ func TestCmdSessionKill_RecordsAgentStopMetric(t *testing.T) { func() {}, nil, nil, + nil, make(chan reloadRequest), make(chan convergenceRequest, 1), make(chan struct{}, 1), diff --git a/cmd/gc/template_resolve.go b/cmd/gc/template_resolve.go index e2bbd6a53f..9962bb0365 100644 --- a/cmd/gc/template_resolve.go +++ b/cmd/gc/template_resolve.go @@ -349,6 +349,10 @@ func resolveTemplate(p *agentBuildParams, cfgAgent *config.Agent, qualifiedName if p.city != nil { beadsCfg = p.city.Beads } + configDir := p.cityPath + if cfgAgent.SourceDir != "" { + configDir = cfgAgent.SourceDir + } prompt = renderPrompt(p.fs, p.cityPath, p.cityName, cfgAgent.PromptTemplate, PromptContext{ CityRoot: p.cityPath, AgentName: qualifiedName, @@ -360,6 +364,8 @@ func resolveTemplate(p *agentBuildParams, cfgAgent *config.Agent, qualifiedName WorkDir: workDir, IssuePrefix: findRigPrefix(rigName, p.rigs), DefaultBranch: defaultBranchForRig(rigName, p.rigs, workDir), + ConfigDir: configDir, + DefaultMergeStrategy: mergeStrategyForRig(rigName, p.rigs, p.city), AssignedInProgressQuery: expandAgentCommandTemplate(p.cityPath, p.cityName, cfgAgent, p.rigs, "assigned_in_progress_query", cfgAgent.EffectiveAssignedInProgressQueryForBeads(beadsCfg), p.stderr), AssignedReadyQuery: expandAgentCommandTemplate(p.cityPath, p.cityName, cfgAgent, p.rigs, "assigned_ready_query", cfgAgent.EffectiveAssignedReadyQueryForBeads(beadsCfg), p.stderr), RoutedPoolQuery: expandAgentCommandTemplate(p.cityPath, p.cityName, cfgAgent, p.rigs, "routed_pool_query", cfgAgent.EffectiveRoutedPoolQueryForBeads(beadsCfg), p.stderr), @@ -433,11 +439,35 @@ func resolveTemplate(p *agentBuildParams, cfgAgent *config.Agent, qualifiedName // Step 10: Merge environment layers. Workspace.Env sits between // passthrough and provider so a per-provider/agent/patch entry can // still override a workspace-wide default. + // + // Each [env] layer's values are expanded against {passthrough, + // agentEnv} so that ${VAR} references resolve correctly. The + // expansion source must include agentEnv because the supervisor's + // own os.Environ() does NOT carry GT_ROOT, GC_RIG, GC_RIG_ROOT, + // GC_DIR, etc. — those are computed in this function and would + // silently expand to empty under os.ExpandEnv (the previous + // behavior that bit PR #32, PR #34, and gc-rch40w). + // + // workspaceEnv, resolved.Env (provider preset env merged with + // agent.Env via mergeAgentOverrides), and cfgAgent.Env (agent.toml + // [env] only) all expand against the same source. Cross-layer + // expansion (cfgAgent.Env referencing already-expanded resolved.Env + // values) is deliberately not supported here: because + // mergeAgentOverrides already folds agent.Env into resolved.Env, a + // layered expansion would re-expand references like "${PATH}" + // against a PATH that already contains the agent's own augmentation, + // producing duplicated segments. + // + // Final merge order keeps agentEnv last so its values win on key + // collisions; agentEnv values are concrete strings produced by this + // function and are not expanded. var workspaceEnv map[string]string if p.workspace != nil { workspaceEnv = p.workspace.Env } - env := mergeEnv(passthroughEnv(), expandEnvMap(workspaceEnv), expandEnvMap(resolved.Env), expandEnvMap(cfgAgent.Env), agentEnv) + pthru := passthroughEnv() + expansionSrc := mergeEnv(pthru, agentEnv) + env := mergeEnv(pthru, expandEnvMap(expansionSrc, workspaceEnv), expandEnvMap(expansionSrc, resolved.Env), expandEnvMap(expansionSrc, cfgAgent.Env), agentEnv) prependGCBinDirToPATH(env, env["GC_BIN"]) env = convergence.ScrubTokenEnv(env) @@ -489,7 +519,10 @@ func resolveTemplate(p *agentBuildParams, cfgAgent *config.Agent, qualifiedName } // Raw env is the harness-specific escape hatch, merged LAST (wins over the // abstract render and ambient/agent env for the keys it sets). - for k, v := range expandEnvMap(spec.Env) { + // nil expansion source preserves upstream's process-env-only expansion + // (os.ExpandEnv) for the upstream spec's raw env after the fork widened + // expandEnvMap to a two-arg (src, m) signature (gc-rch40w). + for k, v := range expandEnvMap(nil, spec.Env) { env[k] = v } } @@ -498,11 +531,9 @@ func resolveTemplate(p *agentBuildParams, cfgAgent *config.Agent, qualifiedName // re-enable product metrics; Beads telemetry remains independent. env[execenv.UsageMetricsDisableEnv] = execenv.UsageMetricsDisableValue - // Step 11: Expand session setup templates. - configDir := p.cityPath - if cfgAgent.SourceDir != "" { - configDir = cfgAgent.SourceDir - } + // Step 11: Expand session setup templates. configDir resolved above + // (shared with the renderPrompt PromptContext) so both wires use the + // same pack source dir. setupCtx := SessionSetupContext{ Session: sessName, Agent: qualifiedName, @@ -542,7 +573,7 @@ func resolveTemplate(p *agentBuildParams, cfgAgent *config.Agent, qualifiedName sharedCatalog := p.sharedSkillCatalogSnapshotForAgent(cfgAgent) desired := effectiveSkillsForAgent(sharedCatalog, cfgAgent, wsProvider, p.providers, p.stderr) if len(desired) > 0 { - fpExtra = mergeSkillFingerprintEntries(fpExtra, desired) + fpExtra = mergeSkillFingerprintEntries(p.cityPath, fpExtra, desired) if canonWorkDir != scopeRoot { // Pool instances inherit their skill catalog from the // template, not the instance — namepool members (e.g. diff --git a/cmd/gc/template_resolve_env_test.go b/cmd/gc/template_resolve_env_test.go index e8ea835c79..6f17d8101a 100644 --- a/cmd/gc/template_resolve_env_test.go +++ b/cmd/gc/template_resolve_env_test.go @@ -238,3 +238,160 @@ func TestResolveTemplateInjectsPerDispatcherTraceDefault(t *testing.T) { }) } } + +// TestResolveTemplateExpandsAgentEnvVarsInConfiguredEnv verifies that +// agent.toml [env] values can reference city/rig-scoped vars +// (GT_ROOT, GC_ALIAS, GC_RIG, GC_RIG_ROOT, GC_DIR) using ${VAR} syntax +// even when those vars are not present in the supervisor's own process +// environment. This is the gc-rch40w bug that broke PR #32 (BASH_ENV +// expanded to "/rigs/.../init.sh" with a stray leading slash because +// ${GT_ROOT} silently expanded to "") and PR #34 (PATH lost the +// rig-scoped bin dir for the same reason). +// +// The fix expands cfgAgent.Env (and resolved.Env) against an environment +// that includes the in-flight agentEnv, not just os.Environ() of the +// supervisor. ${PATH} must still resolve from the supervisor passthrough. +func TestResolveTemplateExpandsAgentEnvVarsInConfiguredEnv(t *testing.T) { + cityPath := t.TempDir() + writeTemplateResolveCityConfig(t, cityPath, "file") + rigRoot := filepath.Join(cityPath, "demo") + if err := os.MkdirAll(rigRoot, 0o755); err != nil { + t.Fatal(err) + } + sep := string(os.PathListSeparator) + t.Setenv("PATH", "/usr/bin") + // Match the production supervisor: these vars are NOT in the + // process env. Set them to empty defensively in case the test + // harness inherited them from a parent shell — empty is + // indistinguishable from unset under os.Getenv. + t.Setenv("GT_ROOT", "") + t.Setenv("GC_ALIAS", "") + t.Setenv("GC_RIG", "") + t.Setenv("GC_RIG_ROOT", "") + t.Setenv("GC_DIR", "") + + params := &agentBuildParams{ + cityName: "city", + cityPath: cityPath, + workspace: &config.Workspace{Provider: "test"}, + providers: map[string]config.ProviderSpec{"test": {Command: "echo", PromptMode: "none"}}, + lookPath: func(string) (string, error) { return "/bin/echo", nil }, + fs: fsys.OSFS{}, + rigs: []config.Rig{{Name: "demo", Path: rigRoot}}, + beaconTime: time.Unix(0, 0), + beadNames: make(map[string]string), + stderr: io.Discard, + } + + agent := &config.Agent{ + Name: "runner", + Dir: "demo", + Env: map[string]string{ + "PATH": "${GT_ROOT}/rigs/x/bin" + sep + "${PATH}", + "BASH_ENV": "${GC_RIG_ROOT}/init.sh", + "ALIAS_AT": "${GC_ALIAS}", + "RIG_AT": "${GC_RIG}", + "DIR_AT": "${GC_DIR}", + "BEADS_AT": "${BEADS_DIR}", + }, + } + qualifiedName := agent.QualifiedName() + tp, err := resolveTemplate(params, agent, qualifiedName, nil) + if err != nil { + t.Fatalf("resolveTemplate: %v", err) + } + + // PATH must contain the GT_ROOT-prefixed segment, not the broken + // form "/rigs/x/bin" produced when ${GT_ROOT} silently expanded + // to empty. The supervisor's ${PATH} must also remain resolvable. + wantSegment := cityPath + "/rigs/x/bin" + pathSegments := strings.Split(tp.Env["PATH"], sep) + foundExpanded := false + for _, seg := range pathSegments { + if seg == "/rigs/x/bin" { + t.Fatalf("PATH = %q contains broken segment %q — ${GT_ROOT} expanded to empty", tp.Env["PATH"], seg) + } + if seg == wantSegment { + foundExpanded = true + } + } + if !foundExpanded { + t.Fatalf("PATH = %q, want a segment %q (expanded ${GT_ROOT})", tp.Env["PATH"], wantSegment) + } + foundUsrBin := false + for _, seg := range pathSegments { + if seg == "/usr/bin" { + foundUsrBin = true + break + } + } + if !foundUsrBin { + t.Fatalf("PATH = %q, want a segment /usr/bin (expanded ${PATH} from passthroughEnv)", tp.Env["PATH"]) + } + + wantBashEnv := rigRoot + "/init.sh" + if got := tp.Env["BASH_ENV"]; got != wantBashEnv { + t.Fatalf("BASH_ENV = %q, want %q (expanded ${GC_RIG_ROOT})", got, wantBashEnv) + } + if got := tp.Env["ALIAS_AT"]; got != qualifiedName { + t.Fatalf("ALIAS_AT = %q, want %q (expanded ${GC_ALIAS})", got, qualifiedName) + } + if got := tp.Env["RIG_AT"]; got != "demo" { + t.Fatalf("RIG_AT = %q, want %q (expanded ${GC_RIG})", got, "demo") + } + if got := tp.Env["DIR_AT"]; got != tp.WorkDir { + t.Fatalf("DIR_AT = %q, want %q (expanded ${GC_DIR} = tp.WorkDir)", got, tp.WorkDir) + } + wantBeadsDir := filepath.Join(rigRoot, ".beads") + if got := tp.Env["BEADS_AT"]; got != wantBeadsDir { + t.Fatalf("BEADS_AT = %q, want %q (expanded ${BEADS_DIR})", got, wantBeadsDir) + } +} + +// TestResolveTemplateExpandsAgentEnvVarsInProviderEnv verifies that +// provider [env] entries (resolved.Env) also see agentEnv vars during +// expansion. Symmetry with cfgAgent.Env matters because anything a +// provider preset wants to compute from the city/rig layout (e.g. a +// trace-file path under ${GT_ROOT}) would have failed for the same +// gc-rch40w reason. Supervisor passthrough must also remain visible. +func TestResolveTemplateExpandsAgentEnvVarsInProviderEnv(t *testing.T) { + cityPath := t.TempDir() + writeTemplateResolveCityConfig(t, cityPath, "file") + t.Setenv("PATH", "/usr/bin") + t.Setenv("GT_ROOT", "") + + params := &agentBuildParams{ + cityName: "city", + cityPath: cityPath, + workspace: &config.Workspace{Provider: "test"}, + providers: map[string]config.ProviderSpec{ + "test": { + Command: "echo", + PromptMode: "none", + Env: map[string]string{ + "PROVIDER_ROOT": "${GT_ROOT}/provider-stuff", + "PROVIDER_PATH": "${PATH}", + }, + }, + }, + lookPath: func(string) (string, error) { return "/bin/echo", nil }, + fs: fsys.OSFS{}, + beaconTime: time.Unix(0, 0), + beadNames: make(map[string]string), + stderr: io.Discard, + } + + agent := &config.Agent{Name: "runner"} + tp, err := resolveTemplate(params, agent, agent.QualifiedName(), nil) + if err != nil { + t.Fatalf("resolveTemplate: %v", err) + } + + wantProviderRoot := cityPath + "/provider-stuff" + if got := tp.Env["PROVIDER_ROOT"]; got != wantProviderRoot { + t.Fatalf("PROVIDER_ROOT = %q, want %q (resolved.Env must see agentEnv GT_ROOT)", got, wantProviderRoot) + } + if got := tp.Env["PROVIDER_PATH"]; got != "/usr/bin" { + t.Fatalf("PROVIDER_PATH = %q, want %q (resolved.Env must see passthrough PATH)", got, "/usr/bin") + } +} diff --git a/cmd/gc/template_resolve_workdir_test.go b/cmd/gc/template_resolve_workdir_test.go index 70fe56cb8f..fe14f252fb 100644 --- a/cmd/gc/template_resolve_workdir_test.go +++ b/cmd/gc/template_resolve_workdir_test.go @@ -474,8 +474,10 @@ dolt.auto-start: false if got := tp.Env["BEADS_DOLT_SERVER_PORT"]; got != wantPort { t.Fatalf("BEADS_DOLT_SERVER_PORT = %q, want %q", got, wantPort) } - if got := tp.Env["GC_DOLT_HOST"]; got != "" { - t.Fatalf("GC_DOLT_HOST = %q, want empty for managed target", got) + // Loopback host is projected so bd routes via SQL instead of falling + // back to its CLI mode (which forks `dolt remote -v` on each call). + if got := tp.Env["GC_DOLT_HOST"]; got != "127.0.0.1" { + t.Fatalf("GC_DOLT_HOST = %q, want %q for managed target", got, "127.0.0.1") } // HOME is intentionally passed through to agents (PR #272: // HOME/USER/XDG env passthrough for macOS Keychain and config access). diff --git a/cmd/gc/testdata/doctor_check_names.golden b/cmd/gc/testdata/doctor_check_names.golden index 4951896c06..b02fdaead8 100644 --- a/cmd/gc/testdata/doctor_check_names.golden +++ b/cmd/gc/testdata/doctor_check_names.golden @@ -55,6 +55,7 @@ agent-sessions zombie-sessions orphan-sessions bd-split-store +bd-config-parse beads-store v2-routed-to-namespace run-target-routed-to-backfill @@ -62,8 +63,10 @@ work-option-metadata-migration backlog-depth order-tracking-retention session-model +session-stuck-creating dolt-server fork-rate +tmp-scratch-space dolt-noms-size dolt-journal-size dolt-config diff --git a/cmd/gc/testdata/gastown-convoy.txtar b/cmd/gc/testdata/gastown-convoy.txtar index c001f199f0..dccac76034 100644 --- a/cmd/gc/testdata/gastown-convoy.txtar +++ b/cmd/gc/testdata/gastown-convoy.txtar @@ -151,3 +151,85 @@ stderr 'missing subcommand' # Unknown subcommand ! exec gc convoy blorp stderr 'unknown subcommand "blorp"' + +# --- 13. Rig-scope resolution for `gc convoy create` (gc-nm4d2h) --- +# +# `gc convoy create` must resolve the target rig the same way `gc bd` does +# — explicit --rig flag, then cwd, then GC_RIG — instead of always landing +# in city scope. The city-scope fall-through warns unless --city-scope opts +# in, and --rig + --city-scope is rejected as contradictory. +# +# This file-backend harness collapses every scope onto the single city store +# (the dolt-only prefix split — lx- vs tk- — can't be reproduced here), so +# the resolution OUTCOME is asserted via the warning contract: a city +# fall-through warns, a resolved rig does not. Store-directory placement +# itself is covered by TestResolveConvoyCreateScope. A fresh city keeps these +# assertions independent of the lifecycle scenarios above. + +exec gc init $WORK/scope-city +cd $WORK/scope-city +mkdir $WORK/scope-city/myrig +exec gc rig add $WORK/scope-city/myrig --name myrig --prefix mp +stdout 'Rig added' + +# Seed a couple of city issues for the positional-issue paths. +exec bd create 'city child A' +stdout 'gc-1' +exec bd create 'city child B' +stdout 'gc-2' + +# 13a. No rig signal from city root -> city scope WITH a warning. +exec gc convoy create rs-city-default +stdout 'Created convoy' +stderr 'city scope' + +# 13b. --city-scope opt-in -> city scope, no warning. +exec gc convoy create rs-city-explicit --city-scope +stdout 'Created convoy' +! stderr 'warning' + +# 13c. --rig flag after the subcommand -> rig resolved, no city-scope warning. +exec gc convoy create rs-rigflag --rig myrig +stdout 'Created convoy' +! stderr 'warning' + +# 13d. --rig flag before the subcommand -> rig resolved, no warning. +exec gc --rig myrig convoy create rs-rigflag2 +stdout 'Created convoy' +! stderr 'warning' + +# 13e. --rig + --city-scope is contradictory -> error. +! exec gc convoy create rs-conflict --rig myrig --city-scope +stderr 'city-scope' +stderr 'rig' + +# 13f. Positional issue from the city store, no signal -> city scope, no warning. +exec gc convoy create rs-city-issue gc-1 +stdout 'Created convoy' +! stderr 'warning' + +# 13g. Cross-rig: --rig resolves to myrig but the tracked issue lives in the +# city store. The convoy co-locates with the issue so parent refs +# resolve, and a warning notes the override. +exec gc --rig myrig convoy create rs-xrig gc-2 +stdout 'Created convoy' +stderr 'tracked issues' + +# 13h. cwd inside the rig -> rig resolved, no warning. +cd $WORK/scope-city/myrig +exec gc convoy create rs-cwd +stdout 'Created convoy' +! stderr 'warning' + +# 13i. --city-scope overrides a rig cwd -> city scope, no warning. +exec gc convoy create rs-cwd-cityscope --city-scope +stdout 'Created convoy' +! stderr 'warning' + +# 13j. GC_RIG env from city root -> rig resolved, no warning. Set last; the +# env var persists for the remainder of the script. +cd $WORK/scope-city +env GC_RIG=myrig +exec gc convoy create rs-env +stdout 'Created convoy' +! stderr 'warning' diff --git a/cmd/gc/testenv_test.go b/cmd/gc/testenv_test.go index 380a03666b..c395a6dda9 100644 --- a/cmd/gc/testenv_test.go +++ b/cmd/gc/testenv_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/gastownhall/gascity/internal/config" + "github.com/gastownhall/gascity/internal/testutil" ) // gcEnvVars lists the GC_* identity and session-routing variables that @@ -307,7 +308,12 @@ func TestInstallTestProviderStubsUsesPIDPrefixedDir(t *testing.T) { func writeTestGitIdentity(homeDir string) error { gitConfig := filepath.Join(homeDir, ".gitconfig") - data := []byte("[user]\n\tname = gc-test\n\temail = gc-test@test.local\n[beads]\n\trole = maintainer\n") + // Build on the shared isolated config, appending this fixture's identity and + // beads.role. The trailing [user] block wins (git takes the last value), so + // the commit author is gc-test. + data := []byte(testutil.IsolatedGitConfigContents() + + "[user]\n\tname = gc-test\n\temail = gc-test@test.local\n" + + "[beads]\n\trole = maintainer\n") return os.WriteFile(gitConfig, data, 0o644) } @@ -325,10 +331,8 @@ func gcBeadsBdTestHomeEnv(t *testing.T) []string { if err := writeTestGitIdentity(homeDir); err != nil { t.Fatalf("write test git identity for beads-bd: %v", err) } - return []string{ - "HOME=" + homeDir, - "GIT_CONFIG_GLOBAL=" + filepath.Join(homeDir, ".gitconfig"), - } + return append([]string{"HOME=" + homeDir}, + testutil.IsolatedGitConfigEnv(filepath.Join(homeDir, ".gitconfig"))...) } func writeTestDoltIdentity(homeDir string) error { @@ -355,5 +359,28 @@ func configureTestDoltIdentityEnv(t *testing.T) { } t.Setenv("HOME", homeDir) t.Setenv("GIT_CONFIG_GLOBAL", filepath.Join(homeDir, ".gitconfig")) + t.Setenv("GIT_CONFIG_SYSTEM", os.DevNull) t.Setenv("DOLT_ROOT_PATH", homeDir) } + +// TestMainSeedsWritableIsolatedGlobalGitConfig guards the invariant TestMain +// wires up: GIT_CONFIG_GLOBAL points at a real, writable file (not /dev/null). +// See gc-sms19 / tk-9zgnf for why. +func TestMainSeedsWritableIsolatedGlobalGitConfig(t *testing.T) { + path := os.Getenv("GIT_CONFIG_GLOBAL") + if path == "" || path == os.DevNull { + t.Fatalf("GIT_CONFIG_GLOBAL = %q, want a writable temp file (gc-sms19)", path) + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + t.Fatalf("GIT_CONFIG_GLOBAL %q is not writable: %v", path, err) + } + _ = f.Close() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read GIT_CONFIG_GLOBAL %q: %v", path, err) + } + if !strings.Contains(string(data), "gpgsign = false") { + t.Fatalf("isolated global config %q missing gpgsign=false (tk-9zgnf):\n%s", path, data) + } +} diff --git a/cmd/gc/typedclass_edge_guard_test.go b/cmd/gc/typedclass_edge_guard_test.go index 7d6b232752..95c4e7d3ef 100644 --- a/cmd/gc/typedclass_edge_guard_test.go +++ b/cmd/gc/typedclass_edge_guard_test.go @@ -162,7 +162,10 @@ var typedClassCodecCensus = map[string]map[string]int{ // internals + internal/mail/beadmail's same-module compile dependency. Full sync // typing is a separate out-of-budget W-sync wave (see tickfeed-design §3 // W-unexport). - "cmd/gc/session_beads.go": 1, + // doctor_stuck_creating (gc-c1rpx) lists session beads to flag sessions stuck in + // creating; a doctor diagnostic fold-in pending a front-door migration. + "cmd/gc/doctor_stuck_creating.go": 1, + "cmd/gc/session_beads.go": 1, }, // ResolveSessionBeadByExactID( is now all-zero in the interior: the // worker-boundary resolve+construct site moved to ResolveSessionRecordByExactID diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 3cc1c7a35e..3c03ba5bc8 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1057,6 +1057,13 @@ Create a convoy and optionally link existing issues to it. Creates a convoy bead and tracks any provided issue IDs. Issues can also be added later with "gc convoy add". +The convoy is created in the same rig scope that "gc bd" would resolve: +an explicit --rig flag wins, then the current directory's rig, then the +GC_RIG environment variable, otherwise city scope. When tracked issues +are supplied they anchor the scope so the convoy and its issues share a +store. Pass --city-scope to force city scope (and silence the city-scope +fall-through warning). + ``` gc convoy create [issue-ids...] [flags] ``` @@ -1068,10 +1075,12 @@ gc convoy create sprint-42 gc convoy create sprint-42 issue-1 issue-2 issue-3 gc convoy create deploy --owner mayor --notify mayor --merge mr gc convoy create auth-rewrite --owned --target integration/auth-rewrite +gc convoy create infra-sweep --city-scope ``` | Flag | Type | Default | Description | |------|------|---------|-------------| +| `--city-scope` | bool | | force city scope (overrides --rig/GC_RIG/cwd detection and silences the city-scope warning) | | `--json` | bool | | emit JSONL result | | `--merge` | string | | merge strategy: direct, mr, local | | `--notify` | string | | notification target on completion | @@ -3715,9 +3724,12 @@ gc session attach End a conversation. Stops the runtime if active and closes the bead. Accepts a session ID (e.g., gc-42) or session alias (e.g., mayor). +When called with no argument, defaults to $GC_SESSION_ID — the +canonical way for an agent to self-close from inside its own +runtime. ``` -gc session close [flags] +gc session close [session-id-or-alias] [flags] ``` | Flag | Type | Default | Description | diff --git a/docs/reference/config.md b/docs/reference/config.md index a8c8a9a503..5e4c9748c1 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -24,6 +24,7 @@ City is the top-level configuration for a Gas City instance. | `agent` | []Agent | | | Agents lists all configured agents in this city. Pack-composed cities can compose agents through [imports.*] and ship without any [[agent]] block. | | `named_session` | []NamedSession | | | NamedSessions lists canonical alias-backed sessions built from reusable agent templates. | | `rigs` | []Rig | | | Rigs lists external projects registered in the city. | +| `default_merge_strategy` | string | | | DefaultMergeStrategy is the city-wide default merge strategy ("direct" or "pr") that refinery formulas resolve when a work bead does not carry an explicit `metadata.merge_strategy` and the originating rig does not set Rig.DefaultMergeStrategy. Empty defers to the formula's own default. Set this on a downstream city (e.g., loomington) to require PR review for every refinery merge without touching the upstream pack's formula defaults. | | `patches` | Patches | | | Patches holds targeted modifications applied after fragment merge. | | `beads` | BeadsConfig | | | Beads configures the bead store backend. | | `session` | SessionConfig | | | Session configures the session provider backend. | @@ -45,7 +46,7 @@ City is the top-level configuration for a Gas City instance. | `webhooks` | WebhookPolicyConfig | | | WebhookPolicy holds city-level webhook governance (the [webhooks] table, notably allow_public grants). Authored only in the root city.toml; never merged from packs or fragments so a pack cannot grant itself exposure. | | `github` | GitHubConfig | | | GitHub configures GitHub-facing repository monitors. | | `extmsg` | ExtMsgConfig | | | ExtMsg configures the external-messaging fabric (default routes for inbound conversations with no binding). | -| `agent_defaults` | AgentDefaults | | | AgentDefaults provides root city defaults for agents that don't override them (canonical TOML key: agent_defaults). Pack-local defaults use the same table shape in pack.toml. The runtime currently applies provider, default_sling_formula, and append_fragments; the attachment-list fields remain tombstones, and the other fields are parsed/composed but not yet inherited automatically. | +| `agent_defaults` | AgentDefaults | | | AgentDefaults provides root city defaults for agents that don't override them (canonical TOML key: agent_defaults). Pack-local defaults use the same table shape in pack.toml. The runtime currently applies provider, default_sling_formula, append_fragments, and env; the attachment-list fields remain tombstones, and the other fields are parsed/composed but not yet inherited automatically. | | `pricing` | []ModelPricing | | | Pricing holds per-model cost rate overrides keyed by (provider, model). City-level entries override pack-level entries which override the defaults shipped with the pricing package. See internal/pricing for the estimation seam introduced by issue #1255 (1d). | ## ACPSessionConfig @@ -148,6 +149,7 @@ AgentDefaults provides agent defaults declared via [agent_defaults] in city.toml | `allow_overlay` | []string | | | AllowOverlay is parsed and composed as a config-level allowlist for session overlays, but it is not yet inherited onto agents automatically at runtime. | | `allow_env_override` | []string | | | AllowEnvOverride is parsed and composed as a config-level allowlist for session env overrides, but it is not yet inherited onto agents automatically at runtime. Names must match ^[A-Z][A-Z0-9_]{0,127}$. | | `append_fragments` | []string | | | AppendFragments lists named template fragments to auto-append to .template.md prompts after rendering. Legacy .md.tmpl prompts are still supported during the transition; plain .md remains inert. V2 migration convenience — replaces global_fragments/inject_fragments for config-wide defaults. | +| `env` | map[string]string | | | Env sets baseline environment variables inherited by every agent at composition time. Per-agent [[agent.env]] keys win on collision, so agents can override defaults without restating the rest. Useful for city-wide wiring (PATH augmentation, locale, common feature flags) that would otherwise be duplicated across every [[agent]] entry. Control-dispatcher agents are skipped. | | `skills` | []string | | | Skills is a tombstone field retained for v0.15.1 backwards compatibility. Parsed and composed for migration visibility, but attachment-list fields are accepted but ignored by the active materializer. | | `mcp` | []string | | | MCP is a tombstone field retained for v0.15.1 backwards compatibility. Parsed and composed for migration visibility, but attachment-list fields are accepted but ignored by the active materializer. | @@ -346,6 +348,7 @@ DoctorConfig holds settings for the gc doctor surface. | `worktree_rig_error_size` | string | | `50GB` | WorktreeRigErrorSize is the per-rig error threshold. When any rig exceeds this, the worktree-disk-size check reports an error rather than a warning. Empty or unparseable falls back to the default (50 GB). | | `nested_worktree_prune` | boolean | | `false` | NestedWorktreePrune escalates the nested-worktree-prune check from warning to error severity when safely-prunable nested worktrees are present, so CI / scripted doctor runs fail until the operator runs `gc doctor --fix`. Actual removal still requires --fix; this flag does not auto-prune. Safety is enforced by mechanical checks (no uncommitted changes, no unpushed commits, no stashes) — never by role identity. | | `check` | []LocalDoctorCheck | | | Checks holds city-local inline doctor checks declared via [[doctor.check]] in city.toml. | +| `pack_script_timeout_secs` | integer | | `30` | PackScriptTimeoutSecs bounds how long any single pack doctor script (`gc doctor` Run or `gc doctor --fix`) is allowed to take before it is killed and reported as an error. Nil, zero, or negative values fall back to the package default (30s). A per-script bound is the load-bearing safety against a wedged pack tool (eg `bd --version` hung on a contended lock) blocking the whole doctor run indefinitely. Pointer so the unset value stays empty for TOML marshaling (mirrors DaemonConfig.MaxRestarts). | ## DoltConfig @@ -714,6 +717,7 @@ Rig defines an external project registered in the city. | `path` | string | | | Path is the absolute filesystem path to the rig's repository. | | `prefix` | string | | | Prefix overrides the auto-derived bead ID prefix for this rig. | | `default_branch` | string | | | DefaultBranch is the rig repository's mainline branch (e.g. "main", "master", "develop"). When set, routing formulas use this as the default merge target instead of probing origin/HEAD at sling time. Captured by `gc rig add` from the rig's git config; set manually for rigs whose mainline isn't reachable via origin/HEAD. | +| `default_merge_strategy` | string | | | DefaultMergeStrategy is the rig-scoped default merge strategy ("direct" or "pr") that refinery formulas resolve when a work bead does not carry an explicit `metadata.merge_strategy`. Empty defers to the city-level default (City.DefaultMergeStrategy), then to the formula's own default. Set this on a single rig to override the city-wide policy. | | `suspended` | boolean | | | Suspended is the deprecated pre-runtime-state suspension flag. Parsed for backwards compatibility and treated as an alias for SuspendedOnStart by [Rig.EffectiveSuspendedOnStart], so existing cities with `suspended = true` continue to start their rigs suspended after upgrade. Live suspend/resume commands no longer write this field. `gc doctor` flags it and offers `--fix` to rename to suspended_on_start. | | `suspended_on_start` | boolean | | | SuspendedOnStart is the rig's desired suspension state at city start. When true and no explicit entry exists for this rig in .gc/runtime/suspension-state.json, the rig is treated as suspended. Once the user has explicitly suspended or resumed the rig via `gc rig suspend/resume`, the runtime state wins. | | `formulas_dir` | string | | | FormulasDir is a rig-local formula directory — the highest-priority formula layer, above city pack formulas, the city formulas/ directory, and rig pack formulas. Overrides pack formulas for this rig by filename. Relative paths resolve against the city directory. | @@ -739,6 +743,7 @@ RigPatch modifies an existing rig identified by Name. | `path` | string | | | Path overrides the rig's filesystem path. | | `prefix` | string | | | Prefix overrides the bead ID prefix. | | `default_branch` | string | | | DefaultBranch overrides the rig's recorded mainline branch. | +| `default_merge_strategy` | string | | | DefaultMergeStrategy overrides the rig-scoped default merge strategy resolved by refinery formulas. See Rig.DefaultMergeStrategy. | | `suspended` | boolean | | | Suspended is the deprecated, pre-runtime-state suspension override. Parsed for backwards compatibility; `gc doctor` surfaces it as a warning and recommends the rename to SuspendedOnStart. No behavioral code path reads it. | | `suspended_on_start` | boolean | | | SuspendedOnStart overrides the rig's desired suspension state at city start. Mirrors Rig.SuspendedOnStart. | | `formula_vars` | map[string]string | | | FormulaVars adds or overrides rig-scoped formula var defaults. Additive merge: patch keys win over existing rig keys, unspecified keys are preserved. | diff --git a/docs/reference/schema/city-schema.json b/docs/reference/schema/city-schema.json index 9264b978e8..a9753491bf 100644 --- a/docs/reference/schema/city-schema.json +++ b/docs/reference/schema/city-schema.json @@ -402,6 +402,13 @@ "type": "array", "description": "AppendFragments lists named template fragments to auto-append to\n.template.md prompts after rendering. Legacy .md.tmpl prompts are\nstill supported during the transition; plain .md remains inert.\nV2 migration convenience — replaces global_fragments/inject_fragments\nfor config-wide defaults." }, + "env": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Env sets baseline environment variables inherited by every agent at\ncomposition time. Per-agent [[agent.env]] keys win on collision, so\nagents can override defaults without restating the rest. Useful for\ncity-wide wiring (PATH augmentation, locale, common feature flags)\nthat would otherwise be duplicated across every [[agent]] entry.\nControl-dispatcher agents are skipped." + }, "skills": { "items": { "type": "string" @@ -1123,6 +1130,10 @@ "type": "array", "description": "Rigs lists external projects registered in the city." }, + "default_merge_strategy": { + "type": "string", + "description": "DefaultMergeStrategy is the city-wide default merge strategy\n(\"direct\" or \"pr\") that refinery formulas resolve when a work bead\ndoes not carry an explicit `metadata.merge_strategy` and the\noriginating rig does not set Rig.DefaultMergeStrategy. Empty defers\nto the formula's own default. Set this on a downstream city (e.g.,\nloomington) to require PR review for every refinery merge without\ntouching the upstream pack's formula defaults." + }, "patches": { "$ref": "#/$defs/Patches", "description": "Patches holds targeted modifications applied after fragment merge." @@ -1215,7 +1226,7 @@ }, "agent_defaults": { "$ref": "#/$defs/AgentDefaults", - "description": "AgentDefaults provides root city defaults for agents that don't override\nthem (canonical TOML key: agent_defaults). Pack-local defaults use the\nsame table shape in pack.toml. The runtime currently applies provider,\ndefault_sling_formula, and append_fragments; the attachment-list fields\nremain tombstones, and the other fields are parsed/composed but not yet\ninherited automatically." + "description": "AgentDefaults provides root city defaults for agents that don't override\nthem (canonical TOML key: agent_defaults). Pack-local defaults use the\nsame table shape in pack.toml. The runtime currently applies provider,\ndefault_sling_formula, append_fragments, and env; the attachment-list\nfields remain tombstones, and the other fields are parsed/composed but\nnot yet inherited automatically." }, "pricing": { "items": { @@ -1399,6 +1410,11 @@ }, "type": "array", "description": "Checks holds city-local inline doctor checks declared via\n[[doctor.check]] in city.toml." + }, + "pack_script_timeout_secs": { + "type": "integer", + "description": "PackScriptTimeoutSecs bounds how long any single pack doctor\nscript (`gc doctor` Run or `gc doctor --fix`) is allowed to\ntake before it is killed and reported as an error. Nil, zero,\nor negative values fall back to the package default (30s). A\nper-script bound is the load-bearing safety against a wedged\npack tool (eg `bd --version` hung on a contended lock) blocking\nthe whole doctor run indefinitely. Pointer so the unset value\nstays empty for TOML marshaling (mirrors DaemonConfig.MaxRestarts).", + "default": 30 } }, "additionalProperties": false, @@ -2484,6 +2500,10 @@ "type": "string", "description": "DefaultBranch is the rig repository's mainline branch (e.g. \"main\",\n\"master\", \"develop\"). When set, routing formulas use this as the\ndefault merge target instead of probing origin/HEAD at sling time.\nCaptured by `gc rig add` from the rig's git config; set manually for\nrigs whose mainline isn't reachable via origin/HEAD." }, + "default_merge_strategy": { + "type": "string", + "description": "DefaultMergeStrategy is the rig-scoped default merge strategy\n(\"direct\" or \"pr\") that refinery formulas resolve when a work bead\ndoes not carry an explicit `metadata.merge_strategy`. Empty defers to\nthe city-level default (City.DefaultMergeStrategy), then to the\nformula's own default. Set this on a single rig to override the\ncity-wide policy." + }, "suspended": { "type": "boolean", "description": "Suspended is the deprecated pre-runtime-state suspension flag.\nParsed for backwards compatibility and treated as an alias for\nSuspendedOnStart by [Rig.EffectiveSuspendedOnStart], so existing\ncities with `suspended = true` continue to start their rigs\nsuspended after upgrade. Live suspend/resume commands no longer\nwrite this field. `gc doctor` flags it and offers `--fix` to\nrename to suspended_on_start." @@ -2584,6 +2604,10 @@ "type": "string", "description": "DefaultBranch overrides the rig's recorded mainline branch." }, + "default_merge_strategy": { + "type": "string", + "description": "DefaultMergeStrategy overrides the rig-scoped default merge strategy\nresolved by refinery formulas. See Rig.DefaultMergeStrategy." + }, "suspended": { "type": "boolean", "description": "Suspended is the deprecated, pre-runtime-state suspension override.\nParsed for backwards compatibility; `gc doctor` surfaces it as a\nwarning and recommends the rename to SuspendedOnStart. No behavioral\ncode path reads it." diff --git a/docs/reference/schema/city-schema.txt b/docs/reference/schema/city-schema.txt index 9264b978e8..a9753491bf 100644 --- a/docs/reference/schema/city-schema.txt +++ b/docs/reference/schema/city-schema.txt @@ -402,6 +402,13 @@ "type": "array", "description": "AppendFragments lists named template fragments to auto-append to\n.template.md prompts after rendering. Legacy .md.tmpl prompts are\nstill supported during the transition; plain .md remains inert.\nV2 migration convenience — replaces global_fragments/inject_fragments\nfor config-wide defaults." }, + "env": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Env sets baseline environment variables inherited by every agent at\ncomposition time. Per-agent [[agent.env]] keys win on collision, so\nagents can override defaults without restating the rest. Useful for\ncity-wide wiring (PATH augmentation, locale, common feature flags)\nthat would otherwise be duplicated across every [[agent]] entry.\nControl-dispatcher agents are skipped." + }, "skills": { "items": { "type": "string" @@ -1123,6 +1130,10 @@ "type": "array", "description": "Rigs lists external projects registered in the city." }, + "default_merge_strategy": { + "type": "string", + "description": "DefaultMergeStrategy is the city-wide default merge strategy\n(\"direct\" or \"pr\") that refinery formulas resolve when a work bead\ndoes not carry an explicit `metadata.merge_strategy` and the\noriginating rig does not set Rig.DefaultMergeStrategy. Empty defers\nto the formula's own default. Set this on a downstream city (e.g.,\nloomington) to require PR review for every refinery merge without\ntouching the upstream pack's formula defaults." + }, "patches": { "$ref": "#/$defs/Patches", "description": "Patches holds targeted modifications applied after fragment merge." @@ -1215,7 +1226,7 @@ }, "agent_defaults": { "$ref": "#/$defs/AgentDefaults", - "description": "AgentDefaults provides root city defaults for agents that don't override\nthem (canonical TOML key: agent_defaults). Pack-local defaults use the\nsame table shape in pack.toml. The runtime currently applies provider,\ndefault_sling_formula, and append_fragments; the attachment-list fields\nremain tombstones, and the other fields are parsed/composed but not yet\ninherited automatically." + "description": "AgentDefaults provides root city defaults for agents that don't override\nthem (canonical TOML key: agent_defaults). Pack-local defaults use the\nsame table shape in pack.toml. The runtime currently applies provider,\ndefault_sling_formula, append_fragments, and env; the attachment-list\nfields remain tombstones, and the other fields are parsed/composed but\nnot yet inherited automatically." }, "pricing": { "items": { @@ -1399,6 +1410,11 @@ }, "type": "array", "description": "Checks holds city-local inline doctor checks declared via\n[[doctor.check]] in city.toml." + }, + "pack_script_timeout_secs": { + "type": "integer", + "description": "PackScriptTimeoutSecs bounds how long any single pack doctor\nscript (`gc doctor` Run or `gc doctor --fix`) is allowed to\ntake before it is killed and reported as an error. Nil, zero,\nor negative values fall back to the package default (30s). A\nper-script bound is the load-bearing safety against a wedged\npack tool (eg `bd --version` hung on a contended lock) blocking\nthe whole doctor run indefinitely. Pointer so the unset value\nstays empty for TOML marshaling (mirrors DaemonConfig.MaxRestarts).", + "default": 30 } }, "additionalProperties": false, @@ -2484,6 +2500,10 @@ "type": "string", "description": "DefaultBranch is the rig repository's mainline branch (e.g. \"main\",\n\"master\", \"develop\"). When set, routing formulas use this as the\ndefault merge target instead of probing origin/HEAD at sling time.\nCaptured by `gc rig add` from the rig's git config; set manually for\nrigs whose mainline isn't reachable via origin/HEAD." }, + "default_merge_strategy": { + "type": "string", + "description": "DefaultMergeStrategy is the rig-scoped default merge strategy\n(\"direct\" or \"pr\") that refinery formulas resolve when a work bead\ndoes not carry an explicit `metadata.merge_strategy`. Empty defers to\nthe city-level default (City.DefaultMergeStrategy), then to the\nformula's own default. Set this on a single rig to override the\ncity-wide policy." + }, "suspended": { "type": "boolean", "description": "Suspended is the deprecated pre-runtime-state suspension flag.\nParsed for backwards compatibility and treated as an alias for\nSuspendedOnStart by [Rig.EffectiveSuspendedOnStart], so existing\ncities with `suspended = true` continue to start their rigs\nsuspended after upgrade. Live suspend/resume commands no longer\nwrite this field. `gc doctor` flags it and offers `--fix` to\nrename to suspended_on_start." @@ -2584,6 +2604,10 @@ "type": "string", "description": "DefaultBranch overrides the rig's recorded mainline branch." }, + "default_merge_strategy": { + "type": "string", + "description": "DefaultMergeStrategy overrides the rig-scoped default merge strategy\nresolved by refinery formulas. See Rig.DefaultMergeStrategy." + }, "suspended": { "type": "boolean", "description": "Suspended is the deprecated, pre-runtime-state suspension override.\nParsed for backwards compatibility; `gc doctor` surfaces it as a\nwarning and recommends the rename to SuspendedOnStart. No behavioral\ncode path reads it." diff --git a/docs/reference/schema/openapi.json b/docs/reference/schema/openapi.json index 0c1d093210..c5d5199200 100644 --- a/docs/reference/schema/openapi.json +++ b/docs/reference/schema/openapi.json @@ -619,6 +619,10 @@ "display_name": { "type": "string" }, + "input_tokens": { + "format": "int64", + "type": "integer" + }, "last_output": { "type": "string" }, @@ -6446,6 +6450,12 @@ "null" ] }, + "DefaultMergeStrategy": { + "type": [ + "string", + "null" + ] + }, "FormulaVars": { "additionalProperties": { "type": "string" @@ -6485,6 +6495,7 @@ "Path", "Prefix", "DefaultBranch", + "DefaultMergeStrategy", "Suspended", "SuspendedOnStart", "FormulaVars" @@ -6571,6 +6582,7 @@ "type": "string" }, "prefix": { + "description": "Effective bead-ID prefix. Always populated — explicit when configured, otherwise derived from the rig name.", "type": "string" }, "running_count": { @@ -6584,6 +6596,7 @@ "required": [ "name", "path", + "prefix", "suspended", "agent_count", "running_count" @@ -7460,6 +7473,10 @@ "id": { "type": "string" }, + "input_tokens": { + "format": "int64", + "type": "integer" + }, "kind": { "type": "string" }, @@ -40756,6 +40773,16 @@ "description": "Include last output preview.", "type": "boolean" } + }, + { + "description": "Response detail level. The default (empty, \"summary\", or any unrecognized value) returns only the cheap read-model fields (id, alias, title, state, rig, pool, agent_kind, reason, options, metadata) built from stored metadata with no live runtime probe; it skips per-session enrichment (live running probe, active-bead lookup, model/context transcript read) and leaves the live-observation fields running, active_bead, model, context_pct, last_output, attached, and last_active at their zero values. It takes precedence over peek. \"full\" opts into the enriched response.", + "explode": false, + "in": "query", + "name": "view", + "schema": { + "description": "Response detail level. The default (empty, \"summary\", or any unrecognized value) returns only the cheap read-model fields (id, alias, title, state, rig, pool, agent_kind, reason, options, metadata) built from stored metadata with no live runtime probe; it skips per-session enrichment (live running probe, active-bead lookup, model/context transcript read) and leaves the live-observation fields running, active_bead, model, context_pct, last_output, attached, and last_active at their zero values. It takes precedence over peek. \"full\" opts into the enriched response.", + "type": "string" + } } ], "responses": { diff --git a/docs/reference/schema/openapi.txt b/docs/reference/schema/openapi.txt index 0c1d093210..c5d5199200 100644 --- a/docs/reference/schema/openapi.txt +++ b/docs/reference/schema/openapi.txt @@ -619,6 +619,10 @@ "display_name": { "type": "string" }, + "input_tokens": { + "format": "int64", + "type": "integer" + }, "last_output": { "type": "string" }, @@ -6446,6 +6450,12 @@ "null" ] }, + "DefaultMergeStrategy": { + "type": [ + "string", + "null" + ] + }, "FormulaVars": { "additionalProperties": { "type": "string" @@ -6485,6 +6495,7 @@ "Path", "Prefix", "DefaultBranch", + "DefaultMergeStrategy", "Suspended", "SuspendedOnStart", "FormulaVars" @@ -6571,6 +6582,7 @@ "type": "string" }, "prefix": { + "description": "Effective bead-ID prefix. Always populated — explicit when configured, otherwise derived from the rig name.", "type": "string" }, "running_count": { @@ -6584,6 +6596,7 @@ "required": [ "name", "path", + "prefix", "suspended", "agent_count", "running_count" @@ -7460,6 +7473,10 @@ "id": { "type": "string" }, + "input_tokens": { + "format": "int64", + "type": "integer" + }, "kind": { "type": "string" }, @@ -40756,6 +40773,16 @@ "description": "Include last output preview.", "type": "boolean" } + }, + { + "description": "Response detail level. The default (empty, \"summary\", or any unrecognized value) returns only the cheap read-model fields (id, alias, title, state, rig, pool, agent_kind, reason, options, metadata) built from stored metadata with no live runtime probe; it skips per-session enrichment (live running probe, active-bead lookup, model/context transcript read) and leaves the live-observation fields running, active_bead, model, context_pct, last_output, attached, and last_active at their zero values. It takes precedence over peek. \"full\" opts into the enriched response.", + "explode": false, + "in": "query", + "name": "view", + "schema": { + "description": "Response detail level. The default (empty, \"summary\", or any unrecognized value) returns only the cheap read-model fields (id, alias, title, state, rig, pool, agent_kind, reason, options, metadata) built from stored metadata with no live runtime probe; it skips per-session enrichment (live running probe, active-bead lookup, model/context transcript read) and leaves the live-observation fields running, active_bead, model, context_pct, last_output, attached, and last_active at their zero values. It takes precedence over peek. \"full\" opts into the enriched response.", + "type": "string" + } } ], "responses": { diff --git a/docs/reference/schema/pack-schema.json b/docs/reference/schema/pack-schema.json index cf559bd280..f75ead4d34 100644 --- a/docs/reference/schema/pack-schema.json +++ b/docs/reference/schema/pack-schema.json @@ -341,6 +341,13 @@ "type": "array", "description": "AppendFragments lists named template fragments to auto-append to\n.template.md prompts after rendering. Legacy .md.tmpl prompts are\nstill supported during the transition; plain .md remains inert.\nV2 migration convenience — replaces global_fragments/inject_fragments\nfor config-wide defaults." }, + "env": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Env sets baseline environment variables inherited by every agent at\ncomposition time. Per-agent [[agent.env]] keys win on collision, so\nagents can override defaults without restating the rest. Useful for\ncity-wide wiring (PATH augmentation, locale, common feature flags)\nthat would otherwise be duplicated across every [[agent]] entry.\nControl-dispatcher agents are skipped." + }, "skills": { "items": { "type": "string" diff --git a/docs/reference/schema/pack-schema.txt b/docs/reference/schema/pack-schema.txt index cf559bd280..f75ead4d34 100644 --- a/docs/reference/schema/pack-schema.txt +++ b/docs/reference/schema/pack-schema.txt @@ -341,6 +341,13 @@ "type": "array", "description": "AppendFragments lists named template fragments to auto-append to\n.template.md prompts after rendering. Legacy .md.tmpl prompts are\nstill supported during the transition; plain .md remains inert.\nV2 migration convenience — replaces global_fragments/inject_fragments\nfor config-wide defaults." }, + "env": { + "additionalProperties": { + "type": "string" + }, + "type": "object", + "description": "Env sets baseline environment variables inherited by every agent at\ncomposition time. Per-agent [[agent.env]] keys win on collision, so\nagents can override defaults without restating the rest. Useful for\ncity-wide wiring (PATH augmentation, locale, common feature flags)\nthat would otherwise be duplicated across every [[agent]] entry.\nControl-dispatcher agents are skipped." + }, "skills": { "items": { "type": "string" diff --git a/engdocs/architecture/dispatch.md b/engdocs/architecture/dispatch.md index 50dac19d32..37a7b53b03 100644 --- a/engdocs/architecture/dispatch.md +++ b/engdocs/architecture/dispatch.md @@ -177,6 +177,16 @@ ready work with `assignee=` and no generic route metadata, so the reconciler does not also treat the handoff as generic pool demand. +Because only the unassigned shape is pool demand, a bare +`gc sling ` of an existing **open** bead also clears any +stale assignee (a handback target, a prior hand-walked recovery, or the +pool template itself) so the re-poured bead actually re-enters the +routed queue. Without that normalization a re-pour only rewrites +`gc.routed_to` to the same value — invisible to scale_check and to every +work-query tier, stalling the chain silently (gc-q40pm). In-progress +beads are exempt: that status marks a live claim, and only `--reassign` +or the controller's orphan release may strip one. + The shared predicate is the agreement substrate. Failure envelopes intentionally differ: the worker path suppresses `bd ready` stderr and returns `[]` so a session exits cleanly, while the count form propagates diff --git a/engdocs/contributors/dolt-maintenance.md b/engdocs/contributors/dolt-maintenance.md index 3e69f1d164..50544310ee 100644 --- a/engdocs/contributors/dolt-maintenance.md +++ b/engdocs/contributors/dolt-maintenance.md @@ -8,18 +8,28 @@ The loop runs **inside the supervisor process** when opted in via ## Current wiring status -**Observe-only mode (as of this release).** The SQL connection to Dolt -(`OpenDoltOps`) and the backup runner (`OpenDoltBackup`) are not yet -wired in the production controller. When `[maintenance.dolt] -enabled=true`, the loop runs on schedule, emits events, and records -history — but steps 2 (snapshot) and 3 (compaction) below are no-ops. -The supervisor logs `store-maintenance: enabled in observe-only mode -(snapshot and DOLT_GC not yet wired)` at startup. - -Production wiring of the snapshot and GC factories will land in a -follow-up. Until then, `gc maintenance status` and the events are -meaningful for scheduling/alert testing, but a `gc.store.maintenance.done` -event does **not** imply compaction occurred. +**Compaction wired; snapshot not (as of this release).** The SQL +connection (`OpenDoltOps`) is wired in the production controller: when +`[maintenance.dolt] enabled=true`, each cycle runs `CALL DOLT_GC()` +against the managed Dolt server, so a `gc.store.maintenance.done` event +**does** imply compaction occurred. Because this fork runs a +multi-database managed server, the GC iterates every managed user +database (the bd ledgers) and compacts each in turn — `CALL DOLT_GC()` +only compacts the session's selected database, so a single call would +leave every database but the default at its peak on-disk size. The +factory resolves the Dolt port per cycle, so the loop recovers +automatically if Dolt had not finished publishing its port at startup. + +The pre-GC snapshot (`OpenDoltBackup`, step 2 below) is **not** wired. +The snapshot layout in `internal/supervisor` assumes a single Dolt +store, which does not fit this fork's per-database directory layout +(`/.beads/dolt/`); multi-database snapshotting is tracked +separately. The supervisor logs `store-maintenance: DOLT_GC enabled; +pre-GC snapshot not wired (multi-database snapshot tracked in gc-thnww)` +at startup. `CALL DOLT_GC()` is online and safe to run without a +pre-GC snapshot; the post-gc smoke test guards readability, and +operators can still take manual backups (see +`docs/troubleshooting/dolt-bloat-recovery.md`). ## What this runs @@ -30,10 +40,19 @@ For each scheduled cycle the supervisor: 2. **Snapshot** — `dolt backup sync` to `/.beads/dolt-backups/current/`, then atomically rotates `current/` to `success//` (see *Snapshot layout* below). -3. **Compaction** — `CALL DOLT_GC()` against the managed Dolt server, - bounded by `gc_timeout` (default `10m`). -4. **Smoke test** — `SELECT COUNT(*) FROM issues` (5 s timeout) to - confirm the store is readable. +3. **Compaction** — `CALL DOLT_GC()` against **each** managed user + database on the Dolt server (the loop selects each in turn), bounded + collectively by `gc_timeout` (default `10m`). The compaction fails + fast, naming the database, if any one cannot be GC'd; databases + compacted before the failure keep their reclaimed space. Each GC runs + on a freshly-opened connection: an online `CALL DOLT_GC()` invalidates + every connection that was open across it (the next statement on one + fails with Error 1105 "…please reconnect."), so the maintenance pool + retains no idle connection and reconnects between databases. Reusing a + GC-poisoned connection is what previously aborted the multi-database + run at the second database. +4. **Smoke test** — `SELECT COUNT(*) FROM issues` per managed database, + summed (5 s timeout), to confirm the stores are readable after GC. 5. **Prune** — keep the 3 newest successful snapshots and the most recent failed snapshot; older entries are removed. Prune errors are logged but do not regress a successful run. diff --git a/engdocs/design/packv2/doc-pack-v2.md b/engdocs/design/packv2/doc-pack-v2.md index 50fc19fb7c..13171d2a58 100644 --- a/engdocs/design/packv2/doc-pack-v2.md +++ b/engdocs/design/packv2/doc-pack-v2.md @@ -357,6 +357,49 @@ When multiple packs are imported, formulas layer by priority (lowest to highest) The importing pack always wins over its imports. +#### Order scope + +Orders default to rig scope: a pack imported by N rigs contributes +each of its `orders/*.toml` once per importing rig. For maintenance +orders that only make sense city-wide (e.g. ones that target a +city-only pool), pack authors pin the order to a single city-wide +registration with `scope`: + +```toml +# orders/digest-generate.toml +[order] +formula = "mol-digest-generate" +trigger = "cooldown" +interval = "24h" +pool = "dog" +scope = "city" # registered exactly once, however many rigs import the pack +``` + +- `scope = "city"` — instantiated exactly once during pack expansion, + regardless of how many rigs import the pack. A rig-imported copy is + promoted to one city-wide registration (deduplicated by name across + rigs; a city-local order of the same name wins over the promotion). +- `scope = "rig"` — the explicit spelling of the default: the order + registers once per importing rig, stamped with that rig's name. +- omitted — same as `scope = "rig"`. + +The field mirrors the Pack V2 agent and named-session `scope` field. +Unlike agents, an order's omitted `scope` does not inherit from +`agent_defaults` — there is no order-defaults analogue. + +**Convention for new pack authors.** Declare `scope` explicitly on +every order. The bundled packs do — `TestBundledOrdersDeclareScope` +in `internal/builtinpacks/registry_test.go` keeps them honest. Pool- +bound orders almost always want `scope = "city"` (the pool typically +lives at one scope). Exec-based and event-triggered maintenance +orders that touch shared city infrastructure (Dolt, the bead store, +the city-wide event stream, cross-rig branches) also want +`scope = "city"` — per-rig copies are harmful at best, duplicate work +at worst. Only leave `scope` omitted if once-per-importing-rig +genuinely is the right behavior for that order, and pair that +decision with a one-line `# scope:` comment so a future reader sees +the intent rather than oversight. + ### Pack identity and qualified names After composition, every agent, formula, and prompt retains its pack provenance. diff --git a/examples/bd/dolt/assets/scripts/mol-dog-doctor.sh b/examples/bd/dolt/assets/scripts/mol-dog-doctor.sh index 70aa53b56c..4ad19ecf98 100755 --- a/examples/bd/dolt/assets/scripts/mol-dog-doctor.sh +++ b/examples/bd/dolt/assets/scripts/mol-dog-doctor.sh @@ -87,7 +87,7 @@ newest_backup_mtime_for_db() { newest_mtime="$backup_mtime" fi fi - done < <(find "$BACKUP_ARTIFACT_DIR" -type f -print0 2>/dev/null) + done < <(find -L "$BACKUP_ARTIFACT_DIR" -type f -print0 2>/dev/null) printf '%s\n' "$newest_mtime" } diff --git a/examples/bd/dolt/assets/scripts/runtime.sh b/examples/bd/dolt/assets/scripts/runtime.sh index 11bc1e233a..59aa67e592 100644 --- a/examples/bd/dolt/assets/scripts/runtime.sh +++ b/examples/bd/dolt/assets/scripts/runtime.sh @@ -128,22 +128,52 @@ managed_runtime_listener_pid() ( ;; esac - if ! command -v lsof >/dev/null 2>&1; then + _emit_first_running_holder() { + while IFS= read -r holder_pid; do + case "$holder_pid" in + ''|*[!0-9]*) + continue + ;; + esac + if pid_is_running "$holder_pid"; then + printf '%s\n' "$holder_pid" + return 0 + fi + done + } + + # ss (iproute2) is preferred on Linux: it reads via netlink and correctly + # reports MPTCP listening sockets, which lsof 4.99.6 misclassifies as + # protocol "MPTCPv6" and thus excludes from `-iTCP:PORT` results. Modern + # Go's net package on Linux kernels with MPTCP enabled by default + # (Ubuntu 24.04+, recent Debian/Fedora) creates these sockets, so an + # lsof-only probe fails to discover the listener and the managed runtime + # gets misreported as zombie. Extraction is done in shell rather than + # piping through sed/awk, both of which fully-buffer when stdout is a + # pipe and would delay holder-pid emission until ss exits — by which + # time test fakes that synthesize a transient process have already gone. + if command -v ss >/dev/null 2>&1; then + ss -Hltnp "sport = :$port" 2>/dev/null \ + | while IFS= read -r line; do + case "$line" in + *pid=*) + rest=${line#*pid=} + pid_candidate=${rest%%[!0-9]*} + [ -n "$pid_candidate" ] && printf '%s\n' "$pid_candidate" + ;; + esac + done \ + | _emit_first_running_holder return 0 fi - lsof -nP -t -iTCP:"$port" -sTCP:LISTEN 2>/dev/null \ - | while IFS= read -r holder_pid; do - case "$holder_pid" in - ''|*[!0-9]*) - continue - ;; - esac - if pid_is_running "$holder_pid"; then - printf '%s\n' "$holder_pid" - break - fi - done + # macOS lacks ss; lsof is correct there because Go on Darwin does not + # create MPTCP sockets, so the lsof MPTCP-blind-spot does not apply. + if command -v lsof >/dev/null 2>&1; then + lsof -nP -t -iTCP:"$port" -sTCP:LISTEN 2>/dev/null \ + | _emit_first_running_holder + return 0 + fi ) managed_runtime_tcp_reachable() ( diff --git a/examples/bd/dolt/commands/sync/run.sh b/examples/bd/dolt/commands/sync/run.sh index d51ada1c0c..0af89827c8 100755 --- a/examples/bd/dolt/commands/sync/run.sh +++ b/examples/bd/dolt/commands/sync/run.sh @@ -578,6 +578,47 @@ sync_database_cli() { return 1 } +# Concurrency guard: a second `gc dolt sync` must not run while one is already +# in flight. The dolt-remotes-patrol order fires on a 15m cooldown, so a slow or +# hung push lets each tick stack another concurrent DOLT_PUSH — the 2026-06-05 +# incident stacked 16 pushes (load 62) via a git cat-file enumeration storm. +# flock gives a crash-safe mutex: the kernel drops the lock when the holding +# process exits (even on SIGKILL), so unlike a PID/status file it never goes +# stale. The lock lives beside the other dolt runtime artifacts (dolt.pid, +# dolt.log) under DOLT_STATE_DIR. --dry-run performs no push, so it neither +# needs the lock nor should be blocked by an in-flight sync. +sync_lock_file="$DOLT_STATE_DIR/dolt-sync.lock" + +# acquire_sync_lock — take the non-blocking sync lock on fd 9, held until this +# process exits. Returns 0 when we hold the lock (or the guard is unavailable +# and we deliberately proceed unguarded); returns 1 only when another sync +# already holds it, in which case the caller skips this run. +acquire_sync_lock() { + # flock is util-linux; it is absent on stock macOS. Degrade loudly rather than + # failing the sync: the patrol that motivates this guard runs on Linux, and a + # single-operator dev box has no 15m patrol to stack pushes. + if ! command -v flock >/dev/null 2>&1; then + echo "gc dolt sync: WARN: flock not found; running without a concurrency guard" >&2 + return 0 + fi + # A bare `exec 9>BAD` aborts a non-interactive dash outright — even inside an + # `if` — so prove the lock file is creatable/writable with non-fatal commands + # BEFORE the exec. If we cannot, warn and proceed unguarded rather than dying. + if ! { mkdir -p "$DOLT_STATE_DIR" 2>/dev/null && : >>"$sync_lock_file" 2>/dev/null; }; then + echo "gc dolt sync: WARN: cannot create lock file $sync_lock_file; running without a concurrency guard" >&2 + return 0 + fi + exec 9>>"$sync_lock_file" + flock -n 9 +} + +if [ "$dry_run" != true ]; then + if ! acquire_sync_lock; then + echo "gc dolt sync: another sync is already in flight; skipping this run" + exit 0 + fi +fi + # Optional GC phase: purge closed ephemerals while server is still up. if [ "$do_gc" = true ] && [ -d "$data_dir" ]; then for d in "$data_dir"/*/; do diff --git a/examples/bd/dolt/dog_exec_scripts_test.go b/examples/bd/dolt/dog_exec_scripts_test.go index 85241b9a18..a7e130aa97 100644 --- a/examples/bd/dolt/dog_exec_scripts_test.go +++ b/examples/bd/dolt/dog_exec_scripts_test.go @@ -4396,6 +4396,24 @@ func TestBackupScriptCountsFailedRemoteAutoConfiguration(t *testing.T) { } } +// backupFreshnessStaleEnv is the staleness threshold for the doctor +// backup-freshness tests that assert a FRESH fixture is NOT reported stale. +// +// mol-dog-doctor.sh compares `date +%s` at script time against the artifact's +// mtime, so the threshold must exceed the wall-clock delay between the fixture +// being written and the script sampling the clock. The previous value of 1s +// left no margin: under parallel suite load that delay (script startup, dolt +// connect, SHOW DATABASES — one query alone was observed at 673ms) routinely +// reached 2s+, and a just-written backup was reported stale as "0h old", +// failing the very assertion that it must not be. The tests passed in +// isolation and failed only under load, which made it read as a mystery flake. +// +// 1h keeps both sides unambiguous: a fresh fixture stays fresh for an hour of +// scheduling delay, while the deliberately-aged fixtures (2h) stay stale. The +// "missing" assertions are unaffected — those come from a separate branch that +// fires when no artifact exists at all, independent of this threshold. +const backupFreshnessStaleEnv = "GC_DOCTOR_BACKUP_STALE_S=3600" + func TestDoctorScriptChecksBackupArtifactFreshnessPerDatabase(t *testing.T) { cityPath := t.TempDir() dataDir := filepath.Join(cityPath, "dolt-data") @@ -4447,7 +4465,7 @@ esac exit 0 `) - out := runDogScript(t, "mol-dog-doctor.sh", binDir, cityPath, dataDir, "GC_DOCTOR_BACKUP_STALE_S=1") + out := runDogScript(t, "mol-dog-doctor.sh", binDir, cityPath, dataDir, backupFreshnessStaleEnv) if !strings.Contains(out, "server: ok") { t.Fatalf("unexpected doctor output:\n%s", out) } @@ -4659,7 +4677,7 @@ esac exit 0 `) - out := runDogScript(t, "mol-dog-doctor.sh", binDir, cityPath, dataDir, "GC_DOCTOR_BACKUP_STALE_S=1") + out := runDogScript(t, "mol-dog-doctor.sh", binDir, cityPath, dataDir, backupFreshnessStaleEnv) if !strings.Contains(out, "server: ok") { t.Fatalf("unexpected doctor output:\n%s", out) } diff --git a/examples/bd/dolt/health_test.go b/examples/bd/dolt/health_test.go index d4443b8f8c..404624ea31 100644 --- a/examples/bd/dolt/health_test.go +++ b/examples/bd/dolt/health_test.go @@ -598,13 +598,15 @@ func TestRuntimeScriptPortPrecedenceToleratesInconclusiveLsof(t *testing.T) { tests := []struct { name string lsofBody string + ssBody string ncBody func(port string) string wantManaged bool wantExit78 bool }{ { - name: "inconclusive lsof accepts reachable port", + name: "inconclusive listener probe accepts reachable port", lsofBody: "#!/bin/sh\nexit 0\n", + ssBody: "#!/bin/sh\nexit 0\n", ncBody: func(port string) string { return `#!/bin/sh host="$2" @@ -618,8 +620,9 @@ exit 1 wantManaged: true, }, { - name: "mismatched lsof pid still rejects port", + name: "mismatched listener pid still rejects port", lsofBody: "#!/bin/sh\necho $$\nsleep 5\n", + ssBody: "#!/bin/sh\nprintf 'pid=%s\\n' \"$$\"\nsleep 5\n", ncBody: func(_ string) string { return `#!/bin/sh exit 0 @@ -628,8 +631,9 @@ exit 0 wantExit78: true, }, { - name: "inconclusive lsof with unreachable port still rejects port", + name: "inconclusive listener probe with unreachable port still rejects port", lsofBody: "#!/bin/sh\nexit 0\n", + ssBody: "#!/bin/sh\nexit 0\n", ncBody: func(_ string) string { return `#!/bin/sh exit 1 @@ -659,6 +663,7 @@ exit 1 writeManagedRuntimeStateForScript(t, cityPath, port) writeExecutable(t, filepath.Join(fakeBin, "lsof"), tt.lsofBody) + writeExecutable(t, filepath.Join(fakeBin, "ss"), tt.ssBody) writeExecutable(t, filepath.Join(fakeBin, "nc"), tt.ncBody(managedPort)) cmd := exec.Command("sh", "-c", `. "$GC_PACK_DIR/assets/scripts/runtime.sh"; printf '%s\n' "$GC_DOLT_PORT"`) @@ -705,11 +710,13 @@ func TestRuntimeScriptPortPrecedenceAcceptsPsConfirmedPid(t *testing.T) { tests := []struct { name string lsofBody string + ssBody string ncBody func(port string) string }{ { name: "listener pid match via ps fallback", lsofBody: "#!/bin/sh\necho 424242\n", + ssBody: "#!/bin/sh\necho 'pid=424242'\n", ncBody: func(_ string) string { return `#!/bin/sh exit 1 @@ -717,8 +724,9 @@ exit 1 }, }, { - name: "reachable port via ps fallback when lsof is inconclusive", + name: "reachable port via ps fallback when listener probe is inconclusive", lsofBody: "#!/bin/sh\nexit 0\n", + ssBody: "#!/bin/sh\nexit 0\n", ncBody: func(port string) string { return `#!/bin/sh host="$2" @@ -748,6 +756,7 @@ exit 1 writeManagedRuntimeStateForScriptWithPID(t, cityPath, port, 424242) writeExecutable(t, filepath.Join(fakeBin, "lsof"), tt.lsofBody) + writeExecutable(t, filepath.Join(fakeBin, "ss"), tt.ssBody) writeExecutable(t, filepath.Join(fakeBin, "nc"), tt.ncBody(managedPort)) writeExecutable(t, filepath.Join(fakeBin, "ps"), `#!/bin/sh if [ "$1" = "-p" ] && [ "$2" = "424242" ]; then @@ -832,6 +841,9 @@ func TestHealthScriptReportsRunningWhenLsofIsInconclusive(t *testing.T) { writeExecutable(t, filepath.Join(fakeBin, "lsof"), `#!/bin/sh exit 0 +`) + writeExecutable(t, filepath.Join(fakeBin, "ss"), `#!/bin/sh +exit 0 `) writeExecutable(t, filepath.Join(fakeBin, "nc"), `#!/bin/sh host="$2" @@ -890,6 +902,9 @@ func TestHealthScriptPortableTimestampFallbacksRemainNumeric(t *testing.T) { writeExecutable(t, filepath.Join(fakeBin, "lsof"), `#!/bin/sh exit 0 +`) + writeExecutable(t, filepath.Join(fakeBin, "ss"), `#!/bin/sh +exit 0 `) writeExecutable(t, filepath.Join(fakeBin, "nc"), `#!/bin/sh host="$2" @@ -1413,6 +1428,21 @@ for arg in "$@"; do esac done exit 1 +`, mainPort, mainPID, rigPort, rigPID)) + + // Fake ss: maps "sport = :PORT" filter args to ss-formatted output + // so the listener PID extractor pulls out the matching PID. Mirrors + // the lsof fake — ss is preferred on Linux because Go's MPTCP + // listening sockets are invisible to lsof. + writeExecutable(t, filepath.Join(fakeBin, "ss"), + fmt.Sprintf(`#!/bin/sh +for arg in "$@"; do + case "$arg" in + "sport = :%s") printf 'pid=%s\n'; exit 0 ;; + "sport = :%s") printf 'pid=%s\n'; exit 0 ;; + esac +done +exit 0 `, mainPort, mainPID, rigPort, rigPID)) // Fake ps: handles pid_is_running (`-p -o pid=`) and the zombie @@ -1557,6 +1587,19 @@ for arg in "$@"; do esac done exit 1 +`, mainPort, mainPID)) + + // Fake ss: maps the city port to mainPID so server_pid resolves on + // Linux test hosts, where ss-first listener detection runs before + // lsof (Go's MPTCP listening sockets are invisible to lsof). + writeExecutable(t, filepath.Join(fakeBin, "ss"), + fmt.Sprintf(`#!/bin/sh +for arg in "$@"; do + case "$arg" in + "sport = :%s") printf 'pid=%s\n'; exit 0 ;; + esac +done +exit 0 `, mainPort, mainPID)) // Fake ps: the bounded scan calls `ps -eo pid=,stat=,args=`. Emit the @@ -1666,6 +1709,18 @@ for arg in "$@"; do esac done exit 1 +`, mainPort, mainPID)) + // Fake ss: maps the city port to mainPID so server_pid resolves on + // Linux test hosts, where ss-first listener detection runs before + // lsof (Go's MPTCP listening sockets are invisible to lsof). + writeExecutable(t, filepath.Join(fakeBin, "ss"), + fmt.Sprintf(`#!/bin/sh +for arg in "$@"; do + case "$arg" in + "sport = :%s") printf 'pid=%s\n'; exit 0 ;; + esac +done +exit 0 `, mainPort, mainPID)) // Bounded `ps -eo` pass: the suspect PID carries `--config ` but // its sibling dolt.pid records a different PID, so the foreign-managed @@ -1767,6 +1822,22 @@ for arg in "$@"; do esac done exit 1 +`, mainPort, mainPID)) + + // Fake ss: runtime.sh prefers ss for listener detection on Linux (the + // kept MPTCP-correct ss-first rework); without an ss fake the real ss + // answers, finds no listener on the fixture port, and the city's own + // server PID falls through to the zombie set (zombie_count 2, want 1). + // Mirror the lsof fake — same pattern as the sibling foreign-managed + // zombie-scan tests (gc-9n4v5n). + writeExecutable(t, filepath.Join(fakeBin, "ss"), + fmt.Sprintf(`#!/bin/sh +for arg in "$@"; do + case "$arg" in + "sport = :%s") printf 'pid=%s\n'; exit 0 ;; + esac +done +exit 0 `, mainPort, mainPID)) // Fake ps: the bounded scan calls `ps -eo pid=,stat=,args=`. Emit the @@ -1864,7 +1935,13 @@ func TestHealthScriptZombieScanIsBoundedFork(t *testing.T) { // gc fails -> metadata_files falls back to find (no rigs here). writeExecutable(t, filepath.Join(fakeBin, "gc"), "#!/bin/sh\nexit 1\n") - // lsof maps the city port to the server PID so server_pid resolves. + // ss maps the city port to the server PID so server_pid resolves on + // Linux test hosts, where ss-first listener detection runs before + // lsof (Go's MPTCP listening sockets are invisible to lsof). + writeExecutable(t, filepath.Join(fakeBin, "ss"), + fmt.Sprintf("#!/bin/sh\nfor a in \"$@\"; do case \"$a\" in \"sport = :%s\") printf 'pid=%s\\n'; exit 0 ;; esac; done\nexit 0\n", mainPort, serverPID)) + // lsof maps the city port to the server PID so server_pid resolves on + // macOS, where the ss-first probe is skipped (ss is unavailable). writeExecutable(t, filepath.Join(fakeBin, "lsof"), fmt.Sprintf("#!/bin/sh\nfor a in \"$@\"; do case \"$a\" in -iTCP:%s) echo %s; exit 0 ;; esac; done\nexit 1\n", mainPort, serverPID)) writeExecutable(t, filepath.Join(fakeBin, "nc"), "#!/bin/sh\nexit 1\n") diff --git a/examples/bd/dolt/orders/dolt-health.toml b/examples/bd/dolt/orders/dolt-health.toml index 2e076599a0..58c11ffec6 100644 --- a/examples/bd/dolt/orders/dolt-health.toml +++ b/examples/bd/dolt/orders/dolt-health.toml @@ -3,3 +3,4 @@ description = "Check dolt server health without restarting it" trigger = "cooldown" interval = "30s" exec = "gc dolt health --json | gc dolt health-check" +scope = "city" diff --git a/examples/bd/dolt/orders/dolt-remotes-patrol.toml b/examples/bd/dolt/orders/dolt-remotes-patrol.toml index 658489eb4b..fd44043d55 100644 --- a/examples/bd/dolt/orders/dolt-remotes-patrol.toml +++ b/examples/bd/dolt/orders/dolt-remotes-patrol.toml @@ -3,3 +3,4 @@ description = "Push dolt databases to configured remotes" trigger = "cooldown" interval = "15m" exec = "gc dolt sync" +scope = "city" diff --git a/examples/bd/dolt/orders/mol-dog-backup.toml b/examples/bd/dolt/orders/mol-dog-backup.toml index c4aa070341..8af6463130 100644 --- a/examples/bd/dolt/orders/mol-dog-backup.toml +++ b/examples/bd/dolt/orders/mol-dog-backup.toml @@ -8,3 +8,4 @@ exec = "$PACK_DIR/assets/scripts/mol-dog-backup.sh" trigger = "cooldown" interval = "6h" timeout = "1800s" +scope = "city" diff --git a/examples/bd/dolt/orders/mol-dog-compactor.toml b/examples/bd/dolt/orders/mol-dog-compactor.toml index 0c28effc70..f253133b26 100644 --- a/examples/bd/dolt/orders/mol-dog-compactor.toml +++ b/examples/bd/dolt/orders/mol-dog-compactor.toml @@ -6,3 +6,4 @@ trigger = "cooldown" interval = "2h" exec = "gc dolt compact" timeout = "24h" +scope = "city" diff --git a/examples/bd/dolt/orders/mol-dog-doctor.toml b/examples/bd/dolt/orders/mol-dog-doctor.toml index 19e95439df..17e4a5e39d 100644 --- a/examples/bd/dolt/orders/mol-dog-doctor.toml +++ b/examples/bd/dolt/orders/mol-dog-doctor.toml @@ -6,3 +6,4 @@ description = "Probe Dolt server health and report status" exec = "$PACK_DIR/assets/scripts/mol-dog-doctor.sh" trigger = "cooldown" interval = "5m" +scope = "city" diff --git a/examples/bd/dolt/orders/mol-dog-phantom-db.toml b/examples/bd/dolt/orders/mol-dog-phantom-db.toml index 7923a7f542..0563ecaa63 100644 --- a/examples/bd/dolt/orders/mol-dog-phantom-db.toml +++ b/examples/bd/dolt/orders/mol-dog-phantom-db.toml @@ -7,3 +7,4 @@ description = "Detect phantom database resurrection after cleanup" exec = "$PACK_DIR/assets/scripts/mol-dog-phantom-db.sh" trigger = "cooldown" interval = "1h" +scope = "city" diff --git a/examples/bd/dolt/orders/mol-dog-stale-db.toml b/examples/bd/dolt/orders/mol-dog-stale-db.toml index 6cf5cdde43..eaf39f1827 100644 --- a/examples/bd/dolt/orders/mol-dog-stale-db.toml +++ b/examples/bd/dolt/orders/mol-dog-stale-db.toml @@ -7,3 +7,4 @@ formula = "mol-dog-stale-db" trigger = "cron" schedule = "0 */4 * * *" pool = "dog" +scope = "city" diff --git a/examples/bd/dolt/sync_concurrency_test.go b/examples/bd/dolt/sync_concurrency_test.go new file mode 100644 index 0000000000..90ae395f8c --- /dev/null +++ b/examples/bd/dolt/sync_concurrency_test.go @@ -0,0 +1,334 @@ +package dolt_test + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// These tests cover the concurrency guard in commands/sync/run.sh: a second +// `gc dolt sync` must not run while one is already in flight (the +// dolt-remotes-patrol order fires every 15m and a slow push would otherwise let +// each tick stack another concurrent DOLT_PUSH — incident 2026-06-05). The +// guard is a non-blocking flock; they are skipped where flock is unavailable +// because the guard there deliberately degrades to a warn-and-proceed no-op, +// leaving nothing to assert. + +// requireFlock skips the calling test when flock is not on PATH. +func requireFlock(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("flock"); err != nil { + t.Skip("flock unavailable; the concurrency guard degrades to a warn-and-proceed no-op here") + } +} + +// writeSyncFakeDoltBlockingPush installs a fake dolt that answers the SQL-mode +// remote-lookup and active-branch queries, then BLOCKS on DOLT_PUSH: it creates +// startMarker (signaling the push — and thus the held lock — has been reached) +// and spins until releaseMarker appears, then exits 0. This pins one +// `gc dolt sync` mid-push (holding the concurrency lock) while a test races a +// second sync, then releases it deterministically. The spin is bounded (~60s) +// so a forgotten release cannot leak the process indefinitely. +func writeSyncFakeDoltBlockingPush(t *testing.T, dir, startMarker, releaseMarker string) { + t.Helper() + logPath := filepath.Join(dir, "dolt.log") + body := `#!/bin/sh +printf '%s\n' "$*" >> "` + logPath + `" +case "$*" in + *"SELECT name, url FROM dolt_remotes LIMIT 1"*) + printf 'name,url\norigin,https://example.invalid/repo\n' + exit 0 + ;; + *"SELECT active_branch()"*) + printf 'active_branch()\nmain\n' + exit 0 + ;; + *"CALL DOLT_FETCH("*) + exit 0 + ;; + *"..remotes/origin/"*) + printf 'n\n0\n' + exit 0 + ;; + *"dolt_log('remotes/origin/"*) + printf 'n\n1\n' + exit 0 + ;; + *DOLT_PUSH*) + : > "` + startMarker + `" + i=0 + while [ ! -f "` + releaseMarker + `" ]; do + i=$((i + 1)) + [ "$i" -ge 1200 ] && break + sleep 0.05 + done + exit 0 + ;; +esac +exit 0 +` + if err := os.WriteFile(filepath.Join(dir, "dolt"), []byte(body), 0o755); err != nil { + t.Fatalf("write blocking fake dolt: %v", err) + } +} + +// syncOutcome carries the combined output and exit error of a backgrounded sync. +type syncOutcome struct { + out []byte + err error +} + +// startSyncCmd launches `sh