diff --git a/docs/specs/2026-08-03-asset-route-selection-freshness.md b/docs/specs/2026-08-03-asset-route-selection-freshness.md new file mode 100644 index 0000000..ad48f99 --- /dev/null +++ b/docs/specs/2026-08-03-asset-route-selection-freshness.md @@ -0,0 +1,52 @@ +# Asset route selection and freshness + +## Traceability + +- Spec ID: asset-route-selection-freshness +- Status: Implemented + +## Intent + +Keep the compact 16-route evidence budget without making a normal Better Harness review fail merely because more assets exist. The sampled routes must be the most recently modified routes available, and the envelope must state when collection occurred and how much timestamp evidence was available so reviewers can distinguish current evidence from an unqualified inventory snapshot. + +## Acceptance Scenarios + +- AC-1: When more than 16 owner routes exist, the baseline returns 16 items ordered by descending filesystem modification time, with deterministic fallback ordering for routes whose time is unavailable. +- AC-2: The owner-route envelope preserves `total`, `omitted`, and `truncated`, and adds an explicit `latest-modified` selection contract instead of presenting the sample as exhaustive. +- AC-3: Owner-route evidence records collection time, timestamp source, timestamped/untimestamped counts, and per-item `modifiedAt` when observable; missing file metadata remains bounded and does not abort inventory collection. +- AC-4: Intentional owner-route sampling does not make an otherwise healthy baseline partial. Truncated lint or integrity findings and unavailable stages continue to lower status exactly as before. +- AC-5: The normal evidence bundle accepts an agent-customize lane whose only incompleteness is the disclosed 16-route sample. + +## Non-goals + +- Increase the compact owner-route limit above 16. +- Read asset bodies, Memory bodies, or raw session content. +- Infer freshness from names, versions, route order, Git timestamps, or install metadata. +- Change finding truncation semantics or the authority boundary for user-home assets. + +## Plan and Tasks + +1. Add deterministic failing tests for latest-16 selection, freshness coverage, and non-blocking sampling. +2. Make the asset-baseline compactor collect file metadata through an injectable stat seam and clock, with at most 32 concurrent metadata probes. +3. Separate intentional `sampledStages` from evidence-loss `truncatedStages`; only the latter affects baseline status. +4. Add an evidence-bundle regression showing normal depth remains complete for a healthy sampled baseline. +5. Update operator-facing Better Harness reference text if it currently equates every compact owner-route omission with unavailable evidence. + +Affected modules: `scripts/coding-agent-practices/asset-baseline.mjs`, the Better Harness root routing instruction, and focused baseline/evidence/Skill contract tests. The evidence-bundle adapter requires no production change because it already maps `complete` baselines to an available lane. + +## Test and Review Evidence + +- AC-1..AC-4: `node --test test/agent-asset-baseline.test.mjs` +- AC-5: `node --test test/better-harness-evidence-bundle.test.mjs` +- Full regression: `npm test` +- Generated doc routing after this new spec: `node scripts/doc-link-graph/cli.mjs skills/better-harness` and `node --test test/doc-link-graph.test.mjs` +- Review readiness: run the repository's Change Traceability Review over the final local diff before commit. +- Risk: filesystem timestamps may be absent or coarse. The contract reports timestamp coverage and uses deterministic fallback ordering rather than claiming false recency. + +Observed on 2026-08-03: + +- AC-1..AC-4: focused asset-baseline test passed, 11/11. +- AC-5: focused evidence-bundle test passed, 23/23. +- Root routing contract: focused Better Harness Skill test passed, 12/12. +- Full regression: `npm test` passed, 1020/1020, after installing lockfile dependencies. +- Documentation routing: generated graph remained current and the focused link suite passed, 6/6. diff --git a/scripts/coding-agent-practices/asset-baseline.mjs b/scripts/coding-agent-practices/asset-baseline.mjs index 85c15ff..54987d5 100644 --- a/scripts/coding-agent-practices/asset-baseline.mjs +++ b/scripts/coding-agent-practices/asset-baseline.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node import path from "node:path"; +import { stat as statPath } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import { collectAgentCustomizeInventory } from "../agent-customize/index.mjs"; @@ -14,6 +15,7 @@ export const ASSET_BASELINE_KIND = "agent-asset-baseline"; export const ASSET_BASELINE_SCHEMA_VERSION = 1; export const MAX_BASELINE_FINDINGS = 16; export const MAX_BASELINE_OWNER_ROUTES = 16; +const MAX_OWNER_ROUTE_STAT_CONCURRENCY = 32; const PROVIDERS = new Set(["qoder", "codex", "claude", "cursor", "qwen", "copilot", "pi", "workbuddy"]); const SEVERITY_RANK = Object.freeze({ error: 0, warning: 1, advisory: 2 }); @@ -79,13 +81,20 @@ function workspaceRoute(filePath, workspace) { return relative.split(path.sep).join("/"); } -function ownerRoutes(inventory, workspace) { +function ownerRouteFallbackOrder(left, right) { + return (OWNER_KIND_RANK[left.kind] ?? 99) - (OWNER_KIND_RANK[right.kind] ?? 99) + || (OWNER_SCOPE_RANK[left.scope] ?? 9) - (OWNER_SCOPE_RANK[right.scope] ?? 9) + || String(left.name).localeCompare(String(right.name)); +} + +async function ownerRoutes(inventory, workspace, options = {}) { const routes = new Map(); for (const surface of inventory?.surfaces ?? []) { if (!new Set(["rules", "skills", "mcps", "memories", "agents", "hooks", "commands", "workflows", "plugins"]).has(surface?.type)) continue; for (const item of surface.items ?? []) { const name = text(item?.displayName ?? item?.name ?? item?.label, 96); if (!name) continue; + const sourcePath = item?.path ?? item?.filePath ?? item?.rootPath; const route = Object.fromEntries(Object.entries({ kind: text(surface.type, 32), scope: text(item?.scope ?? surface.scope, 24), @@ -97,42 +106,56 @@ function ownerRoutes(inventory, workspace) { effectiveTarget: text(item?.effectiveTarget, 180), }).filter(([, value]) => value !== undefined && value !== "")); const key = [route.kind, route.scope, route.name, route.version, route.owner, route.route].join(":"); - if (!routes.has(key)) routes.set(key, route); + if (!routes.has(key)) routes.set(key, { route, sourcePath }); } } - const ordered = [...routes.values()].sort((left, right) => - (OWNER_KIND_RANK[left.kind] ?? 99) - (OWNER_KIND_RANK[right.kind] ?? 99) - || (OWNER_SCOPE_RANK[left.scope] ?? 9) - (OWNER_SCOPE_RANK[right.scope] ?? 9) - || String(left.name).localeCompare(String(right.name)), - ); - const byKind = new Map(); - for (const route of ordered) { - const group = byKind.get(route.kind) ?? []; - group.push(route); - byKind.set(route.kind, group); - } - const selected = []; - const groups = [...byKind.values()]; - for (let depth = 0; selected.length < MAX_BASELINE_OWNER_ROUTES; depth += 1) { - let found = false; - for (const group of groups) { - if (group[depth]) { - selected.push(group[depth]); - found = true; - if (selected.length === MAX_BASELINE_OWNER_ROUTES) break; + const stat = options.stat ?? statPath; + const candidates = [...routes.values()]; + const timestamped = new Array(candidates.length); + let nextCandidate = 0; + await Promise.all(Array.from({ length: Math.min(MAX_OWNER_ROUTE_STAT_CONCURRENCY, candidates.length) }, async () => { + while (nextCandidate < candidates.length) { + const index = nextCandidate; + nextCandidate += 1; + const { route, sourcePath } = candidates[index]; + if (!sourcePath) { + timestamped[index] = route; + continue; + } + try { + const info = await stat(sourcePath); + const modifiedAt = info?.mtime instanceof Date ? info.mtime.toISOString() : undefined; + timestamped[index] = modifiedAt ? { ...route, modifiedAt } : route; + } catch { + timestamped[index] = route; } } - if (!found) break; - } + })); + const ordered = timestamped.sort((left, right) => { + const leftTime = left.modifiedAt ? Date.parse(left.modifiedAt) : Number.NEGATIVE_INFINITY; + const rightTime = right.modifiedAt ? Date.parse(right.modifiedAt) : Number.NEGATIVE_INFINITY; + return rightTime - leftTime || ownerRouteFallbackOrder(left, right); + }); + const selected = ordered.slice(0, MAX_BASELINE_OWNER_ROUTES); + const observedAt = (options.now?.() ?? new Date()).toISOString(); + const timestampedCount = ordered.filter((route) => route.modifiedAt).length; return { items: selected, total: ordered.length, omitted: Math.max(0, ordered.length - MAX_BASELINE_OWNER_ROUTES), truncated: ordered.length > MAX_BASELINE_OWNER_ROUTES, + selection: { + strategy: "latest-modified", + limit: MAX_BASELINE_OWNER_ROUTES, + observedAt, + timestampSource: "filesystem-mtime", + timestamped: timestampedCount, + untimestamped: ordered.length - timestampedCount, + }, }; } -function compactInventory(inventory, workspace) { +async function compactInventory(inventory, workspace, options = {}) { const memoryCategories = (inventory?.memories?.categories ?? []) .map((category) => ({ category: text(category?.category ?? category?.name, 72), @@ -161,7 +184,7 @@ function compactInventory(inventory, workspace) { }, summary: inventorySummary, coverageRows, - ownerRoutes: ownerRoutes(inventory, workspace), + ownerRoutes: await ownerRoutes(inventory, workspace, options), memories: { included: Boolean(inventory?.memories?.included), contentPolicy: inventory?.memories?.contentPolicy ?? "raw-memory-content-not-read", @@ -340,7 +363,10 @@ export async function collectAssetBaseline(options = {}, dependencies = {}) { ? available(compactLint(lintResult.value)) : unavailable(lintResult.reason, "lint"); const inventoryEnvelope = inventoryResult.status === "fulfilled" - ? available(compactInventory(inventoryResult.value, workspace)) + ? available(await compactInventory(inventoryResult.value, workspace, { + stat: dependencies.stat, + now: dependencies.now, + })) : unavailable(inventoryResult.reason, "inventory"); let integrityEnvelope; if (inventoryResult.status === "fulfilled") { @@ -357,9 +383,11 @@ export async function collectAssetBaseline(options = {}, dependencies = {}) { const availableCount = Object.values(envelopes).filter((envelope) => envelope.status === "available").length; const truncatedStages = [ lintEnvelope.data?.findings?.truncated ? "lint-findings" : null, - inventoryEnvelope.data?.ownerRoutes?.truncated ? "inventory-owner-routes" : null, integrityEnvelope.data?.findings?.truncated ? "integrity-findings" : null, ].filter(Boolean); + const sampledStages = [ + inventoryEnvelope.data?.ownerRoutes?.truncated ? "inventory-owner-routes" : null, + ].filter(Boolean); return { kind: ASSET_BASELINE_KIND, schemaVersion: ASSET_BASELINE_SCHEMA_VERSION, @@ -377,6 +405,7 @@ export async function collectAssetBaseline(options = {}, dependencies = {}) { ownerRouteLimit: MAX_BASELINE_OWNER_ROUTES, inheritedWorkspaceCount: rawInventory?.diagnostics?.inheritedWorkspaceCount ?? 0, truncatedStages, + sampledStages, }, }; } diff --git a/skills/better-harness/SKILL.md b/skills/better-harness/SKILL.md index 41c9e77..fd6d6c2 100644 --- a/skills/better-harness/SKILL.md +++ b/skills/better-harness/SKILL.md @@ -53,7 +53,7 @@ lane contains the bounded `lint`, `inventory`, and `integrity` envelopes from one shared asset snapshot. Keep every lane and stage status and each provider distinct. Use the individual `session-analysis facts`, `core-change-watch evidence-pack`, `coding-agent-practices asset-baseline`, or `harness analyze` -command only to diagnose a named unavailable or truncated owner; do not +command only to diagnose a named unavailable or evidence-loss stage; do not substitute diagnostic output into the bundle or rerun all owners. Counts for Rules, Skills, MCP, Memory, Agents, Hooks, Commands, Workflows, and Plugins only route inspection. Zero or high counts never create findings or scores. A diff --git a/test/agent-asset-baseline.test.mjs b/test/agent-asset-baseline.test.mjs index b0362c6..226c73c 100644 --- a/test/agent-asset-baseline.test.mjs +++ b/test/agent-asset-baseline.test.mjs @@ -8,6 +8,7 @@ import test from "node:test"; import { ASSET_BASELINE_KIND, MAX_BASELINE_FINDINGS, + MAX_BASELINE_OWNER_ROUTES, collectAssetBaseline, formatAssetBaselineMarkdown, } from "../scripts/coding-agent-practices/asset-baseline.mjs"; @@ -123,9 +124,9 @@ test("asset baseline shares one inventory snapshot and emits compact AI envelope assert.equal(result.envelopes.inventory.data.ownerRoutes.truncated, true); assert.deepEqual(result.diagnostics.truncatedStages, [ "lint-findings", - "inventory-owner-routes", "integrity-findings", ]); + assert.deepEqual(result.diagnostics.sampledStages, ["inventory-owner-routes"]); assert.equal(Object.hasOwn(result.envelopes.inventory.data.summary, "practiceCoverageRows"), false); const serialized = JSON.stringify(result); assert.ok(Buffer.byteLength(serialized) < 12_000, "fixture baseline must stay compact for AI reading"); @@ -152,6 +153,66 @@ test("asset baseline preserves partial stage failures without hiding healthy env assert.match(markdown, /inventory: unavailable/); }); +test("asset baseline samples the latest 16 owner routes with explicit freshness coverage", async () => { + const workspace = path.resolve("/tmp/better-harness-latest-owner-routes"); + const observedAt = new Date("2026-08-03T08:00:00.000Z"); + const modifiedBase = Date.parse("2026-08-01T00:00:00.000Z"); + let activeStats = 0; + let maxActiveStats = 0; + const items = Array.from({ length: 40 }, (_, index) => ({ + name: `skill-${String(index).padStart(2, "0")}`, + scope: "workspace", + path: path.join(workspace, `skill-${String(index).padStart(2, "0")}.md`), + })); + const result = await collectAssetBaseline({ provider: "codex", workspace }, { + now: () => observedAt, + stat: async (filePath) => { + activeStats += 1; + maxActiveStats = Math.max(maxActiveStats, activeStats); + await new Promise((resolve) => setImmediate(resolve)); + activeStats -= 1; + return { mtime: new Date(modifiedBase + Number(path.basename(filePath).match(/\d+/u)?.[0]) * 1_000) }; + }, + collectRawInventory: async () => ({}), + runLint: async () => ({ kind: "agent-lint", profile: "agent-assets-review", summary: {}, findings: [] }), + collectPublicInventory: async () => ({ + scope: { platform: "codex", includeUserHome: false }, + summary: {}, + surfaces: [{ type: "skills", scope: "workspace", items }], + memories: { included: false, categories: [] }, + warnings: [], + }), + reviewIntegrity: () => ({ + kind: "asset-integrity-review", + profile: "asset-integrity-review", + status: "reviewed", + summary: { findingCount: 0 }, + findings: [], + }), + }); + + const routes = result.envelopes.inventory.data.ownerRoutes; + assert.equal(result.status, "complete"); + assert.equal(routes.items.length, MAX_BASELINE_OWNER_ROUTES); + assert.deepEqual(routes.items.map((item) => item.name), + Array.from({ length: 16 }, (_, index) => `skill-${String(39 - index).padStart(2, "0")}`)); + assert.equal(routes.items[0].modifiedAt, new Date(modifiedBase + 39_000).toISOString()); + assert.equal(routes.total, 40); + assert.equal(routes.omitted, 24); + assert.equal(routes.truncated, true); + assert.deepEqual(routes.selection, { + strategy: "latest-modified", + limit: 16, + observedAt: observedAt.toISOString(), + timestampSource: "filesystem-mtime", + timestamped: 40, + untimestamped: 0, + }); + assert.ok(maxActiveStats > 1 && maxActiveStats <= 32); + assert.deepEqual(result.diagnostics.truncatedStages, []); + assert.deepEqual(result.diagnostics.sampledStages, ["inventory-owner-routes"]); +}); + test("Qoder asset baseline includes selected-project Memory titles by default", async () => { let publicOptions; const result = await collectAssetBaseline({ provider: "qoder", workspace: "/tmp/qoder-project" }, { diff --git a/test/better-harness-evidence-bundle.test.mjs b/test/better-harness-evidence-bundle.test.mjs index 0b16130..b08c513 100644 --- a/test/better-harness-evidence-bundle.test.mjs +++ b/test/better-harness-evidence-bundle.test.mjs @@ -411,6 +411,40 @@ test("Claude agentCustomize lane routes the provider and isolated config paths", assert.equal(received["include-user-home"], true); }); +test("normal agentCustomize evidence accepts a disclosed latest-route sample", async () => { + const context = freezeEvidenceBundleContext({ + workspace: ".", + platform: "codex", + depth: "normal", + "include-user-home": true, + }, NOW); + const baseline = { + kind: "agent-asset-baseline", + status: "complete", + envelopes: { + inventory: { + status: "available", + data: { + ownerRoutes: { + items: Array.from({ length: 16 }, (_, index) => ({ name: `asset-${index}` })), + total: 55, + omitted: 39, + truncated: true, + selection: { strategy: "latest-modified", limit: 16 }, + }, + }, + }, + }, + diagnostics: { truncatedStages: [], sampledStages: ["inventory-owner-routes"] }, + }; + const lane = await collectAgentCustomize(context, {}, { + collectAssetBaseline: async () => baseline, + }); + + assert.equal(lane.status, "available"); + assert.equal(lane.data, baseline); +}); + test("Qwen agentCustomize lane routes the provider and isolated config paths", async () => { const context = freezeEvidenceBundleContext({ workspace: ".", diff --git a/test/better-harness-skill.test.mjs b/test/better-harness-skill.test.mjs index 1859645..35607ec 100644 --- a/test/better-harness-skill.test.mjs +++ b/test/better-harness-skill.test.mjs @@ -116,7 +116,7 @@ test("Step 1 establishes one provider-labelled evidence bundle", () => { assert.match(skill, /`memberRoute` or `null`/); assert.match(skill, /Providers must agree/); assert.match(skill, /bounded `lint`, `inventory`, and `integrity` envelopes/); - assert.match(skill, /individual [\s\S]+command only to diagnose a named unavailable or truncated owner/); + assert.match(skill, /individual [\s\S]+command only to diagnose a named unavailable or evidence-loss stage/); assert.doesNotMatch(skill, / agent-lint --workspace /); assert.match(skill, /Rules,\s+Skills, MCP, Memory, Agents, Hooks, Commands, Workflows, and Plugins/); assert.match(skill, /Zero or high counts never create findings or scores/);