diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a520e6..a81d5d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Routing eval (GW-049): optional `contribution_mode` on outcomes, additive eval `ground_truth.routing`, reconstructed routing corpus kept out of frozen headline accuracy, `false_build_occupied` stays 0 for definitive ownership. - Local watch registry (GW-048): `gitworthy watch add|list|show|recheck|remove` and MCP `watch_*`. Fingerprint recheck reports exact deltas. No auto-create from WATCH routes and no GitHub writes. - Contribution routing v2 (GW-043–047): `worth_check` can attach a `routing` decision; `gitworthy portfolio` / MCP `portfolio` ranks issue+PR opportunities by contribution mode with separate `dispatch_state`; `gitworthy prs` / MCP `pr_scan` is a bounded two-stage PR inventory. Verdict policy is unchanged. Hermes `contribution_profile` examples stay in docs, not global defaults. Org portfolio fans out PR scans to at most 5 hunt repos (inventory ≤25, enrich ≤5). Advisory `failed_checks` do not demote BUILD. - Agent Plugins v1.0.0 packaging: `plugin.json`, `mcp.json`, canonical `skills/gitworthy/SKILL.md`, CI sync check (`pnpm agent-plugins:check`). diff --git a/docs/EVALS.md b/docs/EVALS.md index 98ef7e6..6a06a50 100644 --- a/docs/EVALS.md +++ b/docs/EVALS.md @@ -11,6 +11,7 @@ Full detail: [`../eval/README.md`](../eval/README.md), [`../eval/METRICS.md`](.. | **frozen** | Offline fixture replay | Yes | | **live** | Public GitHub/npm/git | Advisory | | **private** | Local captures (gitignored) | No | +| **reconstructed routing** | In-process `routeContribution` fixtures | Advisory; never mixed into frozen headline accuracy | ```sh pnpm eval:frozen diff --git a/eval/reconstructed/README.md b/eval/reconstructed/README.md new file mode 100644 index 0000000..39a8808 --- /dev/null +++ b/eval/reconstructed/README.md @@ -0,0 +1,8 @@ +# Reconstructed routing eval + +This partition is **not** snapshot-backed frozen eval. + +- Cases live in `src/core/routing-eval-cases.ts`. +- Headline accuracy (`mode_top1_accuracy`) uses only `partition: snapshot` cases. +- Reconstructed / research cases never mix into frozen-suite accuracy. +- `false_build_occupied` must stay `0` for definitive-ownership fixtures. diff --git a/schemas/gitworthy-eval-case-catalog.v1.schema.json b/schemas/gitworthy-eval-case-catalog.v1.schema.json index 087b779..fd53f7d 100644 --- a/schemas/gitworthy-eval-case-catalog.v1.schema.json +++ b/schemas/gitworthy-eval-case-catalog.v1.schema.json @@ -185,6 +185,72 @@ "type": "string", "minLength": 1 } + }, + "routing": { + "type": "object", + "properties": { + "primary_mode": { + "type": "string", + "enum": [ + "BUILD", + "REVIEW", + "SALVAGE", + "REPRODUCE", + "EVAL", + "DOC", + "WATCH", + "PASS" + ] + }, + "acceptable_modes": { + "default": [], + "type": "array", + "items": { + "type": "string", + "enum": [ + "BUILD", + "REVIEW", + "SALVAGE", + "REPRODUCE", + "EVAL", + "DOC", + "WATCH", + "PASS" + ] + } + }, + "forbidden_modes": { + "default": [], + "type": "array", + "items": { + "type": "string", + "enum": [ + "BUILD", + "REVIEW", + "SALVAGE", + "REPRODUCE", + "EVAL", + "DOC", + "WATCH", + "PASS" + ] + } + }, + "build_contention": { + "type": "string", + "enum": [ + "GREEN", + "YELLOW", + "RED" + ] + } + }, + "required": [ + "primary_mode", + "acceptable_modes", + "forbidden_modes" + ], + "additionalProperties": false } }, "required": [ diff --git a/schemas/gitworthy-eval-case.v1.schema.json b/schemas/gitworthy-eval-case.v1.schema.json index 5a6b281..b2c19db 100644 --- a/schemas/gitworthy-eval-case.v1.schema.json +++ b/schemas/gitworthy-eval-case.v1.schema.json @@ -162,6 +162,72 @@ "type": "string", "minLength": 1 } + }, + "routing": { + "type": "object", + "properties": { + "primary_mode": { + "type": "string", + "enum": [ + "BUILD", + "REVIEW", + "SALVAGE", + "REPRODUCE", + "EVAL", + "DOC", + "WATCH", + "PASS" + ] + }, + "acceptable_modes": { + "default": [], + "type": "array", + "items": { + "type": "string", + "enum": [ + "BUILD", + "REVIEW", + "SALVAGE", + "REPRODUCE", + "EVAL", + "DOC", + "WATCH", + "PASS" + ] + } + }, + "forbidden_modes": { + "default": [], + "type": "array", + "items": { + "type": "string", + "enum": [ + "BUILD", + "REVIEW", + "SALVAGE", + "REPRODUCE", + "EVAL", + "DOC", + "WATCH", + "PASS" + ] + } + }, + "build_contention": { + "type": "string", + "enum": [ + "GREEN", + "YELLOW", + "RED" + ] + } + }, + "required": [ + "primary_mode", + "acceptable_modes", + "forbidden_modes" + ], + "additionalProperties": false } }, "required": [ diff --git a/schemas/gitworthy-outcome-event.v1.schema.json b/schemas/gitworthy-outcome-event.v1.schema.json index 9e76c3b..d7309a0 100644 --- a/schemas/gitworthy-outcome-event.v1.schema.json +++ b/schemas/gitworthy-outcome-event.v1.schema.json @@ -93,6 +93,31 @@ "pr_url": { "type": "string", "format": "uri" + }, + "contribution_mode": { + "type": "string", + "enum": [ + "BUILD", + "REVIEW", + "SALVAGE", + "REPRODUCE", + "EVAL", + "DOC", + "WATCH", + "PASS" + ] + }, + "investigation_class": { + "type": "string", + "enum": [ + "repro", + "ci_triage", + "history", + "review", + "salvage", + "docs", + "eval" + ] } }, "required": [ diff --git a/schemas/gitworthy-routing-eval-metrics.v1.schema.json b/schemas/gitworthy-routing-eval-metrics.v1.schema.json new file mode 100644 index 0000000..b3a94ba --- /dev/null +++ b/schemas/gitworthy-routing-eval-metrics.v1.schema.json @@ -0,0 +1,112 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "routing_cases": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "snapshot_cases": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "reconstructed_cases": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "mode_top1_accuracy": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "acceptable_mode_accuracy": { + "anyOf": [ + { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + { + "type": "null" + } + ] + }, + "false_build_occupied": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "false_build_blocked": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "false_pass_actionable": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "counts_by_mode": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "counts_by_contention": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "counts_by_confidence": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + } + }, + "headline_excludes_reconstructed": { + "type": "boolean", + "const": true + } + }, + "required": [ + "routing_cases", + "snapshot_cases", + "reconstructed_cases", + "mode_top1_accuracy", + "acceptable_mode_accuracy", + "false_build_occupied", + "false_build_blocked", + "false_pass_actionable", + "counts_by_mode", + "counts_by_contention", + "counts_by_confidence", + "headline_excludes_reconstructed" + ], + "additionalProperties": false +} diff --git a/scripts/generate-schemas.ts b/scripts/generate-schemas.ts index 9747045..ff9dd6c 100644 --- a/scripts/generate-schemas.ts +++ b/scripts/generate-schemas.ts @@ -22,6 +22,7 @@ import { PortfolioItemSchema } from '../src/contracts/portfolio.js'; import { PrScanResultSchema } from '../src/contracts/pr-scan.js'; import { OpportunityTargetSchema } from '../src/contracts/opportunities.js'; import { ContributionProfileSchema } from '../src/contracts/contribution-profile.js'; +import { RoutingEvalMetricsSchema } from '../src/contracts/routing-eval.js'; import { TrackOCovariatesRecordSchema, TrackOContingencyTableSchema, @@ -61,6 +62,7 @@ const docs: Array<{ file: string; schema: z.ZodType }> = [ { file: 'gitworthy-pr-scan.v1.schema.json', schema: PrScanResultSchema }, { file: 'gitworthy-portfolio-item.v1.schema.json', schema: PortfolioItemSchema }, { file: 'gitworthy-contribution-profile.v1.schema.json', schema: ContributionProfileSchema }, + { file: 'gitworthy-routing-eval-metrics.v1.schema.json', schema: RoutingEvalMetricsSchema }, { file: 'gitworthy-track-o-join-key.v1.schema.json', schema: TrackOJoinKeySchema }, { file: 'gitworthy-track-o-covariates.v1.schema.json', schema: TrackOCovariatesRecordSchema }, { file: 'gitworthy-track-o-contingency.v1.schema.json', schema: TrackOContingencyTableSchema } diff --git a/src/contracts/eval.ts b/src/contracts/eval.ts index 8519c9a..bd63058 100644 --- a/src/contracts/eval.ts +++ b/src/contracts/eval.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; import { DispositionSchema, SCHEMA_VERSION, SchemaVersionSchema, VerdictSchema } from './common.js'; +import { BuildContentionSchema, ContributionModeSchema } from './routing.js'; /** Case/adjudication contract shared by frozen, live, and private eval suites (GW-021). */ export const EVAL_CASE_VERSION = 1 as const; @@ -35,7 +36,13 @@ export const EvalGroundTruthSchema = z.object({ required_findings: z.array(z.string().min(1)).default([]), forbidden_findings: z.array(z.string().min(1)).default([]), required_signals: z.array(z.string().min(1)).default([]), - forbidden_signals: z.array(z.string().min(1)).default([]) + forbidden_signals: z.array(z.string().min(1)).default([]), + routing: z.object({ + primary_mode: ContributionModeSchema, + acceptable_modes: z.array(ContributionModeSchema).default([]), + forbidden_modes: z.array(ContributionModeSchema).default([]), + build_contention: BuildContentionSchema.optional() + }).strict().optional() }).strict(); export const EvalCaseClassificationSchema = z.enum([ diff --git a/src/contracts/outcomes.ts b/src/contracts/outcomes.ts index 529827e..c8e373f 100644 --- a/src/contracts/outcomes.ts +++ b/src/contracts/outcomes.ts @@ -1,4 +1,5 @@ import { z } from 'zod'; +import { ContributionModeSchema } from './routing.js'; /** Outcome event schema for the local durable store (Track O T1 labels map onto these events). */ export const OutcomeEventNameSchema = z.enum([ @@ -46,7 +47,18 @@ export const OutcomeEventSchema = z.object({ /** True when the contribution proceeded despite a soft SKIP (Track O anti-SKIP policy). */ acted_against_skip: z.boolean().optional(), /** PR URL once known; join key companion to decision_id. */ - pr_url: z.string().url().optional() + pr_url: z.string().url().optional(), + /** Optional routing annotation. Legacy outcomes without this still parse. */ + contribution_mode: ContributionModeSchema.optional(), + investigation_class: z.enum([ + 'repro', + 'ci_triage', + 'history', + 'review', + 'salvage', + 'docs', + 'eval' + ]).optional() }).strict().superRefine((value, ctx) => { if (value.close_reason !== undefined && value.event !== 'closed_unmerged') { ctx.addIssue({ diff --git a/src/contracts/routing-eval.ts b/src/contracts/routing-eval.ts new file mode 100644 index 0000000..74dc905 --- /dev/null +++ b/src/contracts/routing-eval.ts @@ -0,0 +1,44 @@ +import { z } from 'zod'; +import { BuildContentionSchema, ContributionModeSchema } from './routing.js'; + +export const DecisionAccelerationSchema = z.object({ + from_mode: ContributionModeSchema, + to_mode: ContributionModeSchema, + hours_saved: z.number().nonnegative(), + reason: z.string().min(1) +}).strict(); + +export const RoutingEvalPartitionSchema = z.enum(['snapshot', 'reconstructed']); + +export const RoutingEvalCaseSchema = z.object({ + id: z.string().min(1), + name: z.string().min(1), + partition: RoutingEvalPartitionSchema, + adversarial: z.boolean().default(false), + facts: z.record(z.string(), z.unknown()), + expected: z.object({ + primary_mode: ContributionModeSchema, + acceptable_modes: z.array(ContributionModeSchema).default([]), + forbidden_modes: z.array(ContributionModeSchema).default([]), + build_contention: BuildContentionSchema.optional() + }).strict() +}).strict(); + +export const RoutingEvalMetricsSchema = z.object({ + routing_cases: z.number().int().nonnegative(), + snapshot_cases: z.number().int().nonnegative(), + reconstructed_cases: z.number().int().nonnegative(), + mode_top1_accuracy: z.number().min(0).max(1).nullable(), + acceptable_mode_accuracy: z.number().min(0).max(1).nullable(), + false_build_occupied: z.number().int().nonnegative(), + false_build_blocked: z.number().int().nonnegative(), + false_pass_actionable: z.number().int().nonnegative(), + counts_by_mode: z.record(z.string(), z.number().int().nonnegative()), + counts_by_contention: z.record(z.string(), z.number().int().nonnegative()), + counts_by_confidence: z.record(z.string(), z.number().int().nonnegative()), + headline_excludes_reconstructed: z.literal(true) +}).strict(); + +export type DecisionAcceleration = z.infer; +export type RoutingEvalCase = z.infer; +export type RoutingEvalMetrics = z.infer; diff --git a/src/core/index.ts b/src/core/index.ts index 1a924af..a3691ae 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -41,3 +41,4 @@ export type { ProbeTemplate, ProbeTemplateId } from './probe-templates.js'; export { parseSkillProfile, resolveSkillProfile, scoreSkillFit } from './skill-fit.js'; export type { SkillProfile } from './skill-fit.js'; export { routeContribution, scoreEvidenceability } from '../decision/contribution-route.js'; +export { runReconstructedRoutingEval, scoreRoutingCases } from './routing-eval.js'; diff --git a/src/core/portfolio.ts b/src/core/portfolio.ts index 6dcb0ee..2f6a6fd 100644 --- a/src/core/portfolio.ts +++ b/src/core/portfolio.ts @@ -105,7 +105,7 @@ export function computeCapacity( const lastActive = [...ordered].reverse().find((event) => ACTIVE_EVENTS.has(event.event)); if (!lastActive) continue; if (lastTerminal && lastTerminal.occurred_at >= lastActive.occurred_at) continue; - const modeRaw = lastActive.data.contribution_mode; + const modeRaw = lastActive.contribution_mode ?? lastActive.data.contribution_mode; const mode = typeof modeRaw === 'string' ? modeRaw : 'BUILD'; if (typeof modeRaw !== 'string') { missingMode += 1; diff --git a/src/core/routing-eval-cases.ts b/src/core/routing-eval-cases.ts new file mode 100644 index 0000000..d2bc50f --- /dev/null +++ b/src/core/routing-eval-cases.ts @@ -0,0 +1,212 @@ +import type { RoutingEvalCase } from '../contracts/routing-eval.js'; +import type { RouteFacts } from '../contracts/routing.js'; + +const complete = { + mandatory_checks_complete: true, + failed_checks: [] as string[], + skipped_checks: [] as string[], + budget_truncated: false, + rate_limit_degraded: false, + advisory_missing: [] as string[] +}; + +function facts(partial: Partial & Pick): RouteFacts { + return { + findings: [], + mandatoryFailures: [], + coverage: complete, + ...partial + }; +} + +const reconstructedRoutingCaseDrafts: Array & { adversarial?: boolean }> = [ + { + id: 'recon-unoccupied-bug-build', + name: 'Unoccupied clear bug → BUILD', + partition: 'reconstructed', + facts: facts({ + verdict: 'ACT', + disposition: 'greenfield', + quality: { looksLikeBug: true, repro: 'present', softAsk: false }, + linked: { activeClosers: 0, activeRelatedPrs: 0, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false } + }), + expected: { primary_mode: 'BUILD', acceptable_modes: [], forbidden_modes: ['PASS'] } + }, + { + id: 'recon-missing-repro', + name: 'Missing repro → REPRODUCE', + partition: 'reconstructed', + facts: facts({ + verdict: 'VERIFY', + disposition: 'greenfield', + quality: { looksLikeBug: true, repro: 'missing', softAsk: false }, + linked: { activeClosers: 0, activeRelatedPrs: 0, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false } + }), + expected: { primary_mode: 'REPRODUCE', acceptable_modes: ['WATCH'], forbidden_modes: ['BUILD'] } + }, + { + id: 'recon-explicit-closer', + name: 'Explicit closer → REVIEW', + partition: 'reconstructed', + facts: facts({ + verdict: 'SKIP', + disposition: 'land_only', + findings: [{ id: '1', type: 'linked_pr_open', strength: 'definitive', effect: 'block', source: 'test', message: 'closer', data: {} }], + linked: { activeClosers: 1, activeRelatedPrs: 1, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false, healthyActiveCloser: false } + }), + expected: { primary_mode: 'REVIEW', acceptable_modes: ['WATCH'], forbidden_modes: ['BUILD'] } + }, + { + id: 'recon-multiple-closers', + name: 'Multiple closers → REVIEW', + partition: 'reconstructed', + facts: facts({ + verdict: 'SKIP', + disposition: 'land_only', + findings: [{ id: '1', type: 'linked_pr_open', strength: 'definitive', effect: 'block', source: 'test', message: 'closer', data: {} }], + linked: { activeClosers: 2, activeRelatedPrs: 2, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false } + }), + expected: { primary_mode: 'REVIEW', acceptable_modes: [], forbidden_modes: ['BUILD'] } + }, + { + id: 'recon-healthy-active-pr', + name: 'Healthy active PR → WATCH', + partition: 'reconstructed', + facts: facts({ + verdict: 'SKIP', + disposition: 'land_only', + findings: [{ id: '1', type: 'linked_pr_open', strength: 'definitive', effect: 'block', source: 'test', message: 'closer', data: {} }], + linked: { activeClosers: 1, activeRelatedPrs: 1, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false, healthyActiveCloser: true } + }), + expected: { primary_mode: 'WATCH', acceptable_modes: ['REVIEW'], forbidden_modes: ['BUILD'] } + }, + { + id: 'recon-closed-unmerged', + name: 'Closed unmerged credible → SALVAGE or REVIEW', + partition: 'reconstructed', + facts: facts({ + verdict: 'VERIFY', + disposition: 'review', + linked: { + activeClosers: 0, activeRelatedPrs: 0, closedUnmergedAttempts: 1, mergedClosers: 0, + assigned: false, claimRequired: false, issueOpen: true, substantivePriorAttempt: true + } + }), + expected: { primary_mode: 'SALVAGE', acceptable_modes: ['REVIEW'], forbidden_modes: [] } + }, + { + id: 'recon-claim-required', + name: 'Claim required → WATCH', + partition: 'reconstructed', + facts: facts({ + verdict: 'VERIFY', + disposition: 'claim_first', + linked: { activeClosers: 0, activeRelatedPrs: 0, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: true } + }), + expected: { primary_mode: 'WATCH', acceptable_modes: [], forbidden_modes: ['BUILD'] } + }, + { + id: 'recon-assigned', + name: 'Assigned → WATCH', + partition: 'reconstructed', + facts: facts({ + verdict: 'VERIFY', + disposition: 'greenfield', + linked: { activeClosers: 0, activeRelatedPrs: 0, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: true, claimRequired: false } + }), + expected: { primary_mode: 'WATCH', acceptable_modes: [], forbidden_modes: ['BUILD'] } + }, + { + id: 'recon-released-fix', + name: 'Released fix → PASS', + partition: 'reconstructed', + facts: facts({ + verdict: 'SKIP', + disposition: 'blocked', + findings: [{ id: '1', type: 'released_fix', strength: 'definitive', effect: 'block', source: 'test', message: 'released', data: {} }], + linked: { activeClosers: 0, activeRelatedPrs: 0, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false } + }), + expected: { primary_mode: 'PASS', acceptable_modes: [], forbidden_modes: ['BUILD'] } + }, + { + id: 'recon-no-pr-path', + name: 'No PR path → PASS', + partition: 'reconstructed', + facts: facts({ + verdict: 'SKIP', + disposition: 'blocked', + findings: [{ id: '1', type: 'no_pr_path', strength: 'definitive', effect: 'block', source: 'test', message: 'no pr', data: {} }], + linked: { activeClosers: 0, activeRelatedPrs: 0, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false } + }), + expected: { primary_mode: 'PASS', acceptable_modes: [], forbidden_modes: ['BUILD'] } + }, + { + id: 'recon-docs-only', + name: 'Docs-only → DOC', + partition: 'reconstructed', + facts: facts({ + verdict: 'ACT', + disposition: 'greenfield', + categoryHints: { documentation: true, evaluation: false }, + linked: { activeClosers: 0, activeRelatedPrs: 0, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false } + }), + expected: { primary_mode: 'DOC', acceptable_modes: ['BUILD'], forbidden_modes: [] } + }, + { + id: 'recon-provider-failure', + name: 'Provider failure → low-confidence non-BUILD', + partition: 'reconstructed', + facts: facts({ + verdict: 'VERIFY', + disposition: 'greenfield', + coverage: { ...complete, mandatory_checks_complete: false, failed_checks: ['linked_work'] }, + linked: { activeClosers: 0, activeRelatedPrs: 0, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false } + }), + expected: { primary_mode: 'WATCH', acceptable_modes: ['REVIEW', 'REPRODUCE', 'PASS'], forbidden_modes: ['BUILD'] } + }, + { + id: 'adv-closer-provider-error', + name: 'Closer + provider error still forbids BUILD', + partition: 'reconstructed', + adversarial: true, + facts: facts({ + verdict: 'VERIFY', + disposition: 'land_only', + findings: [{ id: '1', type: 'linked_pr_open', strength: 'definitive', effect: 'block', source: 'test', message: 'closer', data: {} }], + coverage: { ...complete, mandatory_checks_complete: false, failed_checks: ['issue_vs_main'] }, + linked: { activeClosers: 1, activeRelatedPrs: 1, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false } + }), + expected: { primary_mode: 'REVIEW', acceptable_modes: ['WATCH'], forbidden_modes: ['BUILD'] } + }, + { + id: 'adv-title-overlap-only', + name: 'Heuristic title overlap only cannot own the issue', + partition: 'reconstructed', + adversarial: true, + facts: facts({ + verdict: 'VERIFY', + disposition: 'crowded', + findings: [{ id: '1', type: 'title_overlap_pr', strength: 'heuristic', effect: 'verify', source: 'test', message: 'overlap', data: {} }], + linked: { activeClosers: 0, activeRelatedPrs: 1, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: false, claimRequired: false } + }), + expected: { primary_mode: 'REVIEW', acceptable_modes: ['WATCH', 'REPRODUCE'], forbidden_modes: [] } + }, + { + id: 'adv-assigned-plus-closer', + name: 'Assigned + explicit closer still forbids BUILD', + partition: 'reconstructed', + adversarial: true, + facts: facts({ + verdict: 'SKIP', + disposition: 'land_only', + findings: [{ id: '1', type: 'linked_pr_open', strength: 'definitive', effect: 'block', source: 'test', message: 'closer', data: {} }], + linked: { activeClosers: 1, activeRelatedPrs: 1, closedUnmergedAttempts: 0, mergedClosers: 0, assigned: true, claimRequired: false } + }), + expected: { primary_mode: 'REVIEW', acceptable_modes: ['WATCH'], forbidden_modes: ['BUILD'] } + } +]; + +export const reconstructedRoutingCases: RoutingEvalCase[] = reconstructedRoutingCaseDrafts.map((item) => ({ + adversarial: false, + ...item +})); diff --git a/src/core/routing-eval.ts b/src/core/routing-eval.ts new file mode 100644 index 0000000..8eb0898 --- /dev/null +++ b/src/core/routing-eval.ts @@ -0,0 +1,72 @@ +import type { RouteFacts, ContributionMode } from '../contracts/routing.js'; +import { routeContribution } from '../decision/contribution-route.js'; +import { + RoutingEvalMetricsSchema, + type RoutingEvalCase, + type RoutingEvalMetrics +} from '../contracts/routing-eval.js'; +import { reconstructedRoutingCases } from './routing-eval-cases.js'; + +function asFacts(value: Record): RouteFacts { + return value as unknown as RouteFacts; +} + +export function scoreRoutingCases(cases: RoutingEvalCase[]): RoutingEvalMetrics { + let top1 = 0; + let acceptable = 0; + let falseBuildOccupied = 0; + let falseBuildBlocked = 0; + let falsePassActionable = 0; + const counts_by_mode: Record = {}; + const counts_by_contention: Record = {}; + const counts_by_confidence: Record = {}; + const snapshot = cases.filter((item) => item.partition === 'snapshot'); + const reconstructed = cases.filter((item) => item.partition === 'reconstructed'); + const headline = snapshot.length > 0 ? snapshot : []; + + for (const item of cases) { + const decision = routeContribution(asFacts(item.facts)); + counts_by_mode[decision.primary_mode] = (counts_by_mode[decision.primary_mode] ?? 0) + 1; + counts_by_contention[decision.build_contention] = (counts_by_contention[decision.build_contention] ?? 0) + 1; + counts_by_confidence[decision.confidence] = (counts_by_confidence[decision.confidence] ?? 0) + 1; + const okTop = decision.primary_mode === item.expected.primary_mode; + const okAccept = okTop || item.expected.acceptable_modes.includes(decision.primary_mode); + if (item.partition === 'snapshot') { + if (okTop) top1 += 1; + if (okAccept) acceptable += 1; + } + if (item.expected.forbidden_modes.includes('BUILD') && decision.primary_mode === 'BUILD') { + falseBuildOccupied += 1; + } + if (item.expected.primary_mode === 'BUILD' && decision.primary_mode !== 'BUILD' && !item.expected.acceptable_modes.includes(decision.primary_mode)) { + falseBuildBlocked += 1; + } + if (item.expected.forbidden_modes.includes('PASS') && decision.primary_mode === 'PASS') { + falsePassActionable += 1; + } + } + + const headlineCount = headline.length; + return RoutingEvalMetricsSchema.parse({ + routing_cases: cases.length, + snapshot_cases: snapshot.length, + reconstructed_cases: reconstructed.length, + mode_top1_accuracy: headlineCount === 0 ? null : top1 / headlineCount, + acceptable_mode_accuracy: headlineCount === 0 ? null : acceptable / headlineCount, + false_build_occupied: falseBuildOccupied, + false_build_blocked: falseBuildBlocked, + false_pass_actionable: falsePassActionable, + counts_by_mode, + counts_by_contention, + counts_by_confidence, + headline_excludes_reconstructed: true + }); +} + +export function runReconstructedRoutingEval(): RoutingEvalMetrics { + return scoreRoutingCases(reconstructedRoutingCases); +} + +export function coverageHasMode(mode: ContributionMode): boolean { + return reconstructedRoutingCases.some((item) => item.expected.primary_mode === mode); +} diff --git a/test/portfolio.test.ts b/test/portfolio.test.ts index 6f9c415..0155dbe 100644 --- a/test/portfolio.test.ts +++ b/test/portfolio.test.ts @@ -79,8 +79,8 @@ describe('portfolio capacity and dispatch', () => { fetched_at: '2026-08-01T00:00:00.000Z' }), listOutcomes: async () => [ - outcome({ event: 'selected', data: { contribution_mode: 'BUILD' } }), - outcome({ event: 'pr_opened', target: { repo: 'o/r', issue_number: 2 }, data: { contribution_mode: 'BUILD' } }) + outcome({ event: 'selected', contribution_mode: 'BUILD' }), + outcome({ event: 'pr_opened', target: { repo: 'o/r', issue_number: 2 }, contribution_mode: 'BUILD' }) ] }); expect(result.items[0]?.primary_mode).toBe('BUILD'); diff --git a/test/routing-eval.test.ts b/test/routing-eval.test.ts new file mode 100644 index 0000000..328dbc8 --- /dev/null +++ b/test/routing-eval.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from 'vitest'; +import { OutcomeEventSchema } from '../src/contracts/outcomes.js'; +import { EvalGroundTruthSchema } from '../src/contracts/eval.js'; +import { runReconstructedRoutingEval } from '../src/core/routing-eval.js'; +import { reconstructedRoutingCases } from '../src/core/routing-eval-cases.js'; +import { routeContribution } from '../src/decision/contribution-route.js'; +import type { RouteFacts } from '../src/contracts/routing.js'; + +describe('routing eval reconstructed partition', () => { + it('keeps reconstructed cases out of headline accuracy and forbids occupied BUILD', () => { + const metrics = runReconstructedRoutingEval(); + expect(metrics.reconstructed_cases).toBe(reconstructedRoutingCases.length); + expect(metrics.snapshot_cases).toBe(0); + expect(metrics.mode_top1_accuracy).toBeNull(); + expect(metrics.headline_excludes_reconstructed).toBe(true); + expect(metrics.false_build_occupied).toBe(0); + for (const item of reconstructedRoutingCases) { + const decision = routeContribution(item.facts as unknown as RouteFacts); + expect( + [item.expected.primary_mode, ...item.expected.acceptable_modes], + item.id + ).toContain(decision.primary_mode); + expect(item.expected.forbidden_modes, item.id).not.toContain(decision.primary_mode); + } + }); + + it('covers seed modes including adversarial ownership cases', () => { + const ids = reconstructedRoutingCases.map((item) => item.id); + expect(ids).toEqual(expect.arrayContaining([ + 'recon-unoccupied-bug-build', + 'recon-missing-repro', + 'recon-explicit-closer', + 'recon-released-fix', + 'adv-closer-provider-error', + 'adv-title-overlap-only' + ])); + }); + + it('still parses legacy outcomes without contribution_mode', () => { + const parsed = OutcomeEventSchema.parse({ + event_version: 1, + event_id: 'e1', + decision_id: 'd1', + run_id: 'r1', + target: { repo: 'o/r', issue_number: 1 }, + event: 'selected', + occurred_at: '2026-08-01T00:00:00.000Z', + source: 'test', + data: {}, + notes: '' + }); + expect(parsed.contribution_mode).toBeUndefined(); + }); + + it('accepts additive routing ground truth', () => { + const parsed = EvalGroundTruthSchema.parse({ + verdict: 'SKIP', + disposition: 'land_only', + failure_mode: 'occupied_closer', + adjudicator_rationale: 'explicit closer owns the issue', + evidence_urls: ['https://example.com/issue'], + routing: { + primary_mode: 'REVIEW', + acceptable_modes: ['WATCH'], + forbidden_modes: ['BUILD'], + build_contention: 'RED' + } + }); + expect(parsed.routing?.forbidden_modes).toContain('BUILD'); + }); +});