From e9fb66caf365af3653d7e88852c84e633c178bfd Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:07:17 +0800 Subject: [PATCH 01/40] Complete Run 94 graph storage runtime wiring --- package.json | 1 + .../apps/runtime-host-bridge/src/cli.ts | 37 + .../apps/runtime-host-bridge/src/index.ts | 370 +++-- .../runtime-host-bridge/src/package-sea.ts | 24 +- .../src/track-b-operations.ts | 218 ++- .../src/track-b-runtime.ts | 1199 ++++++++++++++--- .../runtime-host-bridge/src/validate-tools.ts | 12 +- .../runtime-host-bridge/src/validate-ui.ts | 50 +- .../src/validate-vendors.ts | 28 +- .../test/candidate-profile-scaling.test.ts | 2 +- .../test/executable.test.ts | 34 + .../recursive-87-shadow-extension.mjs | 7 +- .../runtime-host-bridge/test/index.test.ts | 64 +- .../test/recursive-87-shadow-pipeline.test.ts | 152 ++- .../test/run88-public-runtime-probes.ts | 51 +- .../test/run94-local-graph-capture.test.ts | 29 + .../test/run94-sp5-sp10.test.ts | 388 ++++++ .../test/track-b-operations-api.test.ts | 269 +++- .../test/validate-tools.test.ts | 12 +- .../test/validate-vendors.test.ts | 24 +- .../apps/runtime-ui/app/lib/runtime-api.ts | 135 +- .../run94-storage-contract-red.test.tsx | 15 + .../app/routes/storage-retention.test.tsx | 7 +- .../app/routes/storage-retention.tsx | 61 +- ...oud-roundtrip.sp8.contributions-ui.spec.ts | 61 + ...aph-cloud-roundtrip.sp8.storage-ui.spec.ts | 56 + .../0002_compact_stub_enforcement.sql | 17 + role-model-router/migrations/registry.json | 10 + .../packages/process-supervisor/src/index.ts | 22 +- .../process-supervisor/test/index.test.ts | 38 + .../runtime-observability/src/index.ts | 4 - .../test/run94-bundle-compactness.test.ts | 68 + .../packages/sqlite-memory/src/index.ts | 116 +- .../sqlite-memory/src/legacy-migration.ts | 364 ++++- .../packages/sqlite-memory/test/index.test.ts | 2 +- .../test/legacy-migration.test.ts | 51 +- .../test/run94-compact-stub.test.ts | 253 ++++ .../run94-storage-corrections-red.test.ts | 115 ++ scripts/track-b/storage-audit.mjs | 34 + 39 files changed, 3868 insertions(+), 532 deletions(-) create mode 100644 role-model-router/apps/runtime-host-bridge/test/run94-local-graph-capture.test.ts create mode 100644 role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts create mode 100644 role-model-router/apps/runtime-ui/app/routes/run94-storage-contract-red.test.tsx create mode 100644 role-model-router/apps/runtime-ui/e2e/recursive-94-direct-track-b-storage-graph-cloud-roundtrip.sp8.contributions-ui.spec.ts create mode 100644 role-model-router/apps/runtime-ui/e2e/recursive-94-direct-track-b-storage-graph-cloud-roundtrip.sp8.storage-ui.spec.ts create mode 100644 role-model-router/migrations/0002_compact_stub_enforcement.sql create mode 100644 role-model-router/packages/runtime-observability/test/run94-bundle-compactness.test.ts create mode 100644 role-model-router/packages/sqlite-memory/test/run94-compact-stub.test.ts create mode 100644 role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts create mode 100644 scripts/track-b/storage-audit.mjs diff --git a/package.json b/package.json index cc453384..aa56673e 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "test:track-b-router-continuity": "node scripts/track-b/run-command-test.mjs TB03-CMD-02", "test:track-b-capacity": "node scripts/track-b/run-command-test.mjs TB03-CMD-03", "storage:audit": "node scripts/track-b/run-command-test.mjs TB04-CMD-01", + "storage:audit:read-only": "node scripts/track-b/storage-audit.mjs", "storage:migrate:dry-run": "node scripts/track-b/run-command-test.mjs TB04-CMD-02", "test:track-b-performance-history": "node scripts/track-b/run-command-test.mjs TB04-CMD-03", "storage:migrate:verify-parity": "node scripts/track-b/run-command-test.mjs TB04-CMD-04", diff --git a/role-model-router/apps/runtime-host-bridge/src/cli.ts b/role-model-router/apps/runtime-host-bridge/src/cli.ts index 059fd8c6..b3f62df6 100644 --- a/role-model-router/apps/runtime-host-bridge/src/cli.ts +++ b/role-model-router/apps/runtime-host-bridge/src/cli.ts @@ -21,6 +21,7 @@ import { type RuntimeChannelProfile, readPackagedRuntimeProfile } from "./runtim import { migrateLegacyProductionState } from "./runtime-state-migration.js"; import { resolveRun88StageRuntimeIdentity } from "./runtime-version.js"; import { + type TrackBExtensionClosure, createOwnedTrackBSidecarSpec, createPackagedProductionRuntime, createProductionExtensionRuntime, @@ -30,6 +31,7 @@ import { runTrackBPostObservation, trackBDistributionRequiresSQLiteMaintenance, validateRun88ProviderResponseObservation, + verifyTrackBExtensionClosureAfterRestart, } from "./track-b-runtime.js"; type CliBackend = Pick< @@ -57,6 +59,7 @@ type CliBackend = Pick< | "mutateExtension" | "readTrackBQaExtensions" | "readTrackBShadowReceipts" + | "readTrackBExtensionReadback" | "readGraphMigration" | "advanceGraphMigration" | "rollbackGraphMigration" @@ -98,6 +101,7 @@ type CliBackend = Pick< | "listRecentRequestIds" | "listRecentRequestObservations" | "readRequestObservation" + | "exportVerifiersTrace" | "readEndpointProfile" | "readBenchmarkSuite" | "runBenchmark" @@ -420,6 +424,9 @@ export function createCliServerOptions( readTrackBShadowReceipts: bindBackendMethod( "readTrackBShadowReceipts", ) as StartBridgeServerOptions["readTrackBShadowReceipts"], + readTrackBExtensionReadback: bindBackendMethod( + "readTrackBExtensionReadback", + ) as StartBridgeServerOptions["readTrackBExtensionReadback"], readGraphMigration: bindBackendMethod( "readGraphMigration", ) as StartBridgeServerOptions["readGraphMigration"], @@ -537,6 +544,9 @@ export function createCliServerOptions( readRequestObservation: bindBackendMethod( "readRequestObservation", ) as StartBridgeServerOptions["readRequestObservation"], + exportVerifiersTrace: bindBackendMethod( + "exportVerifiersTrace", + ) as StartBridgeServerOptions["exportVerifiersTrace"], readEndpointProfile: bindBackendMethod( "readEndpointProfile", ) as StartBridgeServerOptions["readEndpointProfile"], @@ -1013,6 +1023,33 @@ export async function main(): Promise { : {}), })), trackBPostObservationReceipts: () => postObservationOutbox.read(), + readTrackBExtensionReadback: async (body) => { + const requestId = String(body.requestId ?? "").trim(); + if (!requestId) throw new Error("Track B extension readback requestId is required"); + const receipt = await postObservationOutbox.readReceipt(requestId); + if (!receipt) throw new Error(`Track B observation receipt not found: ${requestId}`); + const result = receipt.result as Record; + const closure = result.extensionClosure as TrackBExtensionClosure | undefined; + if (!closure) + throw new Error(`Track B observation has no extension closure: ${requestId}`); + const runtime = extensionRuntimeRef.current; + if (!runtime) throw new Error("Track B extension runtime is unavailable"); + return verifyTrackBExtensionClosureAfterRestart(runtime, closure, { + channel: packagedProfile?.channel ?? "development", + scope: options.scopeId, + authorizationEpoch: 1, + readDurableEvidence: async ({ durableLocator, durableOutputId }) => + runtime.invoke("artifact-store", { + requestId: `${requestId}:readback:evidence:${durableOutputId}`, + protocolVersion: "1.1.0", + channel: packagedProfile?.channel ?? "development", + scope: options.scopeId, + authorizationEpoch: 1, + capability: "artifact:read", + payload: { durableLocator, durableOutputId }, + }), + }); + }, ...(trackBManifestText ? { trackBPostObservation: async (observation: Readonly>) => { diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index 35931b78..9d96fb1c 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -71,6 +71,7 @@ import { } from "@role-model-router/runtime-observability"; import { buildAdvisoryMaxDifficultyRecommendation, + buildCompactRuntimeObservationStub, clearAllObservedBenchmarkData, clearBenchmarkRunArtifacts, clearObservedBenchmarkDataForEndpoint, @@ -102,6 +103,7 @@ import { readRuntimeControllerAssignment, readRuntimeMaintenancePolicy, readRuntimeObservationBundle, + readRuntimeObservationStorageRecord, readRuntimeTelemetryRecord, readRuntimeTelemetrySourceSummaries, readRuntimeTelemetrySummary, @@ -162,8 +164,13 @@ import { } from "./request-capability-inference.js"; import { readPackagedRuntimeProfile, resolveRuntimeChannelProfile } from "./runtime-channel.js"; import { type RuntimeVersionInfoRecord, resolveRuntimeVersionInfo } from "./runtime-version.js"; -import { createTrackBOperations as createTrackBOperationsFromState } from "./track-b-operations.js"; -import { createRun88RuntimeCorrelation } from "./track-b-runtime.js"; +import { + buildGraphEvidenceFromCapture, + buildProviderEvidenceFromObservation, + buildVerifiersLiveExport, + createTrackBOperations as createTrackBOperationsFromState, +} from "./track-b-operations.js"; +import { createRun88RuntimeCorrelation, createTrackBFileGraphStore } from "./track-b-runtime.js"; import { type ProviderRequestCapture, @@ -2871,6 +2878,7 @@ export interface StartBridgeServerOptions { readonly mutateExtension?: (body: Record) => Promise; readonly readTrackBQaExtensions?: () => Promise; readonly readTrackBShadowReceipts?: () => Promise; + readonly readTrackBExtensionReadback?: (body: Record) => Promise; readonly readGraphMigration?: () => Promise; readonly advanceGraphMigration?: (body: Record) => Promise; readonly rollbackGraphMigration?: () => Promise; @@ -2933,6 +2941,7 @@ export interface StartBridgeServerOptions { ) => Promise; readonly subscribeTelemetry?: (listener: (event: RuntimeBridgeStreamEvent) => void) => () => void; readonly readRequestObservation?: (requestId: string) => Promise; + readonly exportVerifiersTrace?: (body: Record) => Promise; readonly readEndpointProfile?: (endpointId: string) => Promise; readonly readBenchmarkSuite?: () => Promise; readonly runBenchmark?: (body: Record) => Promise; @@ -3086,6 +3095,7 @@ export interface RuntimeBridgeBackend { mutateExtension(body: Record): Promise; readTrackBQaExtensions(): Promise; readTrackBShadowReceipts(): Promise; + readTrackBExtensionReadback(body: Record): Promise; readGraphMigration(): Promise; advanceGraphMigration(body: Record): Promise; rollbackGraphMigration(): Promise; @@ -3186,6 +3196,7 @@ export interface RuntimeBridgeBackend { queryTelemetryAnalytics(body: Record): Promise; subscribeTelemetry(listener: (event: RuntimeBridgeStreamEvent) => void): () => void; readRequestObservation(requestId: string): Promise; + exportVerifiersTrace(body: Record): Promise; readEndpointProfile(endpointId: string): Promise<{ endpointId: string; latestProfile: ReturnType; @@ -3428,6 +3439,13 @@ export interface CreateRuntimeBridgeBackendOptions { }; readonly unifiedRuntimeConfigPath?: string; readonly networkFetcher?: typeof fetch; + /** + * Credential values owned by this router runtime process. Callers such as Pi, validators, + * observers, and cloud verifiers must not populate the ambient process environment instead. + */ + readonly providerCredentialEnvironment?: Readonly>; + /** Base environment inherited by supervised vendor children. */ + readonly providerChildEnvironment?: Readonly>; readonly fixtureRoot?: string; readonly runtimeVendorStartup?: "enabled" | "disabled"; readonly trackBOperationsEndpoint?: string; @@ -3445,6 +3463,7 @@ export interface CreateRuntimeBridgeBackendOptions { observation: Readonly>, ) => Promise; readonly trackBPostObservationReceipts?: () => Promise; + readonly readTrackBExtensionReadback?: (body: Record) => Promise; readonly codexAuthAdapter?: CodexAuthAdapter; readonly codexExecutionAdapter?: CodexExecutionAdapter; } @@ -4602,7 +4621,6 @@ function buildPreExecutionFailureObservation(input: { observedPerformance: { endpointVersion: "pre-execution-failure", sample, - history: [sample], profile, }, diagnostics, @@ -4691,7 +4709,6 @@ function buildPreExecutionFailureObservation(input: { endpoint: { endpointId: input.endpointId, endpointVersion: "pre-execution-failure", - recentSamples: [], }, }, }; @@ -12591,6 +12608,7 @@ function isRecoveredOauthRuntimeAccount( function readEnvCredentialError( account: ProviderAccountRecord, ignoredAccountIds: ReadonlySet = new Set(), + environment: Readonly> = process.env, ): string | null { if (account.credentialRef.backend !== "env") { return null; @@ -12599,7 +12617,7 @@ function readEnvCredentialError( return null; } - const value = process.env[account.credentialRef.ref]; + const value = environment[account.credentialRef.ref]; if (typeof value === "string" && value.trim().length > 0) { return null; } @@ -12610,9 +12628,10 @@ function readEnvCredentialError( function hydrateEnvProviderAccounts( accounts: readonly ProviderAccountRecord[], ignoredAccountIds: ReadonlySet = new Set(), + environment: Readonly> = process.env, ): ProviderAccountRecord[] { return accounts.map((account) => { - const envCredentialError = readEnvCredentialError(account, ignoredAccountIds); + const envCredentialError = readEnvCredentialError(account, ignoredAccountIds, environment); if (!envCredentialError) { if ( account.authMode === "api-key-static" && @@ -12872,6 +12891,7 @@ async function resolveCredentialValue( networkFetcher?: typeof fetch, deviceId?: string, onRefreshed?: () => void, + credentialEnvironment: Readonly> = process.env, ): Promise { const credentialRef = target.account?.credentialRef; if (!credentialRef) { @@ -12879,7 +12899,7 @@ async function resolveCredentialValue( } if (credentialRef.backend === "env") { - const value = process.env[credentialRef.ref]; + const value = credentialEnvironment[credentialRef.ref]; if (typeof value === "string" && value.trim().length > 0) { return value.trim(); } @@ -14799,6 +14819,43 @@ function createRequestHandler(options: StartBridgeServerOptions) { return; } + if ( + request.method === "POST" && + url.pathname === "/api/role-model/track-b/extension-readback" + ) { + if (!options.readTrackBExtensionReadback) { + writeJson(response, 404, { error: "not found" }); + return; + } + try { + writeJson( + response, + 200, + await options.readTrackBExtensionReadback(await readJsonBody(request)), + ); + } catch (error) { + writeJson(response, 400, { + error: error instanceof Error ? error.message : String(error), + }); + } + return; + } + + if (request.method === "POST" && url.pathname === "/api/role-model/track-b/verifiers-export") { + if (!options.exportVerifiersTrace) { + writeJson(response, 404, { error: "not found" }); + return; + } + try { + writeJson(response, 200, await options.exportVerifiersTrace(await readJsonBody(request))); + } catch (error) { + writeJson(response, 409, { + error: error instanceof Error ? error.message : String(error), + }); + } + return; + } + if (request.method === "GET" && url.pathname === "/api/role-model/graph-migration") { if (!options.readGraphMigration) { writeJson(response, 404, { error: "not found" }); @@ -16427,6 +16484,7 @@ export async function createRuntimeBridgeBackend( operationsToken: options.trackBOperationsToken, }); const networkFetcher = options.networkFetcher ?? fetch; + const providerCredentialEnvironment = options.providerCredentialEnvironment ?? process.env; const codexAuthAdapter = options.codexAuthAdapter ?? createSystemCodexAuthAdapter(networkFetcher); const codexExecutionAdapter = options.codexExecutionAdapter ?? @@ -16505,7 +16563,9 @@ export async function createRuntimeBridgeBackend( "utf8", ); } - const supervisor = options.unifiedRuntimeConfigPath ? new ProcessSupervisor() : null; + const supervisor = options.unifiedRuntimeConfigPath + ? new ProcessSupervisor({ baseEnvironment: options.providerChildEnvironment }) + : null; const baseCatalog = await readNormalizedCatalogFile( path.join( options.repoRoot, @@ -16561,11 +16621,6 @@ export async function createRuntimeBridgeBackend( const captureFixtureMap = useFixtures ? await readJson(path.join(fixtureBasePath, "adapter-captures.json")) : { byEndpointId: {}, byRequestId: {} }; - const observabilityHistory = useFixtures - ? await readJson<{ - byEndpointId: Record; - }>(path.join(fixtureBasePath, "observability-history.json")) - : { byEndpointId: {} }; const observabilityPolicy = useFixtures ? await readJson(path.join(fixtureBasePath, "observability-policy.json")) : ({ captureMode: "none" } as RuntimeCapturePolicy); @@ -16594,6 +16649,48 @@ export async function createRuntimeBridgeBackend( scopeId: options.scopeId, channel: runtimeChannel, }); + const configuredTrackBOperationsEndpoint = + options.trackBOperationsEndpoint ?? process.env.ROLE_MODEL_TRACK_B_OPERATIONS_URL?.trim(); + const runtimeTrackBOperations = createTrackBOperations({ + statePath: path.join( + options.runtimeStateRoot, + options.scopeId, + "track-b-production-bridge.json", + ), + catalog: [], + }); + const readExactRouteCapture = async ( + requestId: string, + ): Promise | null> => { + if (!configuredTrackBOperationsEndpoint) return null; + return (await runtimeTrackBOperations.readLocalRouteCapture({ requestId })) as Record< + string, + unknown + >; + }; + const localGraphStore = + useFixtures && !configuredTrackBOperationsEndpoint + ? createTrackBFileGraphStore({ + scopeId: options.scopeId, + rootPath: path.join(options.runtimeStateRoot, options.scopeId, "track-b-graph"), + }) + : undefined; + const readPersistedRuntimeObservation = (requestId: string) => { + try { + return readRuntimeObservationBundle({ + databasePath: initialization.databasePath, + requestId, + ...(localGraphStore ? { graphStore: localGraphStore } : {}), + }); + } catch (error) { + if (!localGraphStore && /graph artifact reader required after cutover/i.test(String(error))) + return readRuntimeObservationStorageRecord({ + databasePath: initialization.databasePath, + requestId, + }); + throw error; + } + }; const restartCircuitState = readExecutionCircuitState(initialization.databasePath); const normalizedRestartCircuitState = normalizeExecutionCircuitStateForRestart( restartCircuitState, @@ -17101,7 +17198,11 @@ export async function createRuntimeBridgeBackend( const hydratedAccounts = hydrateOauthProviderAccounts( options.runtimeStateRoot, options.scopeId, - hydrateEnvProviderAccounts(validation.accounts, ignoredAccountIds), + hydrateEnvProviderAccounts( + validation.accounts, + ignoredAccountIds, + providerCredentialEnvironment, + ), ).map(normalizeCodexSubscriptionAccountTruth); const recoveredOauthAccountIds = new Set( hydratedAccounts.flatMap((account, index) => { @@ -17351,7 +17452,7 @@ export async function createRuntimeBridgeBackend( if (account.credentialRef?.backend !== "env") { return false; } - const value = process.env[account.credentialRef.ref]; + const value = providerCredentialEnvironment[account.credentialRef.ref]; return typeof value === "string" && value.trim().length > 0; }; @@ -17897,6 +17998,7 @@ export async function createRuntimeBridgeBackend( networkFetcher, deviceId, rebuildCurrentState, + providerCredentialEnvironment, ); } catch { return null; @@ -21424,10 +21526,10 @@ export async function createRuntimeBridgeBackend( return "/v1/chat/completions"; }; const readActivityCaptureByRequestId = (requestId: string): unknown | null => { - const observation = readRuntimeObservationBundle({ - databasePath: initialization.databasePath, - requestId, - }) as Record | null; + const observation = readPersistedRuntimeObservation(requestId) as Record< + string, + unknown + > | null; if (!observation) { return null; } @@ -21467,10 +21569,10 @@ export async function createRuntimeBridgeBackend( }); return recent .map((entry, index) => { - const observation = readRuntimeObservationBundle({ - databasePath: initialization.databasePath, - requestId: entry.requestId, - }) as Record | null; + const observation = readPersistedRuntimeObservation(entry.requestId) as Record< + string, + unknown + > | null; if (!observation) { return null; } @@ -21991,10 +22093,10 @@ export async function createRuntimeBridgeBackend( }); }; const toRouterDecisionData = (record: BridgeTelemetryRequestRecord) => { - const observation = readRuntimeObservationBundle({ - databasePath: initialization.databasePath, - requestId: record.requestId, - }) as Record | null; + const observation = readPersistedRuntimeObservation(record.requestId) as Record< + string, + unknown + > | null; const routingDiagnostics = asObjectRecord(observation?.routingDiagnostics); const routingMode = asObjectRecord(routingDiagnostics?.routingMode); const decision = asObjectRecord(observation?.decision); @@ -22087,10 +22189,9 @@ export async function createRuntimeBridgeBackend( } const readRouterDecisionData = (requestId: string) => { - const observation = readRuntimeObservationBundle({ - databasePath: initialization.databasePath, - requestId, - }) as (RuntimeObservationBundle & BridgeTelemetryEndpointMeta) | null; + const observation = readPersistedRuntimeObservation(requestId) as + | (RuntimeObservationBundle & BridgeTelemetryEndpointMeta) + | null; if (!observation) { return null; } @@ -22657,6 +22758,7 @@ export async function createRuntimeBridgeBackend( networkFetcher, deviceId, rebuildCurrentState, + providerCredentialEnvironment, ); const oauthVariant = (() => { if (!target.account || target.account.authMode !== "oauth2-device-code") return null; @@ -23137,7 +23239,6 @@ export async function createRuntimeBridgeBackend( endpoint: { endpointId: selectedEndpointId, endpointVersion: selectedCandidate?.identity.runtime_version ?? "unknown", - recentSamples: [], }, }, }; @@ -23212,6 +23313,7 @@ export async function createRuntimeBridgeBackend( structuredInspectionAvailable: capturePolicy.structuredInspectionAvailable, dimensions: selectedEndpointDimensions, observation: failureObservation, + ...(localGraphStore ? { graphStore: localGraphStore } : {}), }); markRuntimeTelemetryPersisted(error); emitTelemetryUpdate(requestId); @@ -23620,7 +23722,7 @@ export async function createRuntimeBridgeBackend( decisionPortfolio.entries.find( (entry) => entry.endpointId === routed.decision.chosen_endpoint_id, )?.profileRevision ?? null; - const bundle = createRuntimeObservationBundle({ + const baseBundle = createRuntimeObservationBundle({ decision: { ...routed.decision, membership_revision: decisionMembershipRevision, @@ -23666,13 +23768,6 @@ export async function createRuntimeBridgeBackend( estimatedTokenCount: envelope.estimatedTokenCount, }, execution, - priorSamples: [ - ...(observabilityHistory.byEndpointId[routed.decision.chosen_endpoint_id] ?? []), - ...readObservedPerformanceSamples({ - databasePath: initialization.databasePath, - endpointId: routed.decision.chosen_endpoint_id, - }), - ], maintenancePolicy: readRuntimeMaintenancePolicy({ databasePath: initialization.databasePath, }), @@ -23740,58 +23835,113 @@ export async function createRuntimeBridgeBackend( } : {}), }); + const run88Correlation = options.run88StageIdentity + ? createRun88RuntimeCorrelation({ + requestId, + routingDecisionId, + endpointId: execution.target.endpointId, + releaseId: options.run88StageIdentity.releaseId, + sourceId: options.run88StageIdentity.sourceId, + deploymentId: `local-${runtimeChannel}:${options.run88StageIdentity.executableSha256}`, + scope: options.scopeId, + }) + : undefined; + const bundle = Object.freeze({ + ...baseBundle, + providerEvidence: buildProviderEvidenceFromObservation( + baseBundle as unknown as Readonly>, + ), + ...(run88Correlation ? { run88Correlation } : {}), + }); let artifactRef: | { readonly scopeId: string; readonly artifactId: string; readonly contentHash: string } | undefined; + let routeCapture: Record | undefined; try { - const requestBody = executionOptions?.requestBody ?? {}; - const captureInput = Array.isArray(requestBody.messages) - ? requestBody.messages - : Array.isArray(requestBody.input) - ? requestBody.input - : []; - const capture = (await createTrackBOperations({ - statePath: path.join( - options.runtimeStateRoot, - options.scopeId, - "track-b-production-bridge.json", - ), - catalog: [], - }).recordLocalRouteCapture({ - requestId, - routingDecisionId, - endpointId: execution.target.endpointId, - modelId: execution.target.candidate.identity.model_id, - reasoningEffort: effectiveEffort.reasoningEffort, - effortSource: effectiveEffort.effortSource, - messages: captureInput, - outputText: execution.normalized.outputText, - toolExecutions: toolExecutionResult.executions, - })) as Record; - if ( - typeof capture.scope === "string" && - typeof capture.rootArtifactId === "string" && - typeof capture.rootArtifactDigest === "string" - ) { + if (localGraphStore) { + const content = JSON.stringify(bundle); + const contentHash = createHash("sha256").update(content).digest("hex"); + const artifact = localGraphStore.write({ + scopeId: localGraphStore.scopeId, + sourceId: requestId, + content, + contentHash, + }); artifactRef = { - scopeId: capture.scope, - artifactId: capture.rootArtifactId, - contentHash: capture.rootArtifactDigest, + scopeId: localGraphStore.scopeId, + artifactId: artifact.artifactId, + contentHash: artifact.contentHash, }; + } else { + const requestBody = executionOptions?.requestBody ?? {}; + const captureInput = Array.isArray(requestBody.messages) + ? requestBody.messages + : Array.isArray(requestBody.input) + ? requestBody.input + : []; + const capture = (await runtimeTrackBOperations.recordLocalRouteCapture({ + requestId, + routingDecisionId, + endpointId: execution.target.endpointId, + modelId: execution.target.candidate.identity.model_id, + reasoningEffort: effectiveEffort.reasoningEffort, + effortSource: effectiveEffort.effortSource, + messages: captureInput, + outputText: execution.normalized.outputText, + toolExecutions: toolExecutionResult.executions, + })) as Record; + routeCapture = capture; + if ( + typeof capture.scope === "string" && + typeof capture.rootArtifactId === "string" && + typeof capture.rootArtifactDigest === "string" + ) { + artifactRef = { + scopeId: capture.scope, + artifactId: capture.rootArtifactId, + contentHash: capture.rootArtifactDigest, + }; + } } } catch { // Capture remains non-routing-critical before graph-primary cutover. + // Run 94 (SP2): without a graph artifact reference the SQLite row must still be + // bounded — persist the compact degradation stub instead of the full bundle. } + const graphEvidence = routeCapture + ? { + rootArtifactId: routeCapture.rootArtifactId, + messageNodeIds: Array.isArray(routeCapture.messageArtifactIds) + ? routeCapture.messageArtifactIds + : [], + responseNodeId: routeCapture.responseArtifactId, + edgeCount: routeCapture.edgeCount, + } + : undefined; + const evidenceBundle = Object.freeze({ + ...bundle, + ...(graphEvidence ? { graphEvidence } : {}), + }); + const persistedObservation = artifactRef + ? evidenceBundle + : ({ + ...buildCompactRuntimeObservationStub( + evidenceBundle as unknown as Readonly>, + ), + statusFamily: "degraded-capture", + captureDegradation: { reason: "track-b-capture-unavailable" }, + } as never); persistRuntimeObservationBundle({ databasePath: initialization.databasePath, channel: runtimeChannel, - observation: bundle, + observation: persistedObservation, ...(artifactRef ? { artifactRef } : {}), + ...(localGraphStore ? { graphStore: localGraphStore } : {}), }); if (options.trackBPostObservation) { try { await options.trackBPostObservation( - bundle as unknown as Readonly>, + evidenceBundle as unknown as Readonly>, ); } catch (error) { console.error("Track B shadow post-observation processing failed", error); @@ -24314,6 +24464,7 @@ export async function createRuntimeBridgeBackend( dimensions, toolingUsed: Boolean(body.tools?.length), }), + ...(localGraphStore ? { graphStore: localGraphStore } : {}), }); }; try { @@ -25197,6 +25348,12 @@ export async function createRuntimeBridgeBackend( } return options.trackBPostObservationReceipts(); }, + async readTrackBExtensionReadback(body: Record): Promise { + if (!options.readTrackBExtensionReadback) { + throw new Error("Track B extension readback is unavailable"); + } + return options.readTrackBExtensionReadback(body); + }, async readGraphMigration(): Promise { return createTrackBOperations({ statePath: path.join( @@ -27094,6 +27251,30 @@ export async function createRuntimeBridgeBackend( }; }, async readRequestObservation(requestId: string): Promise { + const attachLiveEvidence = async ( + value: BridgeRequestObservation, + ): Promise => { + let providerEvidence: Readonly> | undefined; + let graphEvidence: Readonly> | undefined; + try { + providerEvidence = buildProviderEvidenceFromObservation( + value as unknown as Readonly>, + ); + } catch { + // Older telemetry-only rows may not have enough provider identity for attempt evidence. + } + try { + const capture = await readExactRouteCapture(requestId); + if (capture) graphEvidence = buildGraphEvidenceFromCapture(capture); + } catch { + // Rich capture is best effort. Absence remains visible as missing graph evidence. + } + return { + ...value, + ...(providerEvidence ? { providerEvidence } : {}), + ...(graphEvidence ? { graphEvidence } : {}), + } as BridgeRequestObservation; + }; const telemetryRecord = (() => { const record = readRuntimeTelemetryRecord({ databasePath: initialization.databasePath, @@ -27104,10 +27285,9 @@ export async function createRuntimeBridgeBackend( } return enrichTelemetryRequestRecords([record])[0] ?? null; })(); - const observation = readRuntimeObservationBundle({ - databasePath: initialization.databasePath, + const observation = readPersistedRuntimeObservation( requestId, - }) as RuntimeObservationBundle | null; + ) as RuntimeObservationBundle | null; if (!observation) { if (!telemetryRecord) { return null; @@ -27134,7 +27314,7 @@ export async function createRuntimeBridgeBackend( }, ] : []; - return { + const fallbackObservation = { requestId: telemetryRecord.requestId, routingDecisionId: telemetryRecord.routingDecisionId, endpointId: telemetryRecord.endpointId, @@ -27326,8 +27506,9 @@ export async function createRuntimeBridgeBackend( costBaselineSource: telemetryRecord.costBaselineSource, costSavingsSupport: telemetryRecord.costSavingsSupport, } as unknown as BridgeRequestObservation; + return attachLiveEvidence(fallbackObservation); } - return { + const requestDetail = { ...observation, ...getTelemetryEndpointMeta(observation.endpointId), observationAvailability: { @@ -27352,6 +27533,37 @@ export async function createRuntimeBridgeBackend( } : {}), } satisfies BridgeRequestObservation; + // Successful request details expose bounded routing/execution facts. Raw + // request/response inspection remains available through the dedicated + // activity-capture surface and is not copied into the normal observation + // response. Failure details retain their bounded error inspection so + // operators can diagnose rejected requests without reopening rich storage. + const observationRecord = observation as unknown as Readonly>; + const isFailureObservation = + observationRecord.statusFamily === "failure" || + Boolean(observationRecord.failure) || + telemetryRecord?.errorClass !== null; + if (!isFailureObservation) { + const { inspection: _inspection, ...boundedRequestDetail } = + requestDetail as BridgeRequestObservation & { inspection?: unknown }; + return attachLiveEvidence(boundedRequestDetail as unknown as BridgeRequestObservation); + } + return attachLiveEvidence(requestDetail); + }, + async exportVerifiersTrace(body: Record): Promise { + if (runtimeChannel === "production") + throw new Error("live Verifiers export is restricted to development and stage channels"); + const requestId = typeof body.requestId === "string" ? body.requestId : ""; + const observation = await this.readRequestObservation(requestId); + if (!observation) throw new Error("runtime observation not found for Verifiers export"); + const capture = await readExactRouteCapture(requestId); + if (!capture) throw new Error("exact live graph is unavailable for Verifiers export"); + return buildVerifiersLiveExport({ + channel: runtimeChannel, + request: body, + observation: observation as unknown as Readonly>, + capture, + }); }, async listRecentRequestObservations(): Promise< readonly ReturnType[number][] diff --git a/role-model-router/apps/runtime-host-bridge/src/package-sea.ts b/role-model-router/apps/runtime-host-bridge/src/package-sea.ts index c7498612..7e219714 100644 --- a/role-model-router/apps/runtime-host-bridge/src/package-sea.ts +++ b/role-model-router/apps/runtime-host-bridge/src/package-sea.ts @@ -86,6 +86,28 @@ export function resolvePackagedRuntimeSourceTree({ return sourceTree; } +export function resolveReleaseOutputDirectory({ + distRoot, + releaseTarget, + env, +}: Readonly<{ + distRoot: string; + releaseTarget: string; + env: NodeJS.ProcessEnv; +}>): string { + if (!/^[a-z0-9]+-[a-z0-9]+$/i.test(releaseTarget)) { + throw new Error("Packaged runtime release target is invalid"); + } + const explicitRoot = env.ROLE_MODEL_RELEASE_OUTPUT_ROOT?.trim(); + if (explicitRoot && !path.isAbsolute(explicitRoot)) { + throw new Error("ROLE_MODEL_RELEASE_OUTPUT_ROOT must be absolute"); + } + return path.join( + explicitRoot ? path.resolve(explicitRoot) : path.join(distRoot, "release"), + releaseTarget, + ); +} + const NODE_SEA_FUSE = "NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -643,7 +665,7 @@ export async function packageSeaRuntime(): Promise<{ await rm(seaConfigPath, { force: true }); } - const releaseDir = path.join(distRoot, "release", releaseTarget); + const releaseDir = resolveReleaseOutputDirectory({ distRoot, releaseTarget, env: process.env }); const outputPath = path.join(releaseDir, resolvePackagedRuntimeName(profile.name)); const blobPath = path.join(distRoot, "sea-prep.blob"); await rm(releaseDir, { recursive: true, force: true }); diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index dcf78bc0..16183f2a 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -117,6 +117,39 @@ type StorageRecord = { readonly leases?: number; readonly conflicts?: readonly string[]; }; + +function normalizeStorageRetentionContract(value: unknown): Record { + const raw = value && typeof value === "object" ? (value as Record) : {}; + const logicalClasses = Array.isArray(raw.logicalClasses) + ? raw.logicalClasses + : Array.isArray(raw.categories) + ? raw.categories + : []; + const inventory = + raw.storageInventory && typeof raw.storageInventory === "object" + ? (raw.storageInventory as Record) + : undefined; + const physicalResources = Array.isArray(raw.physicalResources) + ? raw.physicalResources + : Array.isArray(inventory?.entries) + ? inventory.entries + : []; + return { + ...raw, + logicalClasses, + categories: logicalClasses, + physicalResources, + ...(inventory + ? { + storageInventory: { + ...inventory, + complete: inventory.complete === true, + entries: physicalResources, + }, + } + : {}), + }; +} type RetentionPlan = { readonly schemaVersion: "role-model.retention-dry-run.v1"; readonly channel: string; @@ -597,6 +630,150 @@ const privateRetentionRequest = async ( return result; }; +function boundedIdentity(value: unknown, label: string): string { + if (typeof value !== "string" || value.length < 1 || value.length > 1024) + throw new Error(`${label} is required`); + return value; +} + +function recordValue(value: unknown): Readonly> { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Readonly>) + : {}; +} + +export function buildProviderEvidenceFromObservation( + observation: Readonly>, +): Readonly> { + const requestId = boundedIdentity(observation.requestId, "provider evidence request id"); + const endpointId = boundedIdentity(observation.endpointId, "provider evidence endpoint id"); + const modelId = boundedIdentity( + recordValue(observation.usageEvent).model_id ?? observation.modelId, + "provider evidence model id", + ); + const semantics = recordValue(observation.executionSemantics); + const failedAttempts = Array.isArray(semantics.failedAttempts) ? semantics.failedAttempts : []; + const failedAttemptIds = failedAttempts.map((attempt, index) => + boundedIdentity( + recordValue(attempt).attemptId ?? recordValue(attempt).routedAttemptId, + `provider failed attempt ${index + 1}`, + ), + ); + const failed = observation.statusFamily === "failure" || Boolean(observation.failure); + const attemptIds = failed + ? failedAttemptIds.length > 0 + ? failedAttemptIds + : [`${requestId}:attempt:1`] + : [...failedAttemptIds, `${requestId}:attempt:${failedAttemptIds.length + 1}`]; + return Object.freeze({ endpointId, modelId, status: failed ? "error" : "ok", attemptIds }); +} + +export function buildGraphEvidenceFromCapture( + capture: Readonly>, +): Readonly> { + const messages = Array.isArray(capture.messages) ? capture.messages : []; + const response = recordValue(capture.response); + const edgeCount = Number(capture.edgeCount); + if (messages.length < 1 || !Number.isSafeInteger(edgeCount) || edgeCount < 1) + throw new Error("exact live graph is incomplete"); + return Object.freeze({ + rootArtifactId: boundedIdentity(capture.rootArtifactId, "graph root artifact id"), + messageNodeIds: messages.map((message, index) => + boundedIdentity(recordValue(message).nodeId, `graph message node ${index + 1}`), + ), + responseNodeId: boundedIdentity(response.nodeId, "graph response node id"), + edgeCount, + }); +} + +export function buildVerifiersLiveExport(input: { + readonly channel: "development" | "stage"; + readonly request: Readonly>; + readonly observation: Readonly>; + readonly capture: Readonly>; +}): Readonly> { + const requestId = boundedIdentity(input.request.requestId, "Verifiers export request id"); + const correlationId = boundedIdentity( + input.request.correlationId, + "Verifiers export correlation id", + ); + const graphRootArtifactId = boundedIdentity( + input.request.graphRootArtifactId, + "Verifiers export graph root artifact id", + ); + if (input.request.readiness !== "semantic") + throw new Error( + "only semantic live Verifiers export is available without token-exact evidence", + ); + const observationCorrelation = boundedIdentity( + recordValue(input.observation.run88Correlation).correlationId, + "observation correlation id", + ); + if ( + input.capture.schemaVersion !== "role-model.route-capture-read.v1" || + input.observation.requestId !== requestId || + input.capture.requestId !== requestId || + input.capture.routingDecisionId !== input.observation.routingDecisionId || + observationCorrelation !== correlationId || + input.capture.rootArtifactId !== graphRootArtifactId + ) + throw new Error("Verifiers export does not reference the exact live graph and router decision"); + const messages = Array.isArray(input.capture.messages) ? input.capture.messages : []; + const response = recordValue(input.capture.response); + const semanticMessages = [...messages, response].map((value, index) => { + const message = recordValue(value); + const role = boundedIdentity(message.role, `Verifiers node ${index + 1} role`); + if (!("content" in message)) throw new Error(`Verifiers node ${index + 1} content is required`); + return { + parent: index === 0 ? null : index - 1, + message: { role, content: message.content }, + sampled: index === messages.length, + token_ids: [], + mask: [], + is_content: [], + logprobs: [], + ...(index === messages.length ? { finish_reason: "stop" } : {}), + }; + }); + const routingDecisionId = boundedIdentity( + input.observation.routingDecisionId, + "Verifiers export route decision id", + ); + const responseNodeId = boundedIdentity(response.nodeId, "Verifiers response node id"); + const traceId = `role-model-${createHash("sha256").update(`${input.channel}\0${requestId}\0${graphRootArtifactId}`).digest("hex")}`; + return Object.freeze({ + schemaVersion: "role-model.verifiers-live-export.v1", + channel: input.channel, + requestId, + correlationId, + graphRootArtifactId, + responseNodeIndex: messages.length, + tokenExactDisposition: "refused_missing_evidence", + trace: { + id: traceId, + task: { type: "RoleModelTraceTask", data: { requestId } }, + nodes: semanticMessages, + rewards: {}, + metrics: {}, + info: { + limitations: ["semantic projection; provider-native tokens unavailable"], + routeDecisionId: routingDecisionId, + roleModelGraphRootArtifactId: graphRootArtifactId, + roleModelResponseNodeId: responseNodeId, + roleModelRequestId: requestId, + roleModelCorrelationId: correlationId, + roleModelToolNodeIds: (Array.isArray(input.capture.tools) ? input.capture.tools : []).map( + (tool, index) => + boundedIdentity(recordValue(tool).nodeId, `graph tool node ${index + 1}`), + ), + }, + is_completed: true, + stop_condition: "role_model_graph_complete", + errors: [], + }, + }); +} + export function createTrackBOperations({ statePath, catalog, @@ -1076,7 +1253,12 @@ export function createTrackBOperations({ }, async readStorageRetention(): Promise { const remote = await requestPrivate("storage-retention"); - if (remote) return remote; + const storageAudit = await requestPrivate("storage-audit"); + if (remote) + return { + ...normalizeStorageRetentionContract(remote), + storageAudit: storageAudit ?? null, + }; const state = await readState(statePath); const categories = state.storageServices.map((row) => ({ id: row.category, @@ -1086,21 +1268,29 @@ export function createTrackBOperations({ count: row.count, serviceId: row.id, })); + const physicalResources = state.storageServices.map((row) => ({ + id: row.id, + owner: row.id, + health: "unavailable", + measurement: "unavailable" as const, + physicalBytes: null, + heldItems: row.holds ?? 0, + retentionState: "not_configured", + })); return { revision: state.revision, totalBytes: categories.reduce((sum, row) => sum + row.bytes, 0), categories, + logicalClasses: categories, + physicalResources, + // Run 94 SP8: the local fallback never fabricates a physical inventory. + // Physical bytes come exclusively from the read-only storage audit; without + // a measurement every entry is honestly unavailable. + storageAudit: storageAudit ?? null, storageInventory: { schemaVersion: "role-model.storage-registry.v1", - entries: state.storageServices.map((row) => ({ - id: row.id, - owner: row.id, - health: "unavailable", - measurement: "unavailable", - physicalBytes: null, - heldItems: row.holds ?? 0, - retentionState: row.conflicts?.length ? "blocked" : "not_configured", - })), + complete: false, + entries: physicalResources, }, managedPolicy: state.retention.managedPolicy, conflicts: state.storageServices.flatMap((row) => @@ -1348,6 +1538,14 @@ export function createTrackBOperations({ throw new Error("local route capture requires a loopback operations boundary"); return requestPrivate("capture/route", { method: "POST", body: input }); }, + async readLocalRouteCapture(input: Record): Promise { + if (!operationsEndpoint) + throw new Error("private operations endpoint is required for exact route capture readback"); + const url = new URL(operationsEndpoint); + if (!["127.0.0.1", "localhost", "::1", "[::1]"].includes(url.hostname)) + throw new Error("local route capture readback requires a loopback operations boundary"); + return requestPrivate("capture/read", { method: "POST", body: input }); + }, async listRecommendations(): Promise { return (await readState(statePath)).recommendations ?? []; }, diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts index 2917ead8..9903ed8e 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts @@ -1,13 +1,17 @@ import { spawn } from "node:child_process"; import { createHash, randomBytes, verify as verifySignature } from "node:crypto"; -import { existsSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { copyFile, lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { createInterface } from "node:readline"; +import { DatabaseSync } from "node:sqlite"; import { pathToFileURL } from "node:url"; import type { RuntimeEffortSource } from "@role-model-router/runtime-observability"; import { + type GraphArtifactReference, + type LegacyArtifactWriteInput, + type LegacyArtifactWriteResult, type LegacyMigrationState, type LegacySqliteMigration, type PersistRuntimeObservationBundleInput, @@ -73,6 +77,87 @@ export interface ManagedArtifactKeyFiles { readonly artifactEncryptionKeyFile?: string; } +/** + * Small local graph adapter for fixture and development runs that do not have + * the private operations sidecar configured. Rich observations live in these + * content-addressed files; SQLite stores only the bounded graph pointer. + */ +export function createTrackBFileGraphStore(input: { + readonly scopeId: string; + readonly rootPath: string; +}): RuntimeObservationGraphStore { + const scopeId = input.scopeId.trim(); + const rootPath = path.resolve(input.rootPath); + const artifactRoot = path.join(rootPath, "artifacts"); + if (!scopeId) throw new Error("local graph scope is required"); + + const assertScope = (candidate: string): void => { + if (candidate !== scopeId) throw new Error("local graph scope mismatch"); + }; + const assertPath = (candidate: string): string => { + const resolved = path.resolve(candidate); + const relative = path.relative(rootPath, resolved); + if (relative.startsWith("..") || path.isAbsolute(relative)) { + throw new Error("local graph artifact path escapes its root"); + } + return resolved; + }; + const artifactPathForDigest = (digest: string): string => + path.join(artifactRoot, `${digest}.json`); + + return { + scopeId, + write(artifact: LegacyArtifactWriteInput): LegacyArtifactWriteResult { + assertScope(artifact.scopeId); + const contentHash = createHash("sha256").update(artifact.content).digest("hex"); + const declaredHash = artifact.contentHash.replace(/^sha256:/, ""); + if (declaredHash !== contentHash) { + throw new Error("local graph content hash mismatch"); + } + mkdirSync(artifactRoot, { recursive: true }); + const artifactPath = artifactPathForDigest(contentHash); + if (existsSync(artifactPath)) { + if (readFileSync(artifactPath, "utf8") !== artifact.content) { + throw new Error("local graph artifact content conflicts with its digest"); + } + } else { + const temporaryPath = `${artifactPath}.${process.pid}.${Date.now()}.tmp`; + writeFileSync(temporaryPath, artifact.content, { encoding: "utf8", flag: "wx" }); + try { + try { + renameSync(temporaryPath, artifactPath); + } catch (error) { + if (!existsSync(artifactPath)) throw error; + } + } finally { + rmSync(temporaryPath, { force: true }); + } + } + return { + artifactId: `sha256:${contentHash}`, + artifactPath, + contentHash, + }; + }, + read(reference: GraphArtifactReference): string { + assertScope(reference.scopeId); + const artifactPath = assertPath( + reference.artifactPath ?? artifactPathForDigest(reference.contentHash), + ); + const content = readFileSync(artifactPath, "utf8"); + const contentHash = createHash("sha256").update(content).digest("hex"); + if (contentHash !== reference.contentHash.replace(/^sha256:/, "")) { + throw new Error("local graph artifact content hash mismatch"); + } + return content; + }, + remove(artifact: LegacyArtifactWriteResult): void { + assertScope(scopeId); + rmSync(assertPath(artifact.artifactPath), { force: true }); + }, + }; +} + async function pathExists(filePath: string): Promise { try { await lstat(filePath); @@ -856,10 +941,515 @@ export interface TrackBPostObservationReceipt { readonly result: unknown; } -interface TrackBPostObservationDurableState { - readonly schemaVersion: "role-model.track-b-post-observation-outbox.v2"; - readonly pending: TrackBPostObservationWorkItem[]; - readonly receipts: TrackBPostObservationReceipt[]; +export const TRACK_B_CANONICAL_EXTENSION_IDS = [ + "artifact-store", + "event-log", + "repository-context", + "background-evidence-scheduler", + "memory-store", + "knowledge-store", + "evaluation-core", + "crowdsourced-learning", + "replay-core", + "evaluation-runner-local", + "trajectory-signals", + "profile-learner", + "knowledge-worker", +] as const; + +export interface TrackBExtensionOutputRecord { + readonly extensionId: string; + readonly capability: string; + readonly requestId: string; + readonly workerPid: number; + readonly durableOutputId: string; + readonly durableLocator: unknown; + readonly evidenceRef: string | null; + readonly readCapability: string | null; + readonly resultDigest: string; +} + +export interface TrackBExtensionClosureEntry { + readonly extensionId: string; + readonly outputs: readonly TrackBExtensionOutputRecord[]; +} + +export interface TrackBExtensionClosure { + readonly schemaVersion: "role-model.track-b-extension-closure.v1"; + readonly requestId: string; + readonly routingDecisionId: string; + readonly scope: string; + readonly channel: string; + readonly authorizationEpoch: number; + readonly registry: Readonly>; +} + +function canonicalExtensionValue(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalExtensionValue); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.keys(value as Record) + .sort() + .map((key) => [key, canonicalExtensionValue((value as Record)[key])]), + ); + } + return value; +} + +function extensionResultRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function extensionEvidenceRef(result: Record): string | null { + if (typeof result.evidenceRef === "string" && result.evidenceRef.trim()) + return result.evidenceRef; + const provenance = extensionResultRecord(result.provenance); + return typeof provenance.evidenceRef === "string" && provenance.evidenceRef.trim() + ? provenance.evidenceRef + : null; +} + +function extensionDurableLocator(result: Record): unknown { + if (result.durableLocator !== undefined) return result.durableLocator; + if (result.artifactRef !== undefined) return result.artifactRef; + if (typeof result.id === "string" && result.id.trim()) return { id: result.id }; + if (typeof result.receiptId === "string" && result.receiptId.trim()) + return { receiptId: result.receiptId }; + return null; +} + +function buildExtensionOutputRecord( + extensionId: string, + envelope: Record, + result: unknown, +): TrackBExtensionOutputRecord { + const record = extensionResultRecord(result); + const workerPid = record.workerPid; + if (!Number.isInteger(workerPid) || Number(workerPid) < 1) + throw new Error(`extension ${extensionId} did not return an actual worker PID`); + const durableLocator = extensionDurableLocator(record); + const evidenceRef = extensionEvidenceRef(record); + const supervisorOnlyKeys = new Set([ + "workerPid", + "health", + "available", + "lifecycle", + "restarts", + "id", + "durableLocator", + "evidenceRef", + "readCapability", + "artifactRef", + "receiptId", + "resultDigest", + ]); + const meaningful = Object.fromEntries( + Object.entries(record).filter(([key]) => !supervisorOnlyKeys.has(key)), + ); + if (durableLocator === null || Object.keys(meaningful).length === 0) + throw new Error(`extension ${extensionId} returned no durable business output`); + const outputIdentity = canonicalExtensionValue({ + durableLocator, + evidenceRef, + businessOutput: record.businessOutput ?? meaningful, + }); + const durableOutputId = `sha256:${createHash("sha256").update(JSON.stringify(outputIdentity)).digest("hex")}`; + return { + extensionId, + capability: String(envelope.capability ?? ""), + requestId: String(envelope.requestId ?? ""), + workerPid: Number(workerPid), + durableOutputId, + durableLocator, + evidenceRef, + readCapability: + typeof record.readCapability === "string" && record.readCapability.trim() + ? record.readCapability + : null, + resultDigest: durableOutputId, + }; +} + +export interface TrackBExtensionReadbackRuntime { + listExtensions(): readonly unknown[] | Promise; + mutateExtension(input: Record): unknown | Promise; + invoke(id: string, envelope: Record): Promise>; +} + +export async function verifyTrackBExtensionClosureAfterRestart( + runtime: TrackBExtensionReadbackRuntime, + closure: TrackBExtensionClosure, + input: { + readonly channel: string; + readonly scope: string; + readonly authorizationEpoch: number; + readonly readDurableEvidence?: (input: { + readonly extensionId: string; + readonly durableLocator: unknown; + readonly durableOutputId: string; + }) => Promise; + }, +) { + if (closure?.schemaVersion !== "role-model.track-b-extension-closure.v1") + throw new Error("extension closure schema is invalid"); + const states = await runtime.listExtensions(); + const results: Array<{ + extensionId: string; + durableOutputId: string; + readbackOutputId: string; + preRestartPid: number; + postRestartPid: number; + }> = []; + for (const extensionId of Object.keys(closure.registry).sort()) { + const entry = closure.registry[extensionId]; + if (!entry || entry.extensionId !== extensionId || entry.outputs.length === 0) + throw new Error(`extension closure entry is incomplete for ${extensionId}`); + const state = states.find( + (candidate) => String((candidate as Record).id ?? "") === extensionId, + ) as Record | undefined; + const preRestartPid = Number(state?.pid); + if (!Number.isInteger(preRestartPid) || preRestartPid < 1) + throw new Error(`extension ${extensionId} has no live pre-restart PID`); + const revision = Number(state?.revision ?? 1); + const mutation = (await runtime.mutateExtension({ + id: extensionId, + action: "restart", + mutationId: `run94-readback:${closure.requestId}:${extensionId}`, + expectedRevision: revision, + })) as Record; + const mutationState = extensionResultRecord(mutation.state); + const postRestartPid = Number(mutationState.pid); + if (!Number.isInteger(postRestartPid) || postRestartPid < 1 || postRestartPid === preRestartPid) + throw new Error(`extension ${extensionId} restart did not produce a distinct worker PID`); + for (const output of entry.outputs) { + let readback: unknown; + if (output.readCapability) { + readback = await runtime.invoke(extensionId, { + requestId: `${closure.requestId}:readback:${output.durableOutputId}`, + protocolVersion: "1.1.0", + channel: input.channel, + scope: input.scope, + authorizationEpoch: input.authorizationEpoch, + capability: output.readCapability, + payload: { + durableLocator: output.durableLocator, + durableOutputId: output.durableOutputId, + evidenceRef: output.evidenceRef, + }, + }); + } else if (input.readDurableEvidence) { + readback = await input.readDurableEvidence({ + extensionId, + durableLocator: output.durableLocator, + durableOutputId: output.durableOutputId, + }); + } else { + throw new Error( + `extension ${extensionId} has no read capability or durable evidence reader`, + ); + } + const readbackRecord = extensionResultRecord(readback); + const readbackOutputId = String( + readbackRecord.readbackOutputId ?? readbackRecord.durableOutputId ?? "", + ); + if (readbackOutputId !== output.durableOutputId) + throw new Error(`extension ${extensionId} durable output readback mismatch`); + results.push({ + extensionId, + durableOutputId: output.durableOutputId, + readbackOutputId, + preRestartPid, + postRestartPid, + }); + } + } + return { + schemaVersion: "role-model.track-b-extension-readback.v1" as const, + requestId: closure.requestId, + outputs: results, + }; +} + +const TRACK_B_OUTBOX_SCHEMA_VERSION = "role-model.track-b-post-observation-outbox.v3" as const; +const TRACK_B_OUTBOX_RECEIPT_CAP_BYTES = 16 * 1024; +const TRACK_B_OUTBOX_SQLITE_HEADER = "SQLite format 3"; + +function boundedJson(value: unknown, capBytes = TRACK_B_OUTBOX_RECEIPT_CAP_BYTES): string { + const json = JSON.stringify(value ?? null); + if (Buffer.byteLength(json, "utf8") <= capBytes) return json; + const digest = createHash("sha256").update(json).digest("hex"); + return JSON.stringify({ + status: "bounded_receipt", + byteLength: Buffer.byteLength(json, "utf8"), + sha256: `sha256:${digest}`, + }); +} + +function outboxSchema(database: DatabaseSync): void { + database.exec(` + CREATE TABLE IF NOT EXISTS track_b_post_observation_pending ( + request_id TEXT PRIMARY KEY, + routing_decision_id TEXT NOT NULL, + endpoint_id TEXT NOT NULL, + model_id TEXT, + reasoning_effort TEXT, + effort_source TEXT, + run88_correlation_json TEXT, + legacy_identity_missing INTEGER NOT NULL DEFAULT 0, + enqueued_at_ms INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS track_b_post_observation_receipts ( + request_id TEXT PRIMARY KEY, + completed_at TEXT NOT NULL, + result_json TEXT NOT NULL, + completed_at_ms INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS track_b_post_observation_legacy_rows ( + source_kind TEXT NOT NULL, + source_index INTEGER NOT NULL, + source_id TEXT, + classification TEXT NOT NULL CHECK (classification IN ('imported', 'quarantined')), + reason TEXT, + source_hash TEXT NOT NULL, + classified_at_ms INTEGER NOT NULL, + PRIMARY KEY (source_kind, source_index) + ); + CREATE INDEX IF NOT EXISTS track_b_post_observation_pending_order + ON track_b_post_observation_pending(enqueued_at_ms, request_id); + CREATE TABLE IF NOT EXISTS track_b_post_observation_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + `); + database + .prepare( + "INSERT OR IGNORE INTO track_b_post_observation_meta (key, value) VALUES ('schemaVersion', ?)", + ) + .run(TRACK_B_OUTBOX_SCHEMA_VERSION); +} + +function sqliteHeader(bytes: Buffer): boolean { + return ( + bytes.subarray(0, TRACK_B_OUTBOX_SQLITE_HEADER.length).toString("utf8") === + TRACK_B_OUTBOX_SQLITE_HEADER + ); +} + +function legacyRowHash(row: unknown): string { + return `sha256:${createHash("sha256") + .update(JSON.stringify(row ?? null)) + .digest("hex")}`; +} + +function legacySourceId(row: unknown): string | null { + if (!row || typeof row !== "object" || Array.isArray(row)) return null; + const value = (row as Record).requestId; + return typeof value === "string" && value.trim() ? value : null; +} + +function insertLegacyClassification( + database: DatabaseSync, + input: { + readonly sourceKind: "pending" | "receipt"; + readonly sourceIndex: number; + readonly sourceId: string | null; + readonly classification: "imported" | "quarantined"; + readonly reason?: string; + readonly sourceHash: string; + }, +): void { + database + .prepare( + `INSERT INTO track_b_post_observation_legacy_rows + (source_kind, source_index, source_id, classification, reason, source_hash, classified_at_ms) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + input.sourceKind, + input.sourceIndex, + input.sourceId, + input.classification, + input.reason ?? null, + input.sourceHash, + Date.now(), + ); +} + +async function initializeTrackBPostObservationOutbox( + filePath: string, + maxItems: number, +): Promise { + await mkdir(path.dirname(filePath), { recursive: true }); + const legacyArchivePath = `${filePath}.n-1.json`; + const bytes = await readFile(filePath).catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + }); + if (bytes && sqliteHeader(bytes)) { + const database = new DatabaseSync(filePath); + try { + outboxSchema(database); + } finally { + database.close(); + } + return; + } + + let legacyValue: unknown = null; + let legacySourcePath: string | null = null; + if (bytes) { + try { + legacyValue = JSON.parse(bytes.toString("utf8")) as unknown; + } catch (error) { + throw new Error( + `Track B post-observation legacy JSON is malformed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + if ( + await readFile(legacyArchivePath) + .then(() => true) + .catch(() => false) + ) { + throw new Error("Track B post-observation legacy JSON archive already exists"); + } + await rename(filePath, legacyArchivePath); + legacySourcePath = legacyArchivePath; + } else if ( + await readFile(legacyArchivePath) + .then(() => true) + .catch(() => false) + ) { + try { + legacyValue = JSON.parse(await readFile(legacyArchivePath, "utf8")); + } catch (error) { + throw new Error( + `Track B post-observation legacy archive is malformed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + const legacyPending = Array.isArray(legacyValue) ? legacyValue : null; + const document = + !legacyPending && legacyValue && typeof legacyValue === "object" ? legacyValue : null; + const pending = legacyPending ?? (document as { pending?: unknown[] } | null)?.pending; + const receipts = (document as { receipts?: unknown[] } | null)?.receipts; + if (legacyValue !== null && (!Array.isArray(pending) || !Array.isArray(receipts ?? []))) { + throw new Error("Track B post-observation legacy JSON document is malformed"); + } + + const database = new DatabaseSync(filePath); + try { + outboxSchema(database); + database.exec("BEGIN IMMEDIATE"); + try { + let importedPendingCount = 0; + let importedReceiptCount = 0; + for (const [sourceIndex, raw] of (pending ?? []).entries()) { + const row = + raw && typeof raw === "object" && !Array.isArray(raw) + ? (raw as Record) + : {}; + const sourceId = legacySourceId(raw); + try { + const normalized = normalizeTrackBVariantIdentity(row); + if (!row.requestId || !row.routingDecisionId) throw new Error("identity incomplete"); + if (importedPendingCount >= maxItems) { + throw new Error("outbox capacity exceeded during legacy import"); + } + database + .prepare( + `INSERT OR IGNORE INTO track_b_post_observation_pending + (request_id, routing_decision_id, endpoint_id, model_id, reasoning_effort, effort_source, + run88_correlation_json, legacy_identity_missing, enqueued_at_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?)`, + ) + .run( + String(row.requestId), + String(row.routingDecisionId), + normalized.endpointId, + normalized.modelId, + normalized.reasoningEffort, + normalized.effortSource, + row.run88Correlation && typeof row.run88Correlation === "object" + ? boundedJson(row.run88Correlation) + : null, + Date.now() + sourceIndex, + ); + importedPendingCount += 1; + insertLegacyClassification(database, { + sourceKind: "pending", + sourceIndex, + sourceId, + classification: "imported", + sourceHash: legacyRowHash(raw), + }); + } catch (error) { + insertLegacyClassification(database, { + sourceKind: "pending", + sourceIndex, + sourceId, + classification: "quarantined", + reason: error instanceof Error ? error.message : String(error), + sourceHash: legacyRowHash(raw), + }); + } + } + for (const [sourceIndex, raw] of (receipts ?? []).entries()) { + const row = + raw && typeof raw === "object" && !Array.isArray(raw) + ? (raw as Record) + : {}; + const sourceId = legacySourceId(raw); + try { + if (!row.requestId || !row.completedAt || !("result" in row)) + throw new Error("receipt identity incomplete"); + if (importedReceiptCount >= maxItems) { + throw new Error("receipt capacity exceeded during legacy import"); + } + database + .prepare( + `INSERT OR IGNORE INTO track_b_post_observation_receipts + (request_id, completed_at, result_json, completed_at_ms) VALUES (?, ?, ?, ?)`, + ) + .run( + String(row.requestId), + String(row.completedAt), + boundedJson(row.result), + Date.now(), + ); + importedReceiptCount += 1; + insertLegacyClassification(database, { + sourceKind: "receipt", + sourceIndex, + sourceId, + classification: "imported", + sourceHash: legacyRowHash(raw), + }); + } catch (error) { + insertLegacyClassification(database, { + sourceKind: "receipt", + sourceIndex, + sourceId, + classification: "quarantined", + reason: error instanceof Error ? error.message : String(error), + sourceHash: legacyRowHash(raw), + }); + } + } + database.exec("COMMIT"); + } catch (error) { + database.exec("ROLLBACK"); + throw error; + } + } catch (error) { + database.close(); + if (legacySourcePath) await rename(legacySourcePath, filePath).catch(() => undefined); + await rm(filePath, { force: true }).catch(() => undefined); + throw error; + } + database.close(); } export function createTrackBPostObservationOutbox({ @@ -873,69 +1463,20 @@ export function createTrackBPostObservationOutbox({ throw new Error("valid Track B post-observation outbox configuration required"); } let operation = Promise.resolve(undefined); - const load = async (): Promise => { - const value = JSON.parse( - await readFile(filePath, "utf8").catch((error: unknown) => { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return "[]"; - throw error; - }), - ) as unknown; - const legacyPending = Array.isArray(value) ? value : null; - const document = !legacyPending && value && typeof value === "object" ? value : null; - const pending = legacyPending ?? (document as { pending?: unknown } | null)?.pending; - const receipts = (document as { receipts?: unknown } | null)?.receipts ?? []; - if (!Array.isArray(pending) || !Array.isArray(receipts)) { - throw new Error("Track B post-observation outbox is malformed"); - } - const normalizedPending = pending.map((item) => { - const row = item as Record; - if (!row.requestId || !row.routingDecisionId || !row.endpointId) { - throw new Error("Track B post-observation work item is incomplete"); - } - const base = { - requestId: String(row.requestId), - routingDecisionId: String(row.routingDecisionId), - endpointId: String(row.endpointId), - ...(row.run88Correlation && typeof row.run88Correlation === "object" - ? { - run88Correlation: normalizeRun88RuntimeCorrelation( - row.run88Correlation as Record, - String((row.run88Correlation as Record).releaseId ?? ""), - ), - } - : {}), - }; - const hasCompleteIdentity = - Object.hasOwn(row, "modelId") && - Object.hasOwn(row, "reasoningEffort") && - Object.hasOwn(row, "effortSource"); - if (!hasCompleteIdentity) { - return { ...base, legacyIdentityMissing: true as const }; - } - return { ...base, ...normalizeTrackBVariantIdentity(row) }; - }); - const normalizedReceipts = receipts.map((item) => { - const row = item as Record; - if (!row.requestId || !row.completedAt || !("result" in row)) { - throw new Error("Track B post-observation receipt is incomplete"); - } - return { - requestId: String(row.requestId), - completedAt: String(row.completedAt), - result: row.result, - }; - }); - return { - schemaVersion: "role-model.track-b-post-observation-outbox.v2", - pending: normalizedPending, - receipts: normalizedReceipts, - }; + let initialized: Promise | null = null; + const ensureInitialized = () => { + if (!initialized) initialized = initializeTrackBPostObservationOutbox(filePath, maxItems); + return initialized; }; - const persist = async (state: TrackBPostObservationDurableState) => { - await mkdir(path.dirname(filePath), { recursive: true }); - const temporary = `${filePath}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`; - await writeFile(temporary, `${JSON.stringify(state)}\n`, "utf8"); - await rename(temporary, filePath); + const withDatabase = async (run: (database: DatabaseSync) => T): Promise => { + await ensureInitialized(); + const database = new DatabaseSync(filePath); + try { + outboxSchema(database); + return run(database); + } finally { + database.close(); + } }; const exclusive = (run: () => Promise): Promise => { const result = operation.then(run, run); @@ -962,50 +1503,114 @@ export function createTrackBPostObservationOutbox({ if (!item.requestId || !item.routingDecisionId || !item.endpointId) { throw new Error("complete Track B post-observation identity required"); } - const state = await load(); - if ( - state.pending.some((existing) => existing.requestId === item.requestId) || - state.receipts.some((existing) => existing.requestId === item.requestId) - ) - return; - if (state.pending.length >= maxItems) - throw new Error("Track B post-observation outbox is full"); - await persist({ ...state, pending: [...state.pending, item] }); + await withDatabase((database) => { + database.exec("BEGIN IMMEDIATE"); + try { + const existing = database + .prepare( + "SELECT 1 AS found FROM track_b_post_observation_pending WHERE request_id=? UNION ALL SELECT 1 FROM track_b_post_observation_receipts WHERE request_id=? LIMIT 1", + ) + .get(item.requestId, item.requestId); + if (existing) { + database.exec("COMMIT"); + return; + } + const count = database + .prepare("SELECT COUNT(*) AS count FROM track_b_post_observation_pending") + .get() as { count: number }; + if (count.count >= maxItems) throw new Error("Track B post-observation outbox is full"); + database + .prepare( + `INSERT INTO track_b_post_observation_pending + (request_id, routing_decision_id, endpoint_id, model_id, reasoning_effort, effort_source, + run88_correlation_json, legacy_identity_missing, enqueued_at_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?)`, + ) + .run( + item.requestId, + item.routingDecisionId, + item.endpointId, + item.modelId, + item.reasoningEffort, + item.effortSource, + item.run88Correlation ? boundedJson(item.run88Correlation) : null, + Date.now(), + ); + database.exec("COMMIT"); + } catch (error) { + database.exec("ROLLBACK"); + throw error; + } + }); }); }, drain( handler: (observation: TrackBPostObservationWorkItem) => Promise, ): Promise { return exclusive(async () => { - const state = await load(); - while (state.pending.length) { - const item = state.pending[0]; + for (;;) { + const item = await withDatabase((database) => { + const row = database + .prepare( + `SELECT request_id, routing_decision_id, endpoint_id, model_id, reasoning_effort, + effort_source, run88_correlation_json, legacy_identity_missing + FROM track_b_post_observation_pending ORDER BY enqueued_at_ms, request_id LIMIT 1`, + ) + .get() as + | { + request_id: string; + routing_decision_id: string; + endpoint_id: string; + model_id: string | null; + reasoning_effort: string | null; + effort_source: RuntimeEffortSource | null; + run88_correlation_json: string | null; + legacy_identity_missing: number; + } + | undefined; + if (!row) return null; + return { + requestId: row.request_id, + routingDecisionId: row.routing_decision_id, + endpointId: row.endpoint_id, + ...(row.model_id !== null ? { modelId: row.model_id } : {}), + ...(row.reasoning_effort !== null ? { reasoningEffort: row.reasoning_effort } : {}), + ...(row.effort_source !== null ? { effortSource: row.effort_source } : {}), + ...(row.run88_correlation_json + ? { run88Correlation: JSON.parse(row.run88_correlation_json) } + : {}), + ...(row.legacy_identity_missing ? { legacyIdentityMissing: true as const } : {}), + } as TrackBPostObservationWorkItem; + }); if (!item) break; - if (item.legacyIdentityMissing) { - state.pending.shift(); - state.receipts.push({ - requestId: item.requestId, - completedAt: new Date().toISOString(), - result: { - status: "retired_legacy_missing_variant_identity", - productionMutation: false, - }, - }); - if (state.receipts.length > maxItems) - state.receipts.splice(0, state.receipts.length - maxItems); - await persist(state); - continue; - } - const result = await handler(item); - state.pending.shift(); - state.receipts.push({ - requestId: item.requestId, - completedAt: new Date().toISOString(), - result: result ?? null, + const result = item.legacyIdentityMissing + ? { status: "retired_legacy_missing_variant_identity", productionMutation: false } + : await handler(item); + await withDatabase((database) => { + database.exec("BEGIN IMMEDIATE"); + try { + database + .prepare("DELETE FROM track_b_post_observation_pending WHERE request_id=?") + .run(item.requestId); + database + .prepare( + `INSERT OR REPLACE INTO track_b_post_observation_receipts + (request_id, completed_at, result_json, completed_at_ms) VALUES (?, ?, ?, ?)`, + ) + .run(item.requestId, new Date().toISOString(), boundedJson(result), Date.now()); + database + .prepare( + `DELETE FROM track_b_post_observation_receipts + WHERE request_id NOT IN + (SELECT request_id FROM track_b_post_observation_receipts ORDER BY completed_at_ms DESC, request_id DESC LIMIT ?)`, + ) + .run(maxItems); + database.exec("COMMIT"); + } catch (error) { + database.exec("ROLLBACK"); + throw error; + } }); - if (state.receipts.length > maxItems) - state.receipts.splice(0, state.receipts.length - maxItems); - await persist(state); } }); }, @@ -1015,12 +1620,48 @@ export function createTrackBPostObservationOutbox({ readonly receipts: readonly TrackBPostObservationReceipt[]; }> { await operation; - const state = await load(); - return { - pendingCount: state.pending.length, - receiptCount: state.receipts.length, - receipts: structuredClone(state.receipts), - }; + return withDatabase((database) => { + const pendingCount = ( + database + .prepare("SELECT COUNT(*) AS count FROM track_b_post_observation_pending") + .get() as { + count: number; + } + ).count; + const rows = database + .prepare( + "SELECT request_id, completed_at, result_json FROM track_b_post_observation_receipts ORDER BY completed_at_ms, request_id", + ) + .all() as Array<{ request_id: string; completed_at: string; result_json: string }>; + return { + pendingCount, + receiptCount: rows.length, + receipts: rows.map((row) => ({ + requestId: row.request_id, + completedAt: row.completed_at, + result: JSON.parse(row.result_json), + })), + }; + }); + }, + async readReceipt(requestId: string): Promise { + await operation; + return withDatabase((database) => { + const row = database + .prepare( + "SELECT request_id, completed_at, result_json FROM track_b_post_observation_receipts WHERE request_id=?", + ) + .get(requestId) as + | { request_id: string; completed_at: string; result_json: string } + | undefined; + return row + ? { + requestId: row.request_id, + completedAt: row.completed_at, + result: JSON.parse(row.result_json), + } + : null; + }); }, }; } @@ -1036,7 +1677,8 @@ export interface TrackBShadowPipelineInput { readonly sourceGraphRef: string; readonly prefix: readonly unknown[]; readonly counterfactuals: readonly { readonly id: string; readonly suffix: readonly unknown[] }[]; - readonly evaluationCases: readonly { readonly expected: unknown; readonly actual: unknown }[]; + readonly comparableEvidence?: Readonly>; + readonly evaluationCases: readonly Record[]; readonly trajectoryEvents: readonly Record[]; readonly identity?: TrackBVariantIdentity; } @@ -1131,6 +1773,32 @@ export async function runTrackBShadowPipeline( if (!input.requestId || !input.scope || !input.routePackage) { throw new Error("complete shadow pipeline identity is required"); } + const comparableEvidence = input.comparableEvidence; + const sourceRollout = comparableEvidence?.source as Record | undefined; + const counterfactualRollouts = Array.isArray(comparableEvidence?.counterfactuals) + ? (comparableEvidence.counterfactuals as Record[]) + : []; + const candidateSet = Array.isArray(comparableEvidence?.candidateSet) + ? (comparableEvidence.candidateSet as Record[]) + : []; + if ( + !sourceRollout || + counterfactualRollouts.length < 1 || + candidateSet.length < 2 || + input.counterfactuals.length < 1 || + input.counterfactuals.every((counterfactual) => counterfactual.id === input.routePackage) || + counterfactualRollouts.some( + (rollout) => + rollout.rolloutId === sourceRollout.rolloutId || + (rollout.routePackage === sourceRollout.routePackage && + rollout.endpointId === sourceRollout.endpointId && + rollout.modelId === sourceRollout.modelId && + rollout.policyId === sourceRollout.policyId && + rollout.reasoningEffort === sourceRollout.reasoningEffort), + ) + ) { + throw new Error("R14_NO_DISTINCT_COUNTERFACTUAL: routing-shadow self-comparison is prohibited"); + } const envelope = (capability: string, value: unknown): Record => ({ requestId: `${input.requestId}:${capability}`, sessionId: input.requestId, @@ -1160,6 +1828,7 @@ export async function runTrackBShadowPipeline( split: "holdout", seed: 87, evidenceRef: input.sourceGraphRef, + comparableEvidence, cases: input.evaluationCases, }), scorerDefinitions: [scorer], @@ -1167,7 +1836,8 @@ export async function runTrackBShadowPipeline( const scores = Array.isArray(evaluation.scores) ? evaluation.scores.filter((score): score is number => Number.isFinite(score)) : []; - if (!scores.length || scores.reduce((sum, score) => sum + score, 0) / scores.length <= 0.5) { + const holdout = evaluation.holdout as Record | undefined; + if (!scores.length || holdout?.passed !== true || typeof holdout.evidenceRef !== "string") { throw new Error("shadow holdout evaluation failed"); } const signals = await runtime.invoke( @@ -1180,54 +1850,45 @@ export async function runTrackBShadowPipeline( ); const profile = await runtime.invoke("profile-learner", { ...envelope("profile:estimate", { - rows: input.evaluationCases.map((row, index) => ({ - model: "shadow-candidate", - endpoint: input.routePackage, + rows: [sourceRollout, ...counterfactualRollouts].map((rollout) => ({ + model: rollout.modelId, + endpoint: rollout.endpointId, prompt: "unchanged", tool: "unchanged", sampling: "deterministic", experience: "routing-evaluation", - routePackage: input.routePackage, - outcome: scores[index] ?? 0, - propensity: 1, - evidenceRef: `${input.sourceGraphRef}#case-${index}`, + routePackage: rollout.routePackage, + outcome: + (rollout.outcome as Record | undefined)?.status === "success" ? 1 : 0, + propensity: rollout.propensity, + evidenceRef: rollout.evidenceRef, })), }), - rows: input.evaluationCases.map((row, index) => ({ - model: "shadow-candidate", - endpoint: input.routePackage, + rows: [sourceRollout, ...counterfactualRollouts].map((rollout) => ({ + model: rollout.modelId, + endpoint: rollout.endpointId, prompt: "unchanged", tool: "unchanged", sampling: "deterministic", experience: "routing-evaluation", - routePackage: input.routePackage, - outcome: scores[index] ?? 0, - propensity: 1, - evidenceRef: `${input.sourceGraphRef}#case-${index}`, + routePackage: rollout.routePackage, + outcome: + (rollout.outcome as Record | undefined)?.status === "success" ? 1 : 0, + propensity: rollout.propensity, + evidenceRef: rollout.evidenceRef, })), }); - const positive = scores.flatMap((score, index) => - score > 0 - ? [ - { - id: `candidate-${index}`, - score, - evidenceRef: `${input.sourceGraphRef}#positive-${index}`, - }, - ] - : [], + const rolloutRows = [sourceRollout, ...counterfactualRollouts]; + const positive = rolloutRows.filter( + (rollout) => (rollout.outcome as Record | undefined)?.status === "success", ); - const negative = scores.flatMap((score, index) => - score <= 0 - ? [ - { - id: `baseline-${index}`, - score, - evidenceRef: `${input.sourceGraphRef}#negative-${index}`, - }, - ] - : [], + const negative = rolloutRows.filter( + (rollout) => (rollout.outcome as Record | undefined)?.status !== "success", ); + if (!positive.length || !negative.length) + throw new Error( + "R14_INSUFFICIENT_ROLLOUT_EVIDENCE: positive and negative rollout evidence is required", + ); const candidate = await runtime.invoke( "knowledge-worker", envelope("knowledge:eval-consumer", { @@ -1243,18 +1904,10 @@ export async function runTrackBShadowPipeline( seed: 87, comparabilityKey: `${input.sourceDecisionId}:holdout`, positive, - negative: negative.length - ? negative - : [{ id: "baseline-control", score: 0, evidenceRef: `${input.sourceGraphRef}#baseline` }], - }, - holdout: { - passed: true, - evidenceRef: String( - (evaluation.provenance && - (evaluation.provenance as Record).evidenceRef) || - input.sourceGraphRef, - ), + negative, + candidateSet, }, + holdout, scope: { routePackage: input.routePackage, channel: input.channel, scopeId: input.scope }, }), ); @@ -1276,6 +1929,101 @@ export async function runTrackBShadowPipeline( }; } +async function runTrackBObservationPipeline( + runtime: TrackBShadowPipelineRuntime, + input: { + readonly requestId: string; + readonly channel: string; + readonly scope: string; + readonly authorizationEpoch: number; + readonly productionState: Readonly>; + readonly routePackage: string; + readonly sourceDecisionId: string; + readonly sourceGraphRef: string; + readonly trajectoryEvents: readonly Record[]; + readonly identity: TrackBVariantIdentity; + }, +) { + const envelope = (capability: string, value: unknown): Record => ({ + requestId: `${input.requestId}:${capability}`, + sessionId: input.requestId, + protocolVersion: "1.1.0", + channel: input.channel, + scope: input.scope, + authorizationEpoch: input.authorizationEpoch, + capability, + identity: input.identity, + value, + }); + const replay = await runtime.invoke( + "replay-core", + envelope("replay:plan-graph", { + sourceDecisionId: input.sourceDecisionId, + sourceGraphRef: input.sourceGraphRef, + prefix: [{ routingDecisionId: input.sourceDecisionId, identity: input.identity }], + counterfactuals: [], + disposition: "observation_only_no_distinct_counterfactual", + }), + ); + const scorer = { id: "run94-observation", version: "1", algorithm: "exact_match" }; + const evaluationCase = { expected: input.routePackage, actual: input.routePackage }; + const evaluation = await runtime.invoke("evaluation-runner-local", { + ...envelope("evaluation:run-local", { + policy: "routing-observation", + task: "route-selection-observation", + scorer: `${scorer.id}@${scorer.version}`, + split: "observed", + seed: 94, + evidenceRef: input.sourceGraphRef, + cases: [evaluationCase], + }), + scorerDefinitions: [scorer], + }); + const signals = await runtime.invoke( + "trajectory-signals", + envelope("signals:analyze", { + routeDecisionId: input.sourceDecisionId, + graphRef: input.sourceGraphRef, + events: input.trajectoryEvents, + }), + ); + const profileRows = [ + { + model: input.identity.modelId, + endpoint: input.identity.endpointId, + prompt: "unchanged", + tool: "unchanged", + sampling: "observed", + experience: "none", + routePackage: input.routePackage, + outcome: 1, + propensity: 1, + evidenceRef: input.sourceGraphRef, + }, + ]; + const profile = await runtime.invoke("profile-learner", { + ...envelope("profile:estimate", { rows: profileRows }), + rows: profileRows, + }); + return { + replay, + evaluation, + signals, + profile, + productionState: structuredClone(input.productionState), + receipt: { + schemaVersion: "role-model.track-b-shadow-pipeline-receipt.v1", + mode: "shadow", + status: "insufficient_comparable_evidence", + refusalCode: "R14_NO_DISTINCT_COUNTERFACTUAL", + requestId: input.requestId, + providerCalls: 0, + productionMutation: false, + candidateId: null, + }, + }; +} + /** Normal post-observation owner for the shadow DAG and its supervised derived consumers. */ export async function runTrackBPostObservation( runtime: TrackBShadowPipelineRuntime, @@ -1313,7 +2061,24 @@ export async function runTrackBPostObservation( ...(run88Correlation ? { run88Correlation } : {}), ...extra, }); - const artifact = await runtime.invoke( + const closureEntries = new Map(); + const durableOutputIds = new Set(); + const observedRuntime: TrackBShadowPipelineRuntime = { + async invoke(id, envelope) { + const result = await runtime.invoke(id, envelope); + const output = buildExtensionOutputRecord(id, envelope, result); + if (durableOutputIds.has(output.durableOutputId)) + throw new Error(`duplicate durable extension output ${output.durableOutputId}`); + durableOutputIds.add(output.durableOutputId); + const prior = closureEntries.get(id); + closureEntries.set(id, { + extensionId: id, + outputs: [...(prior?.outputs ?? []), output], + }); + return result; + }, + }; + const artifact = await observedRuntime.invoke( "artifact-store", businessEnvelope("graph:write", { payload: { @@ -1327,7 +2092,7 @@ export async function runTrackBPostObservation( }), ); const artifactRef = String(artifact.id ?? `observation:${requestId}`); - await runtime.invoke( + await observedRuntime.invoke( "event-log", businessEnvelope("event:append", { payload: { @@ -1339,7 +2104,7 @@ export async function runTrackBPostObservation( }, }), ); - const repositoryContextResult = await runtime.invoke( + const repositoryContextResult = await observedRuntime.invoke( "repository-context", businessEnvelope("repository:read", { payload: { scopeId: input.scope, canonicalIdentity: input.scope, identity }, @@ -1389,14 +2154,14 @@ export async function runTrackBPostObservation( ) { throw new Error("repository-context returned an invalid privacy-safe receipt"); } - await runtime.invoke( + await observedRuntime.invoke( "background-evidence-scheduler", businessEnvelope("scheduler:schedule-and-run", { jobId: `post-observation:${requestId}`, payload: { requestId, sourceDecisionId, artifactRef, identity }, }), ); - await runtime.invoke( + await observedRuntime.invoke( "memory-store", businessEnvelope("memory:write", { payload: { @@ -1404,7 +2169,7 @@ export async function runTrackBPostObservation( }, }), ); - const knowledge = await runtime.invoke( + const knowledge = await observedRuntime.invoke( "knowledge-store", businessEnvelope("knowledge:write", { payload: { @@ -1419,13 +2184,13 @@ export async function runTrackBPostObservation( }, }), ); - await runtime.invoke( + await observedRuntime.invoke( "knowledge-store", businessEnvelope("knowledge:read", { payload: { id: knowledge.id, scope: input.scope, identity }, }), ); - await runtime.invoke( + await observedRuntime.invoke( "crowdsourced-learning", businessEnvelope("aggregate:preview", { input: { @@ -1441,28 +2206,62 @@ export async function runTrackBPostObservation( ); const sourceHash = createHash("sha256").update(JSON.stringify(observation)).digest("hex"); const sourceGraphRef = `sha256:${sourceHash}`; - const pipeline = await runTrackBShadowPipeline(runtime, { - requestId, - channel: input.channel, - scope: input.scope, - authorizationEpoch: input.authorizationEpoch, - productionState: { - routingDecisionId: sourceDecisionId, - endpointId: routePackage, - identity, - immutable: true, - }, - routePackage, - sourceDecisionId, - sourceGraphRef, - prefix: [{ routingDecisionId: sourceDecisionId, identity }], - counterfactuals: [{ id: routePackage, suffix: [{ endpointId: routePackage, identity }] }], - evaluationCases: [{ expected: routePackage, actual: routePackage }], - trajectoryEvents: [ - { requestId, routingDecisionId: sourceDecisionId, endpointId: routePackage, identity }, - ], + const productionState = { + routingDecisionId: sourceDecisionId, + endpointId: routePackage, identity, - }); + immutable: true, + } as const; + const trajectoryEvents = [ + { requestId, routingDecisionId: sourceDecisionId, endpointId: routePackage, identity }, + ]; + const routingShadowEvidence = + observation.routingShadowEvidence && + typeof observation.routingShadowEvidence === "object" && + !Array.isArray(observation.routingShadowEvidence) + ? (observation.routingShadowEvidence as Readonly>) + : null; + const routingShadowCases = Array.isArray(observation.routingShadowCases) + ? observation.routingShadowCases.filter((item): item is Record => + Boolean(item && typeof item === "object" && !Array.isArray(item)), + ) + : []; + const comparableCounterfactuals = Array.isArray(routingShadowEvidence?.counterfactuals) + ? (routingShadowEvidence.counterfactuals as Record[]) + : []; + const pipeline = + routingShadowEvidence && routingShadowCases.length > 0 + ? await runTrackBShadowPipeline(observedRuntime, { + requestId, + channel: input.channel, + scope: input.scope, + authorizationEpoch: input.authorizationEpoch, + productionState, + routePackage, + sourceDecisionId, + sourceGraphRef, + prefix: [{ routingDecisionId: sourceDecisionId, identity }], + counterfactuals: comparableCounterfactuals.map((rollout) => ({ + id: String(rollout.routePackage ?? ""), + suffix: [{ endpointId: rollout.endpointId, modelId: rollout.modelId }], + })), + comparableEvidence: routingShadowEvidence, + evaluationCases: routingShadowCases, + trajectoryEvents, + identity, + }) + : await runTrackBObservationPipeline(observedRuntime, { + requestId, + channel: input.channel, + scope: input.scope, + authorizationEpoch: input.authorizationEpoch, + productionState, + routePackage, + sourceDecisionId, + sourceGraphRef, + trajectoryEvents, + identity, + }); const projection = createProjectionV2({ scope: input.scope, purpose: "routing_shadow", @@ -1484,15 +2283,39 @@ export async function runTrackBPostObservation( routePackage, sourceDecisionId, identity, - candidateId: typeof pipeline.candidate.id === "string" ? pipeline.candidate.id : null, + candidateId: + "candidate" in pipeline && typeof pipeline.candidate?.id === "string" + ? pipeline.candidate.id + : null, }, }); - const consumption = await consumeTrackBProjection(runtime, projection, { + const consumption = await consumeTrackBProjection(observedRuntime, projection, { channel: input.channel, authorizationEpoch: input.authorizationEpoch, identity: { ...identity }, }); - return { pipeline: pipeline.receipt, projection, consumption, repositoryContext }; + const registry = Object.fromEntries( + [...closureEntries.entries()].sort(([left], [right]) => left.localeCompare(right)), + ) as Record; + const missing = TRACK_B_CANONICAL_EXTENSION_IDS.filter((id) => !registry[id]); + if (missing.length) + throw new Error(`extension closure is missing registry outputs: ${missing.join(", ")}`); + const extensionClosure: TrackBExtensionClosure = { + schemaVersion: "role-model.track-b-extension-closure.v1", + requestId, + routingDecisionId: sourceDecisionId, + scope: input.scope, + channel: input.channel, + authorizationEpoch: input.authorizationEpoch, + registry, + }; + return { + pipeline: pipeline.receipt, + projection, + consumption, + repositoryContext, + extensionClosure, + }; } interface ExtensionRuntimeState { diff --git a/role-model-router/apps/runtime-host-bridge/src/validate-tools.ts b/role-model-router/apps/runtime-host-bridge/src/validate-tools.ts index cfc80671..a9efa768 100644 --- a/role-model-router/apps/runtime-host-bridge/src/validate-tools.ts +++ b/role-model-router/apps/runtime-host-bridge/src/validate-tools.ts @@ -3,6 +3,10 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import type { RuntimeObservationBundle } from "@role-model-router/runtime-observability"; +import { + readRuntimeObservationStorageRecord, + resolveSqliteMemoryLocation, +} from "@role-model-router/sqlite-memory"; import type { ToolRegistryExecution } from "@role-model-router/tool-registry"; import { @@ -102,7 +106,13 @@ export async function runRuntimeToolsValidation( requestId, )) as BridgeChatCompletionsExecutionResult; const observation = requireObservation( - await backend.readRequestObservation(requestId), + readRuntimeObservationStorageRecord({ + databasePath: resolveSqliteMemoryLocation({ + runtimeStateRoot: options.runtimeStateRoot, + scopeId: options.scopeId, + }), + requestId, + }) as RuntimeObservationBundle | null, requestId, ); diff --git a/role-model-router/apps/runtime-host-bridge/src/validate-ui.ts b/role-model-router/apps/runtime-host-bridge/src/validate-ui.ts index 5d3cd957..7f92ea8f 100644 --- a/role-model-router/apps/runtime-host-bridge/src/validate-ui.ts +++ b/role-model-router/apps/runtime-host-bridge/src/validate-ui.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { readFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -17,6 +18,7 @@ import { mapChatCompletionsRequest, startBridgeServer, } from "./index.js"; +import { createTrackBFileGraphStore } from "./track-b-runtime.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -135,6 +137,36 @@ export async function runRuntimeUiValidation( ...options, runtimeVendorStartup: "disabled", }); + const localGraphStore = createTrackBFileGraphStore({ + scopeId: options.scopeId, + rootPath: path.join(options.runtimeStateRoot, options.scopeId, "track-b-graph"), + }); + const persistValidationObservation = ( + observation: ReturnType, + ): void => { + const content = JSON.stringify(observation); + const contentHash = createHash("sha256").update(content).digest("hex"); + const artifact = localGraphStore.write({ + scopeId: localGraphStore.scopeId, + sourceId: observation.decision.request_id, + content, + contentHash, + }); + persistRuntimeObservationBundle({ + databasePath: resolveSqliteMemoryLocation({ + runtimeStateRoot: options.runtimeStateRoot, + scopeId: options.scopeId, + }), + channel: "development", + observation: observation as never, + artifactRef: { + scopeId: localGraphStore.scopeId, + artifactId: artifact.artifactId, + contentHash: artifact.contentHash, + }, + graphStore: localGraphStore, + }); + }; traceValidation("createRuntimeBridgeBackend:done"); traceValidation("startBridgeServer:start"); const server = await startBridgeServer({ @@ -543,14 +575,7 @@ export async function runRuntimeUiValidation( rotationState: "stable", }, }); - persistRuntimeObservationBundle({ - databasePath: resolveSqliteMemoryLocation({ - runtimeStateRoot: options.runtimeStateRoot, - scopeId: options.scopeId, - }), - channel: "development", - observation: routedObservation, - }); + persistValidationObservation(routedObservation); traceValidation("routed-observation:persisted"); const mixedAliasRoutingDecisionId = "route-runtime-ui-mixed-alias-001"; @@ -662,14 +687,7 @@ export async function runRuntimeUiValidation( rotationState: "stable", }, }); - persistRuntimeObservationBundle({ - databasePath: resolveSqliteMemoryLocation({ - runtimeStateRoot: options.runtimeStateRoot, - scopeId: options.scopeId, - }), - channel: "development", - observation: mixedAliasObservation, - }); + persistValidationObservation(mixedAliasObservation); traceValidation("mixed-alias-observation:persisted"); traceValidation("final-readback:start"); diff --git a/role-model-router/apps/runtime-host-bridge/src/validate-vendors.ts b/role-model-router/apps/runtime-host-bridge/src/validate-vendors.ts index ee207e26..dff08f12 100644 --- a/role-model-router/apps/runtime-host-bridge/src/validate-vendors.ts +++ b/role-model-router/apps/runtime-host-bridge/src/validate-vendors.ts @@ -16,6 +16,17 @@ import { const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); +const VALIDATION_PROVIDER_CREDENTIAL_ENV = Object.freeze({ + OPENAI_API_KEY: "run94-validator-runtime-only", +}); +const PROVIDER_CREDENTIAL_ENV_SUFFIX = + /(?:API_KEY|API_TOKEN|AUTH_TOKEN|ACCESS_TOKEN|REFRESH_TOKEN|CLIENT_SECRET|SECRET_KEY|TOKEN)$/i; + +function createCredentialStrippedEnvironment(): NodeJS.ProcessEnv { + return Object.fromEntries( + Object.entries(process.env).filter(([name]) => !PROVIDER_CREDENTIAL_ENV_SUFFIX.test(name)), + ); +} function createLocalVendorScript(): string { return `const http=require("node:http");const port=Number(process.env.PORT??process.argv[2]);const server=http.createServer((req,res)=>{if(req.url==="/health"){res.statusCode=200;res.end("ok");return;}if(req.url==="/v1/responses"){let body="";req.on("data",chunk=>body+=chunk);req.on("end",()=>{const parsed=JSON.parse(body||"{}");const joinedInput=typeof parsed.input==="string"?parsed.input:JSON.stringify(parsed.input??"");const isClassifier=joinedInput.includes("ROLE_MODEL_DIFFICULTY_CLASSIFIER");const isHardPrompt=joinedInput.includes("Analyze this code-edit workflow")||joinedInput.includes('\"toolCount\":2')||joinedInput.includes('\"toolCount\": 2')||joinedInput.includes('\"codeOrSchemaBurden\":true')||joinedInput.includes('\"codeOrSchemaBurden\": true');const classifierResponse=isHardPrompt?JSON.stringify({difficulty:"hard"}):JSON.stringify({difficulty:"easy"});if(parsed.stream){res.writeHead(200,{"content-type":"text/event-stream; charset=utf-8"});res.write('data: {"type":"response.created","response":{"id":"resp-local","created_at":1,"model":"local/llama-3.1-8b-instruct"}}'+"\\n\\n");setTimeout(()=>{res.write('data: {"type":"response.output_text.delta","item_id":"msg_1","delta":'+JSON.stringify(isClassifier?classifierResponse:"local llama summary")+'}'+"\\n\\n");setTimeout(()=>{res.end('data: {"type":"response.completed","response":{"usage":{"input_tokens":11,"output_tokens":4}},"_hidden_params":{"response_cost":0.0005,"cache_hit":false}}'+"\\n\\n"+'data: [DONE]'+"\\n\\n");},10);},10);return;}res.setHeader("content-type","application/json");res.end(JSON.stringify({id:"resp-local",output:[{type:"message",role:"assistant",content:[{type:"output_text",text:isClassifier?classifierResponse:"local llama summary"}]}],usage:{input_tokens:11,output_tokens:4},_hidden_params:{response_cost:0.0005,cache_hit:false}}));});return;}if(req.url==="/v1/chat/completions"){let body="";req.on("data",chunk=>body+=chunk);req.on("end",()=>{const parsed=JSON.parse(body||"{}");const joinedMessages=JSON.stringify(parsed.messages??[]);const isClassifier=joinedMessages.includes("ROLE_MODEL_DIFFICULTY_CLASSIFIER");const isHardPrompt=joinedMessages.includes("Analyze this code-edit workflow")||joinedMessages.includes('\"toolCount\":2')||joinedMessages.includes('\"toolCount\": 2')||joinedMessages.includes('\"codeOrSchemaBurden\":true')||joinedMessages.includes('\"codeOrSchemaBurden\": true');const classifierResponse=isHardPrompt?JSON.stringify({difficulty:"hard"}):JSON.stringify({difficulty:"easy"});if(parsed.stream){res.writeHead(200,{"content-type":"text/event-stream; charset=utf-8"});res.write('data: {"id":"chat-local","object":"chat.completion.chunk","created":1,"model":"local/llama-3.1-8b-instruct","choices":[{"index":0,"delta":{"role":"assistant","content":"local "},"finish_reason":null}]}'+"\\n\\n");setTimeout(()=>{res.write('data: {"id":"chat-local","object":"chat.completion.chunk","created":1,"model":"local/llama-3.1-8b-instruct","choices":[{"index":0,"delta":{"content":"llama summary"},"finish_reason":null}]}'+"\\n\\n");setTimeout(()=>{res.end('data: {"id":"chat-local","object":"chat.completion.chunk","created":1,"model":"local/llama-3.1-8b-instruct","choices":[{"index":0,"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":11,"completion_tokens":4},"_hidden_params":{"response_cost":0.0005,"cache_hit":false}}'+"\\n\\n"+'data: [DONE]'+"\\n\\n");},10);},10);return;}res.setHeader("content-type","application/json");res.end(JSON.stringify({id:"chat-local",object:"chat.completion",choices:[{index:0,message:{role:"assistant",content:isClassifier?classifierResponse:"local llama summary"},finish_reason:"stop"}],usage:{prompt_tokens":11,completion_tokens":4,total_tokens:15},_hidden_params:{response_cost:0.0005,cache_hit:false}}));});return;}res.statusCode=404;res.end("missing");});server.listen(port,"127.0.0.1");const shutdown=()=>server.close(()=>process.exit(0));process.on("SIGTERM",shutdown);process.on("SIGINT",shutdown);`; @@ -1343,7 +1354,7 @@ async function startRemoteUpstreamProcess(input: { const child = spawn(process.execPath, [input.scriptPath, String(input.port)], { stdio: "ignore", env: { - ...process.env, + ...createCredentialStrippedEnvironment(), PORT: String(input.port), }, windowsHide: true, @@ -1463,6 +1474,8 @@ async function startRuntimeForConfig(input: { readonly runtimeStateRoot: string; readonly scopeId: string; readonly config: Record; + readonly providerCredentialEnvironment?: Readonly>; + readonly providerChildEnvironment?: Readonly>; readonly codex?: { readonly providerAccountId: string; readonly modelId: string; @@ -1479,6 +1492,9 @@ async function startRuntimeForConfig(input: { runtimeStateRoot: input.runtimeStateRoot, scopeId: input.scopeId, unifiedRuntimeConfigPath: configPath, + providerCredentialEnvironment: input.providerCredentialEnvironment, + providerChildEnvironment: + input.providerChildEnvironment ?? createCredentialStrippedEnvironment(), ...(input.codex ? { codexAuthAdapter: createValidationCodexAuthAdapter(), @@ -1770,8 +1786,6 @@ export async function runRuntimeVendorValidation(options: { options.runtimeStateRoot ?? (await mkdtemp(path.join(os.tmpdir(), "role-model-runtime-vendors-"))); const scopePrefix = options.scopeId ?? "runtime-vendor-validation"; - const previousOpenAiApiKey = process.env.OPENAI_API_KEY; - process.env.OPENAI_API_KEY = previousOpenAiApiKey || "runtime-vendor-validation-key"; const plan = await createRuntimeVendorValidationPlan({ runtimeStateRoot, scopeId: scopePrefix, @@ -1836,6 +1850,7 @@ export async function runRuntimeVendorValidation(options: { runtimeStateRoot, scopeId: `${scopePrefix}-remote`, config: plan.remoteConfig, + providerCredentialEnvironment: VALIDATION_PROVIDER_CREDENTIAL_ENV, }); try { await waitForRuntimeModelEndpointsReady(remoteRuntime.backend, [plan.remoteModelId]); @@ -1862,6 +1877,7 @@ export async function runRuntimeVendorValidation(options: { runtimeStateRoot, scopeId: `${scopePrefix}-hybrid`, config: plan.hybridConfig, + providerCredentialEnvironment: VALIDATION_PROVIDER_CREDENTIAL_ENV, codex: { providerAccountId: VALIDATION_CODEX_PROVIDER_ACCOUNT_ID, modelId: plan.codexModelId, @@ -2007,6 +2023,7 @@ export async function runRuntimeVendorValidation(options: { }, }, }, + providerCredentialEnvironment: VALIDATION_PROVIDER_CREDENTIAL_ENV, codex: { providerAccountId: VALIDATION_CODEX_PROVIDER_ACCOUNT_ID, modelId: plan.codexModelId, @@ -2217,11 +2234,6 @@ export async function runRuntimeVendorValidation(options: { if (!options.runtimeStateRoot) { await rm(runtimeStateRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } - if (previousOpenAiApiKey === undefined) { - process.env.OPENAI_API_KEY = undefined; - } else { - process.env.OPENAI_API_KEY = previousOpenAiApiKey; - } } } diff --git a/role-model-router/apps/runtime-host-bridge/test/candidate-profile-scaling.test.ts b/role-model-router/apps/runtime-host-bridge/test/candidate-profile-scaling.test.ts index 6473ad4f..e6aa27f0 100644 --- a/role-model-router/apps/runtime-host-bridge/test/candidate-profile-scaling.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/candidate-profile-scaling.test.ts @@ -53,7 +53,7 @@ test("persists the selected current benchmark profile revision with the routing const testDir = path.dirname(fileURLToPath(import.meta.url)); const source = await readFile(path.join(testDir, "..", "src", "index.ts"), "utf8"); const start = source.indexOf("const decisionMembershipRevision"); - const end = source.indexOf("const bundle = createRuntimeObservationBundle", start); + const end = source.indexOf("const baseBundle = createRuntimeObservationBundle", start); expect(start).toBeGreaterThan(-1); expect(end).toBeGreaterThan(start); const decisionSlice = source.slice(start, end + 1600); diff --git a/role-model-router/apps/runtime-host-bridge/test/executable.test.ts b/role-model-router/apps/runtime-host-bridge/test/executable.test.ts index a8a418a0..24d82b8b 100644 --- a/role-model-router/apps/runtime-host-bridge/test/executable.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/executable.test.ts @@ -102,6 +102,40 @@ async function collectRuntimeDependencyGraph(): Promise< } describe("runtime-host-bridge executable packaging", () => { + test("resolves an explicit external release root so dev and stage package trees cannot overwrite each other", () => { + const defaultDir = packageSea.resolveReleaseOutputDirectory({ + distRoot: "D:/repo/role-model-router/dist", + releaseTarget: "win32-x64", + env: {}, + }); + expect(path.normalize(defaultDir)).toBe( + path.normalize("D:/repo/role-model-router/dist/release/win32-x64"), + ); + + const isolatedDir = packageSea.resolveReleaseOutputDirectory({ + distRoot: "D:/repo/role-model-router/dist", + releaseTarget: "win32-x64", + env: { ROLE_MODEL_RELEASE_OUTPUT_ROOT: "D:/TEMP/run94-packages/development" }, + }); + expect(path.normalize(isolatedDir)).toBe( + path.normalize("D:/TEMP/run94-packages/development/win32-x64"), + ); + expect(() => + packageSea.resolveReleaseOutputDirectory({ + distRoot: "D:/repo/role-model-router/dist", + releaseTarget: "../stage", + env: {}, + }), + ).toThrow(/target/i); + expect(() => + packageSea.resolveReleaseOutputDirectory({ + distRoot: "D:/repo/role-model-router/dist", + releaseTarget: "win32-x64", + env: { ROLE_MODEL_RELEASE_OUTPUT_ROOT: "relative/output" }, + }), + ).toThrow(/absolute/i); + }); + test("does not treat imported package-sea modules as direct CLI execution", () => { expect( ( diff --git a/role-model-router/apps/runtime-host-bridge/test/fixtures/recursive-87-shadow-extension.mjs b/role-model-router/apps/runtime-host-bridge/test/fixtures/recursive-87-shadow-extension.mjs index 7b269a01..627dd6e0 100644 --- a/role-model-router/apps/runtime-host-bridge/test/fixtures/recursive-87-shadow-extension.mjs +++ b/role-model-router/apps/runtime-host-bridge/test/fixtures/recursive-87-shadow-extension.mjs @@ -11,9 +11,14 @@ export async function run(envelope = {}) { } if (capability === "evaluation:run-local") { const value = envelope.value; + const scores = value.cases.map((row) => (row.expected === row.actual ? 1 : 0)); return { count: value.cases.length, - scores: value.cases.map((row) => (row.expected === row.actual ? 1 : 0)), + scores, + holdout: { + passed: scores.reduce((sum, score) => sum + score, 0) / scores.length > 0.5, + evidenceRef: "sha256:fixture-derived-holdout", + }, environment: "local-routing-evaluation", provenance: { policy: value.policy, diff --git a/role-model-router/apps/runtime-host-bridge/test/index.test.ts b/role-model-router/apps/runtime-host-bridge/test/index.test.ts index 03e8e38e..de29b379 100644 --- a/role-model-router/apps/runtime-host-bridge/test/index.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/index.test.ts @@ -10832,36 +10832,10 @@ describe("runtime-host-bridge", () => { safetyPolicyRefs: ["safety.review"], }, }, - inspection: { - request: { - requestCapture: { - body: { - model: "chat-capture-v1", - input: [ - { - role: "system", - content: "Review carefully and produce a release-readiness assessment.", - }, - { - role: "system", - content: - "You must satisfy these output contracts in your response: review.checklist.", - }, - { - role: "system", - content: - "Apply these safety policies while handling the request: safety.review.", - }, - { - role: "user", - content: "Assess release readiness.", - }, - ], - }, - }, - }, - }, }); + // Run 94 (SP2): request captures (including injected system instructions) are + // graph-external; the inline SQLite row never carries them. + expect(await backend.readRequestObservation(requestId)).not.toHaveProperty("inspection"); } finally { await rm(fixtureRoot, { recursive: true, force: true }); await rm(runtimeStateRoot, { recursive: true, force: true }); @@ -20174,36 +20148,10 @@ describe("runtime-host-bridge", () => { safetyPolicyRefs: ["safety.review"], }, }, - inspection: { - request: { - requestCapture: { - body: { - model: "chat-capture-v1", - input: [ - { - role: "system", - content: "Review carefully and produce a release-readiness assessment.", - }, - { - role: "system", - content: - "You must satisfy these output contracts in your response: review.checklist.", - }, - { - role: "system", - content: - "Apply these safety policies while handling the request: safety.review.", - }, - { - role: "user", - content: "Assess release readiness.", - }, - ], - }, - }, - }, - }, }); + // Run 94 (SP2): request captures (including injected policy messages) are + // graph-external; the inline SQLite row never carries them. + expect(await backend.readRequestObservation(requestId)).not.toHaveProperty("inspection"); } finally { await rm(fixtureRoot, { recursive: true, force: true }); await rm(runtimeStateRoot, { recursive: true, force: true }); diff --git a/role-model-router/apps/runtime-host-bridge/test/recursive-87-shadow-pipeline.test.ts b/role-model-router/apps/runtime-host-bridge/test/recursive-87-shadow-pipeline.test.ts index 425f85ac..01862677 100644 --- a/role-model-router/apps/runtime-host-bridge/test/recursive-87-shadow-pipeline.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/recursive-87-shadow-pipeline.test.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { afterEach, expect, test } from "vitest"; @@ -20,6 +21,67 @@ afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); +const comparableEvidence = () => ({ + source: { + rolloutId: "rollout-source-87", + routePackage: "candidate-local", + endpointId: "endpoint-local", + modelId: "model-local", + policyId: "routing-policy-a", + reasoningEffort: "medium", + effortSource: "variant", + evidenceRef: "evidence:source-87", + artifactRef: "artifact:source-87", + propensity: 0.6, + outcome: { + outcomeId: "outcome-source-87", + outcomeRef: "outcome:source-87", + outcomeDigest: "digest:source-87", + source: "observed", + status: "success", + }, + }, + counterfactuals: [ + { + rolloutId: "rollout-counterfactual-87", + routePackage: "candidate-remote", + endpointId: "endpoint-remote", + modelId: "model-remote", + policyId: "routing-policy-b", + reasoningEffort: "high", + effortSource: "variant", + evidenceRef: "evidence:counterfactual-87", + artifactRef: "artifact:counterfactual-87", + propensity: 0.4, + outcome: { + outcomeId: "outcome-counterfactual-87", + outcomeRef: "outcome:counterfactual-87", + outcomeDigest: "digest:counterfactual-87", + source: "replay", + status: "failure", + }, + }, + ], + candidateSet: [ + { routePackage: "candidate-local", endpointId: "endpoint-local", propensity: 0.6 }, + { routePackage: "candidate-remote", endpointId: "endpoint-remote", propensity: 0.4 }, + ], +}); + +const comparableCases = () => [ + { + id: "case-87", + expected: "expected-route", + actual: "expected-route", + expectedRolloutId: "rollout-source-87", + actualRolloutId: "rollout-counterfactual-87", + expectedOutcomeRef: "outcome:source-87", + actualOutcomeRef: "outcome:counterfactual-87", + expectedEvidenceRef: "evidence:source-87", + actualEvidenceRef: "evidence:counterfactual-87", + }, +]; + test("SP1 runs the useful routing-learning DAG through supervised shadow capabilities", async () => { expect(typeof trackBRuntime.runTrackBShadowPipeline).toBe("function"); const artifactSha256 = createHash("sha256") @@ -63,8 +125,9 @@ test("SP1 runs the useful routing-learning DAG through supervised shadow capabil sourceDecisionId: "route-87", sourceGraphRef: "sha256:graph-87", prefix: ["request", "eligible-endpoints"], - counterfactuals: [{ id: "candidate-local", suffix: ["candidate-local"] }], - evaluationCases: [{ expected: "candidate-local", actual: "candidate-local" }], + counterfactuals: [{ id: "candidate-remote", suffix: ["candidate-remote"] }], + comparableEvidence: comparableEvidence(), + evaluationCases: comparableCases(), trajectoryEvents: [], }); @@ -87,6 +150,35 @@ test("SP1 runs the useful routing-learning DAG through supervised shadow capabil expect(runtime.listExtensions().every((row) => row.lifecycle === "ready")).toBe(true); }); +test("Run 94 R14 refuses self-comparison before creating a learned-experience candidate", async () => { + const invoked: string[] = []; + await expect( + trackBRuntime.runTrackBShadowPipeline( + { + invoke: async (id) => { + invoked.push(id); + return {}; + }, + }, + { + requestId: "run94-self-comparison", + channel: "development", + scope: "tenant:run94", + authorizationEpoch: 94, + productionState: {}, + routePackage: "route-package-a", + sourceDecisionId: "decision-run94-a", + sourceGraphRef: "sha256:graph-run94-a", + prefix: [{ routePackage: "route-package-a" }], + counterfactuals: [{ id: "route-package-a", suffix: [{ routePackage: "route-package-a" }] }], + evaluationCases: [{ expected: "route-package-a", actual: "route-package-a" }], + trajectoryEvents: [], + }, + ), + ).rejects.toThrow(/R14_NO_DISTINCT_COUNTERFACTUAL|distinct counterfactual|self-comparison/i); + expect(invoked).toEqual([]); +}); + test("SP1 fails closed before Knowledge Worker when holdout evaluation fails", async () => { const invoke = async (id: string) => id === "evaluation-runner-local" @@ -104,8 +196,9 @@ test("SP1 fails closed before Knowledge Worker when holdout evaluation fails", a sourceDecisionId: "route-87", sourceGraphRef: "sha256:graph-87", prefix: [], - counterfactuals: [], - evaluationCases: [{ expected: 1, actual: 0 }], + counterfactuals: [{ id: "candidate-remote", suffix: [] }], + comparableEvidence: comparableEvidence(), + evaluationCases: comparableCases(), trajectoryEvents: [], productionState: {}, }, @@ -249,7 +342,7 @@ test("Phase 3.5 post-observation work survives startup and processing failures", }); }); -test("Run 91 safely retires legacy queued work that cannot prove variant identity", async () => { +test("Run 91 quarantines legacy queued work that cannot prove variant identity", async () => { const stateRoot = path.join(os.tmpdir(), `run91-legacy-shadow-outbox-${Date.now()}`); roots.push(stateRoot); const filePath = path.join(stateRoot, "outbox.json"); @@ -275,19 +368,16 @@ test("Run 91 safely retires legacy queued work that cannot prove variant identit return {}; }); expect(replayed).toEqual([]); - expect(await outbox.read()).toMatchObject({ - pendingCount: 0, - receiptCount: 1, - receipts: [ - { - requestId: "legacy-run91", - result: { - status: "retired_legacy_missing_variant_identity", - productionMutation: false, - }, - }, - ], - }); + expect(await outbox.read()).toMatchObject({ pendingCount: 0, receiptCount: 0, receipts: [] }); + const database = new DatabaseSync(filePath); + expect( + database + .prepare( + "SELECT classification, reason FROM track_b_post_observation_legacy_rows WHERE source_id=?", + ) + .get("legacy-run91"), + ).toMatchObject({ classification: "quarantined" }); + database.close(); }); test("Phase 3.5 normal post-observation work executes every canonical business owner", async () => { @@ -295,9 +385,25 @@ test("Phase 3.5 normal post-observation work executes every canonical business o const runtime = { async invoke(id: string, envelope: Record) { invoked.push({ id, envelope }); - if (id === "artifact-store") return { id: "artifact:run87" }; + const base = { + workerPid: 1000 + invoked.length, + durableLocator: { + extensionId: id, + requestId: envelope.requestId, + invocation: invoked.length, + }, + evidenceRef: `evidence:${id}:${String(envelope.requestId)}`, + businessOutput: { + extensionId: id, + capability: envelope.capability, + invocation: invoked.length, + }, + readCapability: "artifact:read", + }; + if (id === "artifact-store") return { ...base, id: "artifact:run87" }; if (id === "repository-context") { return { + ...base, available: true, context: { scopeId: "tenant:run87", @@ -317,15 +423,15 @@ test("Phase 3.5 normal post-observation work executes every canonical business o }; } if (id === "knowledge-store" && envelope.capability === "knowledge:write") { - return { id: "knowledge:run87" }; + return { ...base, id: "knowledge:run87" }; } if (id === "evaluation-runner-local") { - return { scores: [1], provenance: { evidenceRef: "artifact:run87" } }; + return { ...base, scores: [1], provenance: { evidenceRef: "artifact:run87" } }; } if (id === "knowledge-worker") { - return { id: "candidate:run87", state: "shadow", productionEffects: {} }; + return { ...base, id: "candidate:run87", state: "shadow", productionEffects: {} }; } - return {}; + return base; }, }; const identity = { diff --git a/role-model-router/apps/runtime-host-bridge/test/run88-public-runtime-probes.ts b/role-model-router/apps/runtime-host-bridge/test/run88-public-runtime-probes.ts index 1f83645f..9b19ff86 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run88-public-runtime-probes.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run88-public-runtime-probes.ts @@ -268,8 +268,49 @@ const shadowInput = (overrides: Readonly> = {}) => ({ sourceDecisionId: "decision-1", sourceGraphRef: "sha256:graph-1", prefix: ["request"], - counterfactuals: [{ id: "candidate-stage", suffix: ["candidate-stage"] }], - evaluationCases: [{ expected: "candidate-stage", actual: "candidate-stage" }], + counterfactuals: [{ id: "candidate-control", suffix: ["candidate-control"] }], + comparableEvidence: { + source: { + rolloutId: "rollout-stage-candidate", + routePackage: "candidate-stage", + endpointId: "endpoint-stage", + modelId: "model-stage", + policyId: "policy-stage", + reasoningEffort: "medium", + evidenceRef: "evidence:stage-candidate", + propensity: 0.6, + outcome: { status: "success", outcomeRef: "outcome:stage-candidate" }, + }, + counterfactuals: [ + { + rolloutId: "rollout-stage-control", + routePackage: "candidate-control", + endpointId: "endpoint-control", + modelId: "model-control", + policyId: "policy-control", + reasoningEffort: "high", + evidenceRef: "evidence:stage-control", + propensity: 0.4, + outcome: { status: "failure", outcomeRef: "outcome:stage-control" }, + }, + ], + candidateSet: [ + { routePackage: "candidate-stage", endpointId: "endpoint-stage", propensity: 0.6 }, + { routePackage: "candidate-control", endpointId: "endpoint-control", propensity: 0.4 }, + ], + }, + evaluationCases: [ + { + expected: "candidate-stage", + actual: "candidate-stage", + expectedRolloutId: "rollout-stage-candidate", + actualRolloutId: "rollout-stage-control", + expectedOutcomeRef: "outcome:stage-candidate", + actualOutcomeRef: "outcome:stage-control", + expectedEvidenceRef: "evidence:stage-candidate", + actualEvidenceRef: "evidence:stage-control", + }, + ], trajectoryEvents: [], ...overrides, }); @@ -277,7 +318,11 @@ const shadowInput = (overrides: Readonly> = {}) => ({ const shadowRuntime = { async invoke(id: string) { if (id === "evaluation-runner-local") - return { scores: [1], provenance: { evidenceRef: "sha256:graph-1" } }; + return { + scores: [1], + holdout: { passed: true, evidenceRef: "sha256:graph-1" }, + provenance: { evidenceRef: "sha256:graph-1" }, + }; if (id === "knowledge-worker") return { id: "shadow-candidate-1", state: "shadow" }; return { id: `${id}-result` }; }, diff --git a/role-model-router/apps/runtime-host-bridge/test/run94-local-graph-capture.test.ts b/role-model-router/apps/runtime-host-bridge/test/run94-local-graph-capture.test.ts new file mode 100644 index 00000000..74937999 --- /dev/null +++ b/role-model-router/apps/runtime-host-bridge/test/run94-local-graph-capture.test.ts @@ -0,0 +1,29 @@ +import { createHash } from "node:crypto"; +import { mkdtemp } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import { expect, test } from "vitest"; + +import { createTrackBFileGraphStore } from "../src/track-b-runtime.js"; + +test("run94 graph capture: local fixture store writes and reads content-addressed bundles", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "run94-local-graph-store-")); + const store = createTrackBFileGraphStore({ + scopeId: "run94-local-graph", + rootPath: root, + }); + const content = JSON.stringify({ requestId: "req-run94-graph", payload: "rich" }); + const contentHash = `sha256:${createHash("sha256").update(content).digest("hex")}`; + + const artifact = store.write({ + scopeId: store.scopeId, + sourceId: "req-run94-graph", + content, + contentHash, + }); + + expect(artifact.artifactId).toBeTruthy(); + expect(artifact.artifactPath).toContain(root); + expect(store.read({ scopeId: store.scopeId, ...artifact })).toBe(content); +}); diff --git a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts new file mode 100644 index 00000000..1a68f279 --- /dev/null +++ b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts @@ -0,0 +1,388 @@ +import { createHash } from "node:crypto"; +import { readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; + +import { afterEach, expect, test } from "vitest"; + +import { createRuntimeBridgeBackend, startBridgeServer } from "../src/index.js"; +import { + createProductionExtensionRuntime, + createTrackBPostObservationOutbox, + runTrackBPostObservation, + verifyTrackBExtensionClosureAfterRestart, +} from "../src/track-b-runtime.js"; + +const repoRoot = path.resolve(import.meta.dirname, "..", "..", "..", ".."); +const roots: string[] = []; +const runtimes: Array<{ close(): Promise }> = []; + +afterEach(async () => { + await Promise.allSettled(runtimes.splice(0).map((runtime) => runtime.close())); + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +function identity(requestId = "run94-outbox") { + return { + requestId, + routingDecisionId: `decision:${requestId}`, + endpointId: "endpoint:run94", + modelId: "model:run94", + reasoningEffort: null, + effortSource: "none" as const, + }; +} + +test("GREEN: post-observation outbox is a normalized SQLite authority with bounded receipts and restart recovery", async () => { + const root = await import("node:fs/promises").then(({ mkdtemp }) => + mkdtemp(path.join(os.tmpdir(), "run94-sp5-outbox-")), + ); + roots.push(root); + const filePath = path.join(root, "post-observation-outbox.sqlite"); + const outbox = createTrackBPostObservationOutbox({ filePath, maxItems: 8 }); + await outbox.enqueue(identity("queued-1")); + await outbox.enqueue(identity("queued-2")); + + expect((await readFile(filePath)).subarray(0, 15).toString("utf8")).toBe("SQLite format 3"); + const database = new DatabaseSync(filePath); + expect( + database + .prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name IN ('track_b_post_observation_pending','track_b_post_observation_receipts') ORDER BY name", + ) + .all(), + ).toHaveLength(2); + database.close(); + + const restarted = createTrackBPostObservationOutbox({ filePath, maxItems: 8 }); + await restarted.drain(async () => ({ + status: "processed", + extensionClosure: { result: "x".repeat(100_000) }, + })); + const receipt = await restarted.read(); + expect(receipt).toMatchObject({ pendingCount: 0, receiptCount: 2 }); + const afterDrain = new DatabaseSync(filePath); + const row = afterDrain + .prepare("SELECT length(result_json) AS bytes FROM track_b_post_observation_receipts") + .get() as { bytes: number }; + expect(row.bytes).toBeLessThanOrEqual(16 * 1024); + afterDrain.close(); +}); + +test("GREEN: imports N-1 JSON once, classifies every legacy row, and quarantines malformed rows", async () => { + const root = await import("node:fs/promises").then(({ mkdtemp }) => + mkdtemp(path.join(os.tmpdir(), "run94-sp5-legacy-")), + ); + roots.push(root); + const filePath = path.join(root, "post-observation-outbox.json"); + await writeFile( + filePath, + JSON.stringify({ + schemaVersion: "role-model.track-b-post-observation-outbox.v2", + pending: [identity("legacy-valid"), { requestId: "legacy-malformed" }], + receipts: [ + { requestId: "legacy-receipt", completedAt: new Date().toISOString(), result: {} }, + ], + }), + "utf8", + ); + + const outbox = createTrackBPostObservationOutbox({ filePath, maxItems: 8 }); + expect(await outbox.read()).toMatchObject({ pendingCount: 1, receiptCount: 1 }); + expect((await readFile(filePath)).subarray(0, 15).toString("utf8")).toBe("SQLite format 3"); + const database = new DatabaseSync(filePath); + expect( + database + .prepare( + "SELECT source_id, classification FROM track_b_post_observation_legacy_rows ORDER BY source_kind, source_index", + ) + .all(), + ).toEqual([ + { source_id: "legacy-valid", classification: "imported" }, + { source_id: "legacy-malformed", classification: "quarantined" }, + { source_id: "legacy-receipt", classification: "imported" }, + ]); + database.close(); + expect(await readFile(`${filePath}.n-1.json`, "utf8")).toContain("legacy-malformed"); +}); + +test("GREEN: malformed legacy JSON fails closed without replacing the source authority", async () => { + const root = await import("node:fs/promises").then(({ mkdtemp }) => + mkdtemp(path.join(os.tmpdir(), "run94-sp5-malformed-")), + ); + roots.push(root); + const filePath = path.join(root, "post-observation-outbox.json"); + await writeFile(filePath, "{not-json", "utf8"); + const outbox = createTrackBPostObservationOutbox({ filePath }); + await expect(outbox.read()).rejects.toThrow(/malformed|JSON|legacy/i); + expect(await readFile(filePath, "utf8")).toBe("{not-json"); +}); + +test("GREEN: legacy import cannot bypass pending or receipt bounds", async () => { + const root = await import("node:fs/promises").then(({ mkdtemp }) => + mkdtemp(path.join(os.tmpdir(), "run94-sp5-legacy-bound-")), + ); + roots.push(root); + const filePath = path.join(root, "post-observation-outbox.json"); + await writeFile( + filePath, + JSON.stringify({ + schemaVersion: "role-model.track-b-post-observation-outbox.v2", + pending: [identity("legacy-pending-1"), identity("legacy-pending-2")], + receipts: [ + { requestId: "legacy-receipt-1", completedAt: new Date().toISOString(), result: {} }, + { requestId: "legacy-receipt-2", completedAt: new Date().toISOString(), result: {} }, + ], + }), + "utf8", + ); + const outbox = createTrackBPostObservationOutbox({ filePath, maxItems: 1 }); + expect(await outbox.read()).toMatchObject({ pendingCount: 1, receiptCount: 1 }); + const database = new DatabaseSync(filePath); + expect( + database + .prepare( + "SELECT source_kind, source_id, classification FROM track_b_post_observation_legacy_rows ORDER BY source_kind, source_index", + ) + .all(), + ).toEqual([ + { source_kind: "pending", source_id: "legacy-pending-1", classification: "imported" }, + { source_kind: "pending", source_id: "legacy-pending-2", classification: "quarantined" }, + { source_kind: "receipt", source_id: "legacy-receipt-1", classification: "imported" }, + { source_kind: "receipt", source_id: "legacy-receipt-2", classification: "quarantined" }, + ]); + database.close(); +}); + +const canonicalExtensions = [ + ["artifact-store", ["graph:write", "artifact:read"]], + ["event-log", ["event:append", "event:read"]], + ["repository-context", ["repository:read", "artifact:read"]], + ["background-evidence-scheduler", ["scheduler:schedule-and-run", "artifact:read"]], + ["memory-store", ["memory:write", "memory:read", "artifact:read"]], + ["knowledge-store", ["knowledge:write", "knowledge:read", "artifact:read"]], + ["evaluation-core", ["evaluation:consume-projection", "artifact:read"]], + ["crowdsourced-learning", ["aggregate:preview", "artifact:read"]], + ["replay-core", ["replay:plan-graph", "artifact:read"]], + ["evaluation-runner-local", ["evaluation:run-local", "artifact:read"]], + ["trajectory-signals", ["signals:analyze", "artifact:read"]], + ["profile-learner", ["profile:estimate", "profile:consume-projection", "artifact:read"]], + [ + "knowledge-worker", + ["knowledge:eval-consumer", "knowledge:consume-projection", "artifact:read"], + ], +] as const; + +const extensionModule = ` +export async function run(envelope = {}) { + const id = process.env.ROLE_MODEL_EXTENSION_ID; + const capability = envelope.capability; + if (["artifact:read", "event:read", "memory:read", "knowledge:read"].includes(capability) && envelope.payload?.durableOutputId) { + return { readbackOutputId: envelope.payload?.durableOutputId, durableLocator: envelope.payload?.durableLocator }; + } + const base = { + extensionId: id, + capability, + requestId: envelope.requestId, + durableLocator: { extensionId: id, requestId: envelope.requestId, capability }, + evidenceRef: "evidence:" + id + ":" + envelope.requestId, + readCapability: id === "knowledge-worker" ? undefined : id === "event-log" ? "event:read" : id === "memory-store" ? "memory:read" : id === "knowledge-store" ? "knowledge:read" : "artifact:read", + businessOutput: { extensionId: id, capability, value: "business-result" } + }; + if (id === "artifact-store") return { ...base, id: "artifact:" + envelope.requestId }; + if (id === "repository-context") return { ...base, available: true, context: { scopeId: envelope.scope, repoFingerprint: "a".repeat(64), packageId: null, fallbackLevel: "repo_task", branchCompatibility: "unknown", fingerprintEpoch: 1 }, diagnostics: [] }; + if (capability === "evaluation:run-local") return { ...base, count: 1, scores: [1], environment: "local", provenance: { evidenceRef: base.evidenceRef } }; + if (capability === "knowledge:write") return { ...base, id: "knowledge:" + envelope.requestId }; + if (capability === "knowledge:eval-consumer") return { ...base, state: "shadow", productionEffects: {} }; + return base; +} +`; + +async function createRealCanonicalRuntime(root: string) { + const modulePath = path.join(root, "canonical-extension.mjs"); + await writeFile(modulePath, extensionModule, "utf8"); + const artifactSha256 = createHash("sha256").update(extensionModule).digest("hex"); + const runtime = await createProductionExtensionRuntime({ + stateRoot: path.join(root, "extension-runtime"), + authorizationEpoch: 94, + repoRoot, + extensions: canonicalExtensions.map(([id, capabilities]) => ({ + descriptor: { id, protocolVersion: "1.1.0", capabilities }, + modulePath, + artifactSha256, + })), + }); + runtimes.push(runtime); + return runtime; +} + +function observation() { + return { + requestId: "run94-extension-closure", + routingDecisionId: "decision:run94-extension-closure", + endpointId: "endpoint:run94", + modelId: "model:run94", + reasoningEffort: null, + effortSource: "none" as const, + usageEvent: { endpoint_id: "endpoint:run94", model_id: "model:run94" }, + }; +} + +test("GREEN: real process output closure covers every canonical registry key and readback survives restart", async () => { + const root = await import("node:fs/promises").then(({ mkdtemp }) => + mkdtemp(path.join(os.tmpdir(), "run94-sp10-real-workers-")), + ); + roots.push(root); + const runtime = await createRealCanonicalRuntime(root); + const result = await runTrackBPostObservation(runtime, observation(), { + scope: "tenant:run94", + channel: "development", + authorizationEpoch: 94, + }); + const closure = (result as Record).extensionClosure as Record; + expect(Object.keys(closure.registry ?? {})).toEqual(canonicalExtensions.map(([id]) => id).sort()); + const outputs = Object.values(closure.registry as Record).flatMap( + (entry) => entry.outputs, + ); + expect(outputs).toHaveLength(15); + expect((result as Record).pipeline).toMatchObject({ + status: "insufficient_comparable_evidence", + refusalCode: "R14_NO_DISTINCT_COUNTERFACTUAL", + candidateId: null, + providerCalls: 0, + productionMutation: false, + }); + expect(JSON.stringify(result)).not.toContain("baseline-control"); + expect( + new Set(outputs.map((output) => (output as Record).durableOutputId)).size, + ).toBe(outputs.length); + expect( + outputs.every((output) => Number.isInteger((output as Record).workerPid)), + ).toBe(true); + const readback = await verifyTrackBExtensionClosureAfterRestart(runtime, closure as never, { + channel: "development", + scope: "tenant:run94", + authorizationEpoch: 94, + readDurableEvidence: async ({ durableLocator, durableOutputId }) => + runtime.invoke("artifact-store", { + requestId: `run94-readback-evidence:${durableOutputId}`, + protocolVersion: "1.1.0", + channel: "development", + scope: "tenant:run94", + authorizationEpoch: 94, + capability: "artifact:read", + payload: { durableLocator, durableOutputId }, + }), + }); + expect(readback.outputs.every((row) => row.readbackOutputId === row.durableOutputId)).toBe(true); + expect(readback.outputs.every((row) => row.preRestartPid !== row.postRestartPid)).toBe(true); +}); + +test("GREEN: rejects missing, health-only, and duplicate durable extension outputs", async () => { + const base = async (id: string, envelope: Record) => ({ + id: id === "artifact-store" ? "artifact:duplicate" : undefined, + durableLocator: { id: "same" }, + evidenceRef: "evidence:same", + businessOutput: { value: "same" }, + available: true, + workerPid: 1, + ...(id === "repository-context" + ? { + context: { + scopeId: "tenant:run94", + repoFingerprint: "a".repeat(64), + packageId: null, + fallbackLevel: "repo_task", + branchCompatibility: "unknown", + fingerprintEpoch: 1, + }, + diagnostics: [], + } + : {}), + ...(envelope.capability === "evaluation:run-local" + ? { scores: [1], provenance: { evidenceRef: "evidence:same" } } + : {}), + ...(envelope.capability === "knowledge:write" ? { id: "knowledge:duplicate" } : {}), + }); + await expect( + runTrackBPostObservation( + { invoke: async () => ({ health: { available: true }, workerPid: 1 }) }, + observation(), + { scope: "tenant:run94", channel: "development", authorizationEpoch: 94 }, + ), + ).rejects.toThrow(/business output|durable/i); + await expect( + runTrackBPostObservation( + { + invoke: async () => ({ + id: "health-marker", + durableLocator: "health-marker", + workerPid: 1, + available: true, + health: { status: "ready" }, + }), + }, + observation(), + { scope: "tenant:run94", channel: "development", authorizationEpoch: 94 }, + ), + ).rejects.toThrow(/business output|durable/i); + await expect( + runTrackBPostObservation({ invoke: base }, observation(), { + scope: "tenant:run94", + channel: "development", + authorizationEpoch: 94, + }), + ).rejects.toThrow(/duplicate/i); +}); + +test("GREEN: exposes extension readback through the Track B HTTP surface", async () => { + const runtimeStateRoot = await import("node:fs/promises").then(({ mkdtemp }) => + mkdtemp(path.join(os.tmpdir(), "run94-sp10-api-")), + ); + roots.push(runtimeStateRoot); + const backend = await createRuntimeBridgeBackend({ + repoRoot, + fixtureRoot: path.join( + repoRoot, + "role-model-router", + "apps", + "runtime-host-bridge", + "test", + "fixtures", + ), + runtimeStateRoot, + scopeId: "run94-api", + readTrackBExtensionReadback: async (body) => ({ + requestId: body.requestId, + schemaVersion: "role-model.track-b-extension-readback.v1", + }), + }); + const server = await startBridgeServer({ + host: "127.0.0.1", + port: 0, + registry: backend.registry, + getRegistry: () => backend.registry, + executeChatCompletions: backend.executeChatCompletions, + executeResponses: backend.executeResponses, + readTrackBExtensionReadback: backend.readTrackBExtensionReadback, + }); + try { + const response = await fetch( + `http://127.0.0.1:${server.port}/api/role-model/track-b/extension-readback`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ requestId: "run94-extension-closure" }), + }, + ); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + requestId: "run94-extension-closure", + schemaVersion: "role-model.track-b-extension-readback.v1", + }); + } finally { + await server.close(); + await backend.shutdown(); + } +}); diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index af4d128e..6e34e870 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -14,6 +14,8 @@ import { LegacySqliteMigration } from "../../../packages/sqlite-memory/src/legac import { applyRecommendationServiceLauncherConfig } from "../src/cli.js"; import { createRuntimeBridgeBackend, startBridgeServer } from "../src/index.js"; import { + buildProviderEvidenceFromObservation, + buildVerifiersLiveExport, createTrackBOperations, seedTrackBExtensionBridgeState, } from "../src/track-b-operations.js"; @@ -46,6 +48,141 @@ afterEach(async () => { }); describe("Track B operations APIs", () => { + test("builds provider evidence and a semantic Verifiers export from the exact durable graph", () => { + const observation = { + requestId: "request-export-94", + routingDecisionId: "decision-export-94", + endpointId: "endpoint-export-94", + usageEvent: { model_id: "provider/model-export-94" }, + run88Correlation: { correlationId: "correlation-export-94" }, + executionSemantics: { + failedAttempts: [{ attemptId: "request-export-94:attempt:1" }], + }, + }; + expect(buildProviderEvidenceFromObservation(observation)).toEqual({ + endpointId: "endpoint-export-94", + modelId: "provider/model-export-94", + status: "ok", + attemptIds: ["request-export-94:attempt:1", "request-export-94:attempt:2"], + }); + const capture = { + schemaVersion: "role-model.route-capture-read.v1", + requestId: "request-export-94", + routingDecisionId: "decision-export-94", + rootArtifactId: "root-export-94", + messages: [ + { nodeId: "node-system-94", role: "system", content: "route safely" }, + { nodeId: "node-user-94", role: "user", content: "route this" }, + ], + response: { nodeId: "node-response-94", role: "assistant", content: "routed" }, + tools: [], + edgeCount: 3, + }; + const exported = buildVerifiersLiveExport({ + channel: "development", + request: { + requestId: "request-export-94", + correlationId: "correlation-export-94", + graphRootArtifactId: "root-export-94", + readiness: "semantic", + }, + observation, + capture, + }); + expect(exported).toMatchObject({ + schemaVersion: "role-model.verifiers-live-export.v1", + channel: "development", + requestId: "request-export-94", + correlationId: "correlation-export-94", + graphRootArtifactId: "root-export-94", + responseNodeIndex: 2, + tokenExactDisposition: "refused_missing_evidence", + trace: { + nodes: [ + { parent: null, message: { role: "system", content: "route safely" }, sampled: false }, + { parent: 0, message: { role: "user", content: "route this" }, sampled: false }, + { parent: 1, message: { role: "assistant", content: "routed" }, sampled: true }, + ], + info: { + routeDecisionId: "decision-export-94", + roleModelGraphRootArtifactId: "root-export-94", + roleModelResponseNodeId: "node-response-94", + }, + }, + }); + expect(() => + buildVerifiersLiveExport({ + channel: "development", + request: { + requestId: "request-export-94", + correlationId: "correlation-export-94", + graphRootArtifactId: "wrong-root", + readiness: "semantic", + }, + observation, + capture, + }), + ).toThrow(/exact live graph/i); + }); + + test("reads one exact graph capture through the authenticated loopback sidecar", async () => { + const received: Array<{ path: string; authorization?: string; body: unknown }> = []; + const operations = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += chunk; + received.push({ + path: request.url ?? "", + authorization: request.headers.authorization, + body: JSON.parse(body), + }); + response.writeHead(200, { "content-type": "application/json" }); + response.end( + JSON.stringify({ + schemaVersion: "role-model.route-capture-read.v1", + requestId: "request-exact-94", + routingDecisionId: "decision-exact-94", + rootArtifactId: "root-exact-94", + messages: [{ nodeId: "message-exact-94", role: "user", content: "route me" }], + response: { nodeId: "response-exact-94", role: "assistant", content: "routed" }, + tools: [], + edgeCount: 2, + }), + ); + }); + await new Promise((resolve, reject) => { + operations.once("error", reject); + operations.listen(0, "127.0.0.1", resolve); + }); + try { + const address = operations.address(); + if (!address || typeof address === "string") + throw new Error("operations server did not bind"); + const api = createTrackBOperations({ + statePath: path.join(os.tmpdir(), `run94-exact-capture-${Date.now()}.json`), + catalog: [], + operationsEndpoint: `http://127.0.0.1:${address.port}`, + operationsToken: "run94-exact-capture-token-0001", + }); + const exact = await api.readLocalRouteCapture({ requestId: "request-exact-94" }); + expect(exact).toMatchObject({ + schemaVersion: "role-model.route-capture-read.v1", + requestId: "request-exact-94", + rootArtifactId: "root-exact-94", + }); + expect(received).toEqual([ + { + path: "/capture/read", + authorization: "Bearer run94-exact-capture-token-0001", + body: { requestId: "request-exact-94" }, + }, + ]); + } finally { + await new Promise((resolve, reject) => + operations.close((error) => (error ? reject(error) : resolve())), + ); + } + }); + test("fails closed instead of issuing unauthenticated calls to an owned operations endpoint", async () => { const runtimeStateRoot = path.join(os.tmpdir(), `track-b-operations-auth-${Date.now()}`); roots.push(runtimeStateRoot); @@ -156,6 +293,11 @@ describe("Track B operations APIs", () => { listRecommendations: async () => [{ id: "pack-1", signatureValid: true }], applyRecommendation: async () => ({ activePack: { id: "pack-1", version: "1" } }), readActivePack: async () => ({ id: "pack-1", version: "1" }), + exportVerifiersTrace: async (body) => ({ + schemaVersion: "role-model.verifiers-live-export.v1", + requestId: body.requestId, + graphRootArtifactId: body.graphRootArtifactId, + }), }); try { const base = `http://127.0.0.1:${server.port}`; @@ -227,6 +369,22 @@ describe("Track B operations APIs", () => { ).json() ).activeJob.status, ).toBe("running"); + const exported = await fetch(`${base}/api/role-model/track-b/verifiers-export`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + requestId: "request-http-export-94", + correlationId: "correlation-http-export-94", + graphRootArtifactId: "root-http-export-94", + readiness: "semantic", + }), + }); + expect(exported.status).toBe(200); + expect(await exported.json()).toEqual({ + schemaVersion: "role-model.verifiers-live-export.v1", + requestId: "request-http-export-94", + graphRootArtifactId: "root-http-export-94", + }); } finally { await server.close(); await backend.shutdown(); @@ -398,25 +556,46 @@ describe("Track B operations APIs", () => { roots.push(runtimeStateRoot); const received: Array<{ path: string; authorization?: string; body: Record }> = []; + let routeCapture: Record | null = null; const operations = createServer(async (request, response) => { const chunks: Buffer[] = []; for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record; + if (request.url === "/capture/route") routeCapture = body; received.push({ path: request.url ?? "", authorization: request.headers.authorization, - body: JSON.parse(Buffer.concat(chunks).toString("utf8")), + body, }); response.writeHead(200, { "content-type": "application/json" }); response.end( JSON.stringify( - request.url === "/capture/route" + request.url === "/capture/read" ? { - status: "captured", - scope: "tenant:production-upload", + schemaVersion: "role-model.route-capture-read.v1", + requestId: "req-track-b-upload-001", + routingDecisionId: routeCapture?.routingDecisionId, rootArtifactId: "artifact-route-capture", - rootArtifactDigest: "a".repeat(64), + messages: (routeCapture?.messages as unknown[]).map((message, index) => ({ + nodeId: `message-route-capture-${index}`, + ...(message as object), + })), + response: { + nodeId: "response-route-capture", + role: "assistant", + content: routeCapture?.outputText, + }, + tools: [], + edgeCount: 2, } - : { status: "accepted" }, + : request.url === "/capture/route" + ? { + status: "captured", + scope: "tenant:production-upload", + rootArtifactId: "artifact-route-capture", + rootArtifactDigest: "a".repeat(64), + } + : { status: "accepted" }, ), ); }); @@ -435,6 +614,11 @@ describe("Track B operations APIs", () => { runtimeStateRoot, scopeId: "production-upload", trackBOperationsEndpoint, + run88StageIdentity: { + releaseId: `sha256:${"d".repeat(64)}`, + sourceId: "e".repeat(40), + executableSha256: "c".repeat(64), + }, }); try { const databasePath = resolveSqliteMemoryLocation({ @@ -518,6 +702,32 @@ describe("Track B operations APIs", () => { contentHash: "a".repeat(64), }, }); + const detail = await backend.readRequestObservation("req-track-b-upload-001"); + expect(detail?.providerEvidence).toMatchObject({ + endpointId: result.endpointId, + modelId: "deepseek/chat-capture-v1", + status: "ok", + attemptIds: ["req-track-b-upload-001:attempt:1"], + }); + expect(detail?.graphEvidence).toEqual({ + rootArtifactId: "artifact-route-capture", + messageNodeIds: ["message-route-capture-0"], + responseNodeId: "response-route-capture", + edgeCount: 2, + }); + const correlationId = String( + (detail as unknown as { run88Correlation?: { correlationId?: string } })?.run88Correlation + ?.correlationId, + ); + expect(correlationId).not.toBe("undefined"); + expect( + await backend.exportVerifiersTrace({ + requestId: "req-track-b-upload-001", + correlationId, + graphRootArtifactId: "artifact-route-capture", + readiness: "semantic", + }), + ).toMatchObject({ responseNodeIndex: 1, tokenExactDisposition: "refused_missing_evidence" }); } finally { await backend.shutdown(); await new Promise((resolve, reject) => @@ -1143,6 +1353,53 @@ describe("Track B operations APIs", () => { expect(enabled.receipts).toHaveLength(1); }); + test("run94 SP8 local storage fallback never fabricates physical measurements", async () => { + const root = path.join(os.tmpdir(), `track-b-run94-honest-${Date.now()}`); + roots.push(root); + await mkdir(root, { recursive: true }); + const statePath = path.join(root, "bridge.json"); + const catalog = [ + { id: "artifact-store", packageClass: "canonical_extension", routingDependency: true }, + { id: "event-log", packageClass: "canonical_extension", routingDependency: false }, + ]; + await seedTrackBExtensionBridgeState({ statePath, catalog }); + const seeded = JSON.parse(await readFile(statePath, "utf8")) as { + storageServices?: unknown[]; + }; + seeded.storageServices = [ + { + id: "artifact-store", + category: "rich_trace", + tier: "canonical", + scope: "repo:a", + bytes: 10, + count: 1, + holds: 0, + leases: 0, + }, + ]; + await writeFile(statePath, JSON.stringify(seeded)); + const ops = createTrackBOperations({ statePath, catalog }); + const summary = (await ops.readStorageRetention()) as { + storageAudit: unknown; + storageInventory: { + entries: readonly { + id: string; + health: string; + measurement: string; + physicalBytes: number | null; + }[]; + }; + }; + expect(summary.storageAudit).toBeNull(); + expect(summary.storageInventory.entries.length).toBeGreaterThan(0); + for (const entry of summary.storageInventory.entries) { + expect(entry.physicalBytes).toBeNull(); + expect(entry.measurement).toBe("unavailable"); + expect(entry.health).toBe("unavailable"); + } + }); + test("run79 mutateExtension enables disables and sets mode with audit receipts", async () => { const runtimeStateRoot = path.join(os.tmpdir(), `track-b-run79-mutate-${Date.now()}`); roots.push(runtimeStateRoot); diff --git a/role-model-router/apps/runtime-host-bridge/test/validate-tools.test.ts b/role-model-router/apps/runtime-host-bridge/test/validate-tools.test.ts index 95f24751..a9ffa25b 100644 --- a/role-model-router/apps/runtime-host-bridge/test/validate-tools.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/validate-tools.test.ts @@ -33,18 +33,12 @@ describe("runRuntimeToolsValidation", () => { }, ]); expect(result.toolExecutions).toEqual([]); + // Run 94 (SP2): the persisted observation is a compact identity stub; tool payloads + // are graph-external and no longer ride the inline SQLite row. expect(result.observation).toMatchObject({ requestId: result.requestId, endpointId: result.endpointId, - tooling: { - toolCalls: [ - { - toolCallId: "call_1", - toolName: "lookupRegistry", - }, - ], - executions: [], - }, }); + expect(result.observation).not.toHaveProperty("tooling"); }); }); diff --git a/role-model-router/apps/runtime-host-bridge/test/validate-vendors.test.ts b/role-model-router/apps/runtime-host-bridge/test/validate-vendors.test.ts index 2980f9c3..3d2115b1 100644 --- a/role-model-router/apps/runtime-host-bridge/test/validate-vendors.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/validate-vendors.test.ts @@ -29,12 +29,24 @@ describe("runRuntimeVendorValidation", () => { const runtimeStateRoot = await mkdtemp(path.join(os.tmpdir(), "role-model-runtime-vendors-")); tempRoots.push(runtimeStateRoot); - const result = await runRuntimeVendorValidation({ - repoRoot, - runtimeStateRoot, - scopeId: "runtime-vendor-validation", - harnessMode: "mock", - }); + const callerOpenAiApiKey = process.env.OPENAI_API_KEY; + delete process.env.OPENAI_API_KEY; + let result: Awaited>; + try { + result = await runRuntimeVendorValidation({ + repoRoot, + runtimeStateRoot, + scopeId: "runtime-vendor-validation", + harnessMode: "mock", + }); + expect(process.env.OPENAI_API_KEY).toBeUndefined(); + } finally { + if (callerOpenAiApiKey === undefined) { + delete process.env.OPENAI_API_KEY; + } else { + process.env.OPENAI_API_KEY = callerOpenAiApiKey; + } + } expect(result.decisionOnly).toEqual( expect.objectContaining({ diff --git a/role-model-router/apps/runtime-ui/app/lib/runtime-api.ts b/role-model-router/apps/runtime-ui/app/lib/runtime-api.ts index 49f62445..507a2561 100644 --- a/role-model-router/apps/runtime-ui/app/lib/runtime-api.ts +++ b/role-model-router/apps/runtime-ui/app/lib/runtime-api.ts @@ -1768,13 +1768,24 @@ export async function prepareKnowledgeWorkerShadowReady( export interface RuntimeStorageRetentionSummary { readonly revision: number; readonly totalBytes: number; - readonly categories: readonly { + readonly logicalClasses: readonly { readonly id: string; readonly tier: string; readonly scope: string; readonly bytes: number; readonly count: number; }[]; + /** Backward-compatible alias for callers that still use the pre-SP8 name. */ + readonly categories: RuntimeStorageRetentionSummary["logicalClasses"]; + readonly physicalResources: readonly { + readonly id: string; + readonly owner: string; + readonly health: string; + readonly measurement: "measured" | "unavailable"; + readonly physicalBytes: number | null; + readonly heldItems: number; + readonly retentionState: string; + }[]; readonly managedPolicy: boolean; readonly conflicts: readonly { readonly serviceId?: string; @@ -1823,21 +1834,69 @@ export interface RuntimeStorageRetentionSummary { readonly retentionState: string; }[]; }; + // Run 94 SP8: measured physical accounting from the read-only storage audit plus + // the measured policy state; absent until a real measurement exists. + readonly storageAudit?: { + readonly schemaVersion: "role-model.storage-audit.v1"; + readonly available?: boolean; + readonly reason?: string; + readonly allocatedBytes?: number; + readonly logicalBytes?: number; + readonly reclaimableBytes?: number; + readonly heldBytes?: number | null; + readonly unavailableBytes?: number; + readonly observationRows?: number; + readonly graphEdges?: number; + readonly measuredAt?: string; + } | null; + readonly policyState?: { + readonly channel: string; + readonly state: string; + readonly policyId?: string; + readonly scopedBytes?: number; + readonly maxBytes?: number; + readonly policyCount?: number; + readonly error?: string; + }; } export async function fetchStorageRetention( fetcher: RuntimeFetcher = fetch, ): Promise { - return fetchJson("/api/role-model/storage-retention", fetcher); + return normalizeStorageRetentionSummary( + await fetchJson("/api/role-model/storage-retention", fetcher), + ); +} + +function normalizeStorageRetentionSummary( + value: RuntimeStorageRetentionSummary, +): RuntimeStorageRetentionSummary { + const raw = value as RuntimeStorageRetentionSummary & { + readonly logicalClasses?: RuntimeStorageRetentionSummary["logicalClasses"]; + readonly physicalResources?: RuntimeStorageRetentionSummary["physicalResources"]; + }; + const logicalClasses = raw.logicalClasses ?? raw.categories ?? []; + const physicalResources = raw.physicalResources ?? raw.storageInventory?.entries ?? []; + return { + ...raw, + logicalClasses, + categories: logicalClasses, + physicalResources, + storageInventory: raw.storageInventory + ? { ...raw.storageInventory, entries: physicalResources } + : undefined, + }; } export async function requestRetentionDryRun( fetcher: RuntimeFetcher = fetch, ): Promise { - return fetchJson( - "/api/role-model/storage-retention/dry-run", - fetcher, - { method: "POST" }, + return normalizeStorageRetentionSummary( + await fetchJson( + "/api/role-model/storage-retention/dry-run", + fetcher, + { method: "POST" }, + ), ); } @@ -1850,14 +1909,16 @@ export async function updateRetentionPolicy( }, fetcher: RuntimeFetcher = fetch, ): Promise { - return fetchJson( - "/api/role-model/storage-retention/policy", - fetcher, - { - method: "PUT", - headers: { "content-type": "application/json" }, - body: JSON.stringify(policy), - }, + return normalizeStorageRetentionSummary( + await fetchJson( + "/api/role-model/storage-retention/policy", + fetcher, + { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify(policy), + }, + ), ); } export async function executeRetentionPlan( @@ -1865,37 +1926,43 @@ export async function executeRetentionPlan( scope: string, fetcher: RuntimeFetcher = fetch, ): Promise { - return fetchJson( - "/api/role-model/storage-retention/execute", - fetcher, - { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ manifestHash, scope }), - }, + return normalizeStorageRetentionSummary( + await fetchJson( + "/api/role-model/storage-retention/execute", + fetcher, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ manifestHash, scope }), + }, + ), ); } export async function cancelRetentionJob( fetcher: RuntimeFetcher = fetch, ): Promise { - return fetchJson( - "/api/role-model/storage-retention/cancel", - fetcher, - { method: "POST" }, + return normalizeStorageRetentionSummary( + await fetchJson( + "/api/role-model/storage-retention/cancel", + fetcher, + { method: "POST" }, + ), ); } export async function rollbackRetentionReceipt( receiptId: string, fetcher: RuntimeFetcher = fetch, ): Promise { - return fetchJson( - "/api/role-model/storage-retention/rollback", - fetcher, - { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ receiptId }), - }, + return normalizeStorageRetentionSummary( + await fetchJson( + "/api/role-model/storage-retention/rollback", + fetcher, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ receiptId }), + }, + ), ); } export interface RuntimeContributionState { diff --git a/role-model-router/apps/runtime-ui/app/routes/run94-storage-contract-red.test.tsx b/role-model-router/apps/runtime-ui/app/routes/run94-storage-contract-red.test.tsx new file mode 100644 index 00000000..bdbea4fb --- /dev/null +++ b/role-model-router/apps/runtime-ui/app/routes/run94-storage-contract-red.test.tsx @@ -0,0 +1,15 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, test } from "vitest"; + +describe("Run 94 public storage contract", () => { + test("RED: names physical resources and logical classes as separate API/UI surfaces", () => { + const api = readFileSync(new URL("../lib/runtime-api.ts", import.meta.url), "utf8"); + const route = readFileSync(new URL("./storage-retention.tsx", import.meta.url), "utf8"); + expect(api).toContain("physicalResources"); + expect(api).toContain("logicalClasses"); + expect(route).toContain("physicalResources"); + expect(route).toContain("logicalClasses"); + expect(route).toContain("Physical resources"); + expect(route).toContain("Logical classes"); + }); +}); diff --git a/role-model-router/apps/runtime-ui/app/routes/storage-retention.test.tsx b/role-model-router/apps/runtime-ui/app/routes/storage-retention.test.tsx index df51ab7d..55955da2 100644 --- a/role-model-router/apps/runtime-ui/app/routes/storage-retention.test.tsx +++ b/role-model-router/apps/runtime-ui/app/routes/storage-retention.test.tsx @@ -32,7 +32,8 @@ describe("StorageRetentionRoute", () => { "fieldClassName", "Maximum size (GB)", "gbInputToBytes", - "Physical storage inventory", + "Physical resources", + "Logical classes", "row.physicalBytes", "row.heldItems", "row.retentionState", @@ -55,8 +56,8 @@ describe("StorageRetentionRoute", () => { test("renders the existing design-system loading and manual pruning controls", () => { const html = renderToStaticMarkup(); - expect(html).toContain("Tracked"); - expect(html).toContain("Physical stores"); + expect(html).toContain("Physical resources"); + expect(html).toContain("Logical classes"); expect(html).toContain("Legal holds"); expect(html).toContain("Loading storage inventory"); expect(html).toContain("Retention policy"); diff --git a/role-model-router/apps/runtime-ui/app/routes/storage-retention.tsx b/role-model-router/apps/runtime-ui/app/routes/storage-retention.tsx index 723c2997..70bb4cb9 100644 --- a/role-model-router/apps/runtime-ui/app/routes/storage-retention.tsx +++ b/role-model-router/apps/runtime-ui/app/routes/storage-retention.tsx @@ -101,21 +101,54 @@ export function StorageRetentionRouteView() { variant="panel" items={[ { - id: "usage", - label: "Tracked", - value: String(formatBytes(summary?.totalBytes ?? 0)), + id: "physical", + label: "Physical", + value: + summary?.storageAudit?.allocatedBytes != null + ? String(formatBytes(summary.storageAudit.allocatedBytes)) + : "Not measured", }, { - id: "classes", - label: "Physical stores", - value: String(summary?.storageInventory?.entries.length ?? 0), + id: "physical-resources", + label: "Physical resources", + value: String(summary?.physicalResources.length ?? 0), + }, + { + id: "logical-classes", + label: "Logical classes", + value: + summary?.storageAudit?.logicalBytes != null + ? String(formatBytes(summary.storageAudit.logicalBytes)) + : String(formatBytes(summary?.totalBytes ?? 0)), + }, + { + id: "reclaimable", + label: "Reclaimable", + value: + summary?.storageAudit?.reclaimableBytes != null + ? String(formatBytes(summary.storageAudit.reclaimableBytes)) + : "Not measured", + }, + { + id: "unavailable", + label: "Unavailable", + value: + summary?.storageAudit?.unavailableBytes != null + ? String(formatBytes(summary.storageAudit.unavailableBytes)) + : "Not measured", }, { id: "holds", label: "Legal holds", - value: String( - summary?.storageInventory?.entries.reduce((sum, row) => sum + row.heldItems, 0) ?? 0, - ), + value: + summary?.storageAudit === null || summary?.storageAudit === undefined + ? "Not measured" + : String( + summary?.storageInventory?.entries.reduce( + (sum, row) => sum + row.heldItems, + 0, + ) ?? 0, + ), }, { id: "maintenance", @@ -125,7 +158,7 @@ export function StorageRetentionRouteView() { ]} /> {error ? : null} - {summary?.storageInventory ? ( + {summary?.physicalResources ? ( - {summary.storageInventory.entries.map((row) => ( + {summary.physicalResources.map((row) => ( {row.id} {row.owner} @@ -157,7 +190,9 @@ export function StorageRetentionRouteView() { {row.physicalBytes === null ? "Unavailable" : formatBytes(row.physicalBytes)} {row.heldItems} - {row.retentionState} + + {summary.policyState ? summary.policyState.state : row.retentionState} + ))} @@ -187,7 +222,7 @@ export function StorageRetentionRouteView() { - {summary.categories.map((row) => ( + {summary.logicalClasses.map((row) => ( { + test("contribution surface exposes the lifecycle from preview to revocation", async ({ + page, + }) => { + const before = await page.request.get("/api/role-model/contribution"); + expect(before.ok()).toBeTruthy(); + const initial = (await before.json()) as { + authorizationState: string; + queuedCount: number; + revocationEpoch: number; + mode: string; + contributionTier: string; + recommendationTier: string; + recommendationAccess: string; + allowCloudUpload: boolean; + managed: boolean; + disclosureId: string | null; + nextSequence: number; + }; + expect(initial.authorizationState).toEqual(expect.any(String)); + expect(initial.queuedCount).toEqual(expect.any(Number)); + expect(initial.revocationEpoch).toEqual(expect.any(Number)); + expect(initial.mode).toEqual(expect.any(String)); + expect(initial.recommendationAccess).toEqual(expect.any(String)); + + // Destination-scoped authorization: completing disclosure with an invalid + // disclosure identity must be refused before any authorization state change. + const unauthorized = await page.request.put("/api/role-model/contribution", { + data: { action: "complete_disclosure", disclosureId: "!!bad disclosure id!!" }, + }); + expect(unauthorized.ok()).toBe(false); + + const stillPending = (await ( + await page.request.get("/api/role-model/contribution") + ).json()) as { authorizationState: string }; + expect(["pending_disclosure", "active", "revoked"]).toContain(stillPending.authorizationState); + + // Revocation clears any queued rows and bumps the epoch. + const revoked = await page.request.put("/api/role-model/contribution", { + data: { action: "opt_out" }, + }); + expect(revoked.ok()).toBeTruthy(); + const revokedBody = (await revoked.json()) as { queuedCount: number; revocationEpoch: number }; + expect(revokedBody.queuedCount).toBe(0); + expect(revokedBody.revocationEpoch).toBeGreaterThanOrEqual(initial.revocationEpoch); + + const reenabled = await page.request.put("/api/role-model/contribution", { + data: { action: "reenable" }, + }); + expect(reenabled.ok()).toBeTruthy(); + const reenabledBody = (await reenabled.json()) as { authorizationState: string }; + expect(["pending_disclosure", "active", "revoked"]).toContain(reenabledBody.authorizationState); + + await page.goto("/app/system/extensions"); + await expect( + page.getByText(/disclosure|authorization|preview|outbox|revocation/i).first(), + ).toBeVisible(); + }); +}); diff --git a/role-model-router/apps/runtime-ui/e2e/recursive-94-direct-track-b-storage-graph-cloud-roundtrip.sp8.storage-ui.spec.ts b/role-model-router/apps/runtime-ui/e2e/recursive-94-direct-track-b-storage-graph-cloud-roundtrip.sp8.storage-ui.spec.ts new file mode 100644 index 00000000..be8364f7 --- /dev/null +++ b/role-model-router/apps/runtime-ui/e2e/recursive-94-direct-track-b-storage-graph-cloud-roundtrip.sp8.storage-ui.spec.ts @@ -0,0 +1,56 @@ +import { expect, test } from "@playwright/test"; + +test.describe("@recursive:94-direct-track-b-storage-graph-cloud-roundtrip @sp8 @smoke", () => { + test("storage UI shows measured physical accounting and honest policy state", async ({ + page, + }) => { + const inventoryResponse = await page.request.get("/api/role-model/storage-retention"); + expect(inventoryResponse.ok()).toBeTruthy(); + const inventory = (await inventoryResponse.json()) as { + totalBytes: number; + storageAudit: { + schemaVersion: string; + available?: boolean; + allocatedBytes?: number; + logicalBytes?: number; + reclaimableBytes?: number; + unavailableBytes?: number; + heldBytes?: number | null; + measuredAt?: string; + } | null; + policyState: { channel: string; state: string }; + storageInventory: { + entries: Array<{ measurement: string; physicalBytes: number | null; health: string }>; + }; + }; + + // Honest accounting: every unmeasured entry stays unavailable with null bytes. + expect(Array.isArray(inventory.storageInventory.entries)).toBe(true); + for (const row of inventory.storageInventory.entries) { + if (row.measurement === "measured") expect(typeof row.physicalBytes).toBe("number"); + else expect(row.physicalBytes).toBeNull(); + } + + // The read-only audit either reports measured bytes or is honestly absent. + if (inventory.storageAudit && inventory.storageAudit.available !== false) { + expect(inventory.storageAudit.schemaVersion).toBe("role-model.storage-audit.v1"); + expect(inventory.storageAudit.allocatedBytes).toEqual(expect.any(Number)); + expect(inventory.storageAudit.measuredAt).toEqual(expect.any(String)); + } + + // Policy state is measured, never fabricated. + expect(["enforced", "absent", "violation", "unavailable", "enforcement_failure"]).toContain( + inventory.policyState.state, + ); + + await page.goto("/app/system/storage-retention"); + await expect(page.getByRole("heading", { name: "Physical storage inventory" })).toBeVisible(); + const summary = page.getByLabel("Storage retention summary"); + await expect(summary.getByText("Physical", { exact: true })).toBeVisible(); + await expect(summary.getByText("Logical", { exact: true })).toBeVisible(); + await expect(summary.getByText("Reclaimable", { exact: true })).toBeVisible(); + await expect(summary.getByText("Unavailable", { exact: true })).toBeVisible(); + await expect(summary.getByText("Legal holds")).toBeVisible(); + await expect(page.getByRole("columnheader", { name: "Enforcement" })).toBeVisible(); + }); +}); diff --git a/role-model-router/migrations/0002_compact_stub_enforcement.sql b/role-model-router/migrations/0002_compact_stub_enforcement.sql new file mode 100644 index 00000000..ee13d765 --- /dev/null +++ b/role-model-router/migrations/0002_compact_stub_enforcement.sql @@ -0,0 +1,17 @@ +-- 0002_compact_stub_enforcement.sql +-- Run 94 (Direct Track B v1.1): enforce the 16 KiB compact-stub invariant on the +-- inline runtime_observations.observation_json column. Existing rows are untouched +-- (legacy rows remain exempt until SP6 retirement rewrites them as compact pointers); +-- new inserts or updates carrying rich inline content fail closed at the schema layer. +CREATE TRIGGER IF NOT EXISTS runtime_observations_compact_stub_enforcement +BEFORE INSERT ON runtime_observations +WHEN NEW.observation_json IS NOT NULL AND length(CAST(NEW.observation_json AS BLOB)) > 16384 +BEGIN + SELECT RAISE(ABORT, 'runtime_observations.observation_json exceeds the 16 KiB compact stub cap'); +END; +CREATE TRIGGER IF NOT EXISTS runtime_observations_compact_stub_update_enforcement +BEFORE UPDATE OF observation_json ON runtime_observations +WHEN NEW.observation_json IS NOT NULL AND length(CAST(NEW.observation_json AS BLOB)) > 16384 +BEGIN + SELECT RAISE(ABORT, 'runtime_observations.observation_json exceeds the 16 KiB compact stub cap'); +END; diff --git a/role-model-router/migrations/registry.json b/role-model-router/migrations/registry.json index ded029a1..fb61ebf4 100644 --- a/role-model-router/migrations/registry.json +++ b/role-model-router/migrations/registry.json @@ -10,6 +10,16 @@ "idempotencyStrategy": "create-if-not-exists-and-source-id-primary-key", "rollbackClass": "reversible", "postconditionQuery": "SELECT COUNT(*) = 3 AS valid FROM sqlite_master WHERE type = 'table' AND name IN ('legacy_migration_journal','legacy_graph_migration_refs','normalized_performance_samples_v2')" + }, + { + "migrationId": "run94-compact-stub-enforcement-v1", + "sqlFile": "0002_compact_stub_enforcement.sql", + "owner": "runtime-storage", + "sha256": "b0e11f67521dee4db650511baf8042a7a7c878bed74b6e0e1c409a34bec2d146", + "transactionMode": "idempotent_guarded", + "idempotencyStrategy": "create-if-not-exists-insert-and-update-triggers", + "rollbackClass": "reversible", + "postconditionQuery": "SELECT COUNT(*) = 2 AS valid FROM sqlite_master WHERE type = 'trigger' AND name IN ('runtime_observations_compact_stub_enforcement','runtime_observations_compact_stub_update_enforcement')" } ] } diff --git a/role-model-router/packages/process-supervisor/src/index.ts b/role-model-router/packages/process-supervisor/src/index.ts index 686226ad..8373be40 100644 --- a/role-model-router/packages/process-supervisor/src/index.ts +++ b/role-model-router/packages/process-supervisor/src/index.ts @@ -24,6 +24,11 @@ export interface StartVendorOptions { readonly required: boolean; } +export interface ProcessSupervisorOptions { + /** Base environment for supervised children. Defaults to the owning runtime process environment. */ + readonly baseEnvironment?: Readonly>; +} + export interface ManagedVendorStatus { readonly vendorId: string; readonly pid: number; @@ -164,11 +169,14 @@ function beginChildLifecycle(record: ManagedVendorRecord, child: ChildProcess): }); } -function spawnVendorProcess(record: ManagedVendorRecord): ChildProcess { +function spawnVendorProcess( + record: ManagedVendorRecord, + baseEnvironment: Readonly>, +): ChildProcess { const child = spawn(record.options.command, [...(record.options.args ?? [])], { cwd: record.options.cwd, env: { - ...process.env, + ...baseEnvironment, ...record.options.env, }, stdio: ["ignore", "pipe", "pipe"], @@ -220,6 +228,12 @@ export class ProcessSupervisor { private readonly crashListeners: VendorCrashListener[] = []; + private readonly baseEnvironment: Readonly>; + + constructor(options: ProcessSupervisorOptions = {}) { + this.baseEnvironment = options.baseEnvironment ?? process.env; + } + onVendorCrash(listener: VendorCrashListener): void { this.crashListeners.push(listener); } @@ -274,7 +288,7 @@ export class ProcessSupervisor { record.stopping = false; this.managedVendors.set(options.vendorId, record); - const child = spawnVendorProcess(record); + const child = spawnVendorProcess(record, this.baseEnvironment); child.once("exit", (exitCode) => { void this.handleUnexpectedExit(record, exitCode); }); @@ -384,7 +398,7 @@ export class ProcessSupervisor { } try { - const child = spawnVendorProcess(record); + const child = spawnVendorProcess(record, this.baseEnvironment); child.once("exit", (exitCode) => { void this.handleUnexpectedExit(record, exitCode); }); diff --git a/role-model-router/packages/process-supervisor/test/index.test.ts b/role-model-router/packages/process-supervisor/test/index.test.ts index e7af8823..6ebf9ef2 100644 --- a/role-model-router/packages/process-supervisor/test/index.test.ts +++ b/role-model-router/packages/process-supervisor/test/index.test.ts @@ -116,6 +116,44 @@ describe("process-supervisor", () => { expect(supervisor.getVendorStatus("mock-health")?.port).toBe(port); }); + test("can run supervised vendors without inheriting provider credential variables", async () => { + const port = await allocatePort(); + const markerPath = path.join( + await mkdtemp(path.join(os.tmpdir(), "role-model-supervisor-env-")), + "credential.txt", + ); + tempRoots.push(path.dirname(markerPath)); + const previousCredential = process.env.RUN94_OPENAI_API_KEY; + process.env.RUN94_OPENAI_API_KEY = "redacted-test-secret"; + const childEnvironment = { ...process.env }; + delete childEnvironment.RUN94_OPENAI_API_KEY; + const supervisor = new ProcessSupervisor({ baseEnvironment: childEnvironment }); + supervisors.push(supervisor); + + try { + await supervisor.startVendor({ + vendorId: "credential-stripped", + command: process.execPath, + args: [ + "-e", + `const fs=require("node:fs");const http=require("node:http");fs.writeFileSync(${JSON.stringify(markerPath)},process.env.RUN94_OPENAI_API_KEY??"");const server=http.createServer((req,res)=>{res.statusCode=req.url==="/healthz"?200:404;res.end(req.url==="/healthz"?"ok":"missing");});server.listen(Number(process.env.PORT),"127.0.0.1");const shutdown=()=>server.close(()=>process.exit(0));process.on("SIGTERM",shutdown);process.on("SIGINT",shutdown);`, + ], + env: { PORT: String(port) }, + healthCheckUrl: `http://127.0.0.1:${port}/healthz`, + startupTimeoutMs: 5000, + required: true, + }); + + expect(await readFile(markerPath, "utf8")).toBe(""); + } finally { + if (previousCredential === undefined) { + delete process.env.RUN94_OPENAI_API_KEY; + } else { + process.env.RUN94_OPENAI_API_KEY = previousCredential; + } + } + }); + test("tracks crash callbacks when a managed vendor exits unexpectedly", async () => { const port = await allocatePort(); const supervisor = new ProcessSupervisor(); diff --git a/role-model-router/packages/runtime-observability/src/index.ts b/role-model-router/packages/runtime-observability/src/index.ts index 86f92e64..6e1dd61f 100644 --- a/role-model-router/packages/runtime-observability/src/index.ts +++ b/role-model-router/packages/runtime-observability/src/index.ts @@ -482,7 +482,6 @@ export interface RuntimeObservationBundle { readonly observedPerformance: { readonly endpointVersion: string; readonly sample: ObservedPerformanceSample; - readonly history: readonly ObservedPerformanceSample[]; readonly profile: ObservedPerformanceProfile; }; readonly diagnostics: { @@ -598,7 +597,6 @@ export interface RuntimeObservationBundle { readonly endpointId: string; readonly endpointVersion: string; readonly latestProfile: ObservedPerformanceProfile; - readonly recentSamples: readonly ObservedPerformanceSample[]; }; }; } @@ -1098,7 +1096,6 @@ export function createRuntimeObservationBundle( observedPerformance: { endpointVersion, sample: currentSample, - history, profile, }, diagnostics, @@ -1166,7 +1163,6 @@ export function createRuntimeObservationBundle( endpointId: input.decision.chosen_endpoint_id, endpointVersion, latestProfile: profile, - recentSamples: history, }, }, }; diff --git a/role-model-router/packages/runtime-observability/test/run94-bundle-compactness.test.ts b/role-model-router/packages/runtime-observability/test/run94-bundle-compactness.test.ts new file mode 100644 index 00000000..d2f836f2 --- /dev/null +++ b/role-model-router/packages/runtime-observability/test/run94-bundle-compactness.test.ts @@ -0,0 +1,68 @@ +import { mkdtemp } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { expect, test } from "vitest"; + +import { runRuntimeAdapterValidation } from "@role-model-router/adapter-execution/cli"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const repoRoot = path.resolve(__dirname, "..", "..", "..", ".."); + +test("run94 F7: bundle output never carries cumulative history or recentSamples", async () => { + const moduleImport = import(pathToFileURL(path.join(__dirname, "..", "src", "index.js")).href); + const runtimeObservability = (await moduleImport) as { + createRuntimeObservationBundle(input: Record): Record; + }; + const runtimeStateRoot = await mkdtemp(path.join(os.tmpdir(), "run94-bundle-compact-")); + const validation = await runRuntimeAdapterValidation({ + repoRoot, + fixtureRoot: path.join(repoRoot, "testdata", "router-runtime", "fixtures"), + runtimeStateRoot, + scopeId: "run94-bundle-compact", + }); + const priorSamples = [ + { + endpoint_id: validation.decision.chosen_endpoint_id, + endpoint_version: "legacy", + source_type: "live_request", + timestamp_ms: Date.now() - 60_000, + latency_ms: 300, + success: true, + }, + { + endpoint_id: validation.decision.chosen_endpoint_id, + endpoint_version: "legacy", + source_type: "live_request", + timestamp_ms: Date.now() - 30_000, + latency_ms: 200, + success: true, + }, + ]; + const bundle = runtimeObservability.createRuntimeObservationBundle({ + decision: validation.decision, + routingDiagnostics: validation.routingDiagnostics, + retrievalReceipt: validation.retrievalReceipt, + contextEnvelope: validation.contextEnvelope, + execution: validation.execution, + priorSamples, + maintenancePolicy: {}, + capturePolicy: { + environment: "development", + redactionLevel: "strict", + retentionClass: "standard", + structuredInspectionMode: "summary", + }, + }) as { + observedPerformance: Record; + inspection: { endpoint: Record }; + }; + expect(bundle.observedPerformance).not.toHaveProperty("history"); + expect(bundle.inspection.endpoint).not.toHaveProperty("recentSamples"); + expect(JSON.stringify(bundle)).not.toContain("recentSamples"); + expect(JSON.stringify(bundle)).not.toContain('"history"'); + // The aggregate profile is still computed for the immediate UI update. + expect(bundle.observedPerformance.profile).toBeTruthy(); +}); diff --git a/role-model-router/packages/sqlite-memory/src/index.ts b/role-model-router/packages/sqlite-memory/src/index.ts index 8180117f..56f410ed 100644 --- a/role-model-router/packages/sqlite-memory/src/index.ts +++ b/role-model-router/packages/sqlite-memory/src/index.ts @@ -31,6 +31,8 @@ import { writePerformanceHistoryPolicy, } from "./history-policy.js"; import { + LEGACY_INLINE_CAP_BYTES, + buildCompactRuntimeObservationStub, hydrateRuntimeObservationGraphPointer, readRuntimeObservationStorageState, recordRuntimeObservationGraphReference, @@ -325,6 +327,18 @@ CREATE TABLE IF NOT EXISTS runtime_observations ( retain_until_ms INTEGER, observation_json TEXT NOT NULL ); +CREATE TRIGGER IF NOT EXISTS runtime_observations_compact_stub_enforcement +BEFORE INSERT ON runtime_observations +WHEN NEW.observation_json IS NOT NULL AND length(CAST(NEW.observation_json AS BLOB)) > 16384 +BEGIN + SELECT RAISE(ABORT, 'runtime_observations.observation_json exceeds the 16 KiB compact stub cap'); +END; +CREATE TRIGGER IF NOT EXISTS runtime_observations_compact_stub_update_enforcement +BEFORE UPDATE OF observation_json ON runtime_observations +WHEN NEW.observation_json IS NOT NULL AND length(CAST(NEW.observation_json AS BLOB)) > 16384 +BEGIN + SELECT RAISE(ABORT, 'runtime_observations.observation_json exceeds the 16 KiB compact stub cap'); +END; CREATE TABLE IF NOT EXISTS observed_performance_samples ( sample_id TEXT PRIMARY KEY, endpoint_id TEXT NOT NULL, @@ -4598,6 +4612,8 @@ export interface PersistRuntimeTelemetryFailureInput { readonly taxonomyToolClassIds?: readonly string[]; readonly dimensions?: Record | null; readonly observation?: Record | null; + readonly artifactRef?: import("./legacy-migration.js").GraphArtifactReference; + readonly graphStore?: import("./legacy-migration.js").RuntimeObservationGraphStore; } export function persistRuntimeTelemetryFailure(input: PersistRuntimeTelemetryFailureInput): void { @@ -4610,32 +4626,88 @@ export function persistRuntimeTelemetryFailure(input: PersistRuntimeTelemetryFai endpointId, createdAtMs, ); - withSqliteBusyRetry(input.databasePath, (database) => { - if (input.observation) { + let artifactRef = input.artifactRef; + let createdArtifact: import("./legacy-migration.js").LegacyArtifactWriteResult | undefined; + if (input.observation && input.graphStore && !artifactRef) { + const content = JSON.stringify(input.observation); + const contentHash = createHash("sha256").update(content).digest("hex"); + createdArtifact = input.graphStore.write({ + scopeId: input.graphStore.scopeId, + sourceId: input.requestId, + content, + contentHash, + }); + artifactRef = { scopeId: input.graphStore.scopeId, ...createdArtifact }; + } + try { + withSqliteBusyRetry(input.databasePath, (database) => { + if (input.observation) { + const sourceObservation = + input.observation && typeof input.observation === "object" + ? (input.observation as Readonly>) + : null; + const stub = sourceObservation + ? buildCompactRuntimeObservationStub(sourceObservation) + : { requestId: input.requestId }; + stub.requestId = input.requestId; + stub.statusFamily = "failure"; + stub.failure = { + statusCode: input.statusCode, + errorClass: input.errorClass, + ...(input.latencyMs != null ? { latencyMs: input.latencyMs } : {}), + }; + if (artifactRef) { + stub.artifactRef = artifactRef; + stub.graphPrimary = true; + } + // Failure rows are classification stubs. Diagnostics and inspection captures may + // contain provider errors or raw response bodies, so they remain graph/artifact + // content and are never copied into this SQLite row. + const payload = JSON.stringify(stub); + if (Buffer.byteLength(payload, "utf8") > LEGACY_INLINE_CAP_BYTES) { + throw new Error( + `runtime telemetry failure classification stub exceeds ${LEGACY_INLINE_CAP_BYTES} bytes`, + ); + } + database + .prepare( + "INSERT OR REPLACE INTO runtime_observations (request_id, routing_decision_id, endpoint_id, conversation_id, created_at_ms, retain_until_ms, taxonomy_role_id, taxonomy_task_type, client_request_id, request_class, observation_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run( + input.requestId, + routingDecisionId, + endpointId, + "conversation-main", + createdAtMs, + input.retainUntil ?? null, + input.taxonomyRoleId ?? null, + input.taxonomyTaskType ?? null, + input.clientRequestId ?? null, + input.requestClass ?? null, + payload, + ); + if (artifactRef) { + recordRuntimeObservationGraphReference(database, { + observation: { + ...sourceObservation, + requestId: input.requestId, + }, + artifactRef, + }); + } + } database .prepare( - "INSERT OR REPLACE INTO runtime_observations (request_id, routing_decision_id, endpoint_id, conversation_id, created_at_ms, retain_until_ms, taxonomy_role_id, taxonomy_task_type, client_request_id, request_class, observation_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + `INSERT OR REPLACE INTO runtime_telemetry_records (${RUNTIME_TELEMETRY_INSERT_COLUMNS.join(", ")}) VALUES (${RUNTIME_TELEMETRY_INSERT_COLUMNS.map(() => "?").join(", ")})`, ) - .run( - input.requestId, - routingDecisionId, - endpointId, - "conversation-main", - createdAtMs, - input.retainUntil ?? null, - input.taxonomyRoleId ?? null, - input.taxonomyTaskType ?? null, - input.clientRequestId ?? null, - input.requestClass ?? null, - JSON.stringify(input.observation), - ); + .run(...runtimeTelemetryInsertValues(telemetryRecord)); + }); + } catch (error) { + if (createdArtifact) { + input.graphStore?.remove?.(createdArtifact); } - database - .prepare( - `INSERT OR REPLACE INTO runtime_telemetry_records (${RUNTIME_TELEMETRY_INSERT_COLUMNS.join(", ")}) VALUES (${RUNTIME_TELEMETRY_INSERT_COLUMNS.map(() => "?").join(", ")})`, - ) - .run(...runtimeTelemetryInsertValues(telemetryRecord)); - }); + throw error; + } } export function readRuntimeObservationBundle( diff --git a/role-model-router/packages/sqlite-memory/src/legacy-migration.ts b/role-model-router/packages/sqlite-memory/src/legacy-migration.ts index 5764d202..328bfc0a 100644 --- a/role-model-router/packages/sqlite-memory/src/legacy-migration.ts +++ b/role-model-router/packages/sqlite-memory/src/legacy-migration.ts @@ -49,6 +49,9 @@ export interface LegacyStorageAudit { readonly sourceHash: string; readonly externalizationCandidateCount: number; readonly inlineCapBytes: number; + readonly quarantinedPointerRows: number; + readonly quarantinedRows: number; + readonly quarantinedRequestIds: readonly string[]; } export interface GraphArtifactReference { @@ -215,6 +218,44 @@ function tableExists(database: DatabaseSync, table: string): boolean { ); } +type LegacyRowClassification = + | { readonly kind: "import" } + | { readonly kind: "quarantine"; readonly reason: "malformed_json" | "unresolved_graph_pointer" }; + +function classifyLegacyRow(observationJson: string): LegacyRowClassification { + try { + const parsed = JSON.parse(observationJson); + return isGraphObservationPointer(parsed) + ? { kind: "quarantine", reason: "unresolved_graph_pointer" } + : { kind: "import" }; + } catch (error) { + if (error instanceof SyntaxError) return { kind: "quarantine", reason: "malformed_json" }; + throw error; + } +} + +function ensureQuarantineTable(database: DatabaseSync): void { + database.exec(` + CREATE TABLE IF NOT EXISTS legacy_migration_quarantine ( + source_table TEXT NOT NULL, + source_id TEXT NOT NULL, + source_hash TEXT NOT NULL, + reason TEXT NOT NULL, + created_at_ms INTEGER NOT NULL, + PRIMARY KEY (source_table, source_id) + ); + `); +} + +function readQuarantine(database: DatabaseSync): Array<{ source_id: string; reason: string }> { + if (!tableExists(database, "legacy_migration_quarantine")) return []; + return database + .prepare( + "SELECT source_id, reason FROM legacy_migration_quarantine WHERE source_table=? ORDER BY source_id", + ) + .all("runtime_observations") as Array<{ source_id: string; reason: string }>; +} + function sourceProof(database: DatabaseSync, pageSize = 1_000): { count: number; hash: string } { if (!tableExists(database, "runtime_observations")) return { count: 0, hash: sha256("") }; const digest = createHash("sha256"); @@ -238,12 +279,12 @@ function sourceProof(database: DatabaseSync, pageSize = 1_000): { count: number; }>; if (!rows.length) break; for (const row of rows) { - if (count) digest.update("\n"); let rowHash = sha256(row.observation_json); try { if (isGraphObservationPointer(JSON.parse(row.observation_json))) { - if (!row.source_hash) - throw new Error("graph observation pointer is missing its source hash"); + // Run 94 SP6: a pointer-shaped row without a matching migration ref is + // unclassified residue — quarantined and excluded from parity inputs. + if (!row.source_hash) continue; rowHash = row.source_hash; } } catch (error) { @@ -253,6 +294,7 @@ function sourceProof(database: DatabaseSync, pageSize = 1_000): { count: number; throw error; } } + if (count) digest.update("\n"); digest.update(`${row.request_id}\0${rowHash}`); count += 1; } @@ -334,27 +376,247 @@ export function readLegacyMigrationJournal(databasePath: string): LegacyMigratio } } +/** + * Builds the <=16 KiB compact identity stub persisted inline in SQLite. + * Rich content (messages, response bodies, tool payloads, captures, cumulative + * history/recentSamples) is NEVER included: it is graph-external by contract. + */ +export function buildCompactRuntimeObservationStub( + observation: Readonly>, +): Record { + const requestId = observation.requestId; + if (typeof requestId !== "string" || requestId.length === 0) { + throw new Error("runtime observation request ID required"); + } + const stub: Record = { requestId }; + const pickRecord = (value: unknown, keys: readonly string[]): Record => { + if (!value || typeof value !== "object" || Array.isArray(value)) return {}; + const source = value as Record; + return Object.fromEntries( + keys.flatMap((key) => (source[key] === undefined ? [] : [[key, source[key]] as const])), + ); + }; + const usageEvent = pickRecord(observation.usageEvent, [ + "timestamp_ms", + "request_id", + "routing_decision_id", + "endpoint_id", + "model_id", + "provider_kind", + "tokens_in", + "tokens_out", + "latency_ms", + "cost_actual", + "cost_estimate", + "currency", + "error_class", + ]); + const cacheObservability = pickRecord(observation.cacheObservability, [ + "promptCacheRequested", + "promptCacheRequestSource", + "promptCacheUsed", + "cacheReadTokens", + "cacheWriteTokens", + "routingCacheAffinity", + ]); + const executionTelemetry = pickRecord(observation.executionTelemetry, [ + "providerFamily", + "vendorId", + "finishReason", + "costProvenance", + ]); + const executionSemantics = pickRecord(observation.executionSemantics, [ + "sourceClient", + "executionFamily", + "adapterFamily", + "statusFamily", + "toolSideEffectState", + "idempotencyDecision", + ]); + const failedAttempts = Array.isArray( + (observation.executionSemantics as Record | undefined)?.failedAttempts, + ) + ? ((observation.executionSemantics as Record).failedAttempts as unknown[]) + .slice(0, 8) + .filter((attempt): attempt is Record => + Boolean(attempt && typeof attempt === "object" && !Array.isArray(attempt)), + ) + .map((attempt) => ({ + ...pickRecord(attempt, [ + "attemptId", + "routedAttemptId", + "requestId", + "routingDecisionId", + "failedEndpointId", + "providerId", + "providerFamily", + "vendorId", + "executionFamily", + "adapterFamily", + "statusCode", + "failureClass", + "retryable", + "fallbackEligible", + "failurePhase", + "cooldownRecorded", + "cooldownFailureCount", + "cooldownUntilMs", + ]), + ...(attempt.errorPreview && typeof attempt.errorPreview === "object" + ? { + errorPreview: pickRecord(attempt.errorPreview, [ + "message", + "statusCode", + "errorClass", + ]), + } + : {}), + })) + : []; + const payloadBytes = pickRecord( + (observation.executionSemantics as Record | undefined)?.payloadBytes, + ["ingress", "translated", "providerCanonical", "providerWire", "providerResponse"], + ); + for (const [key, value] of [ + ["clientRequestId", observation.clientRequestId], + ["routingDecisionId", observation.routingDecisionId], + ["endpointId", observation.endpointId], + ["reasoningEffort", observation.reasoningEffort], + ["effortSource", observation.effortSource], + ["conversationId", observation.conversationId], + ] as const) { + if (value !== undefined) stub[key] = value; + } + if (Object.keys(usageEvent).length) stub.usageEvent = usageEvent; + if (Object.keys(cacheObservability).length) stub.cacheObservability = cacheObservability; + if (Object.keys(executionTelemetry).length) stub.executionTelemetry = executionTelemetry; + if (Object.keys(executionSemantics).length) { + stub.executionSemantics = { + ...executionSemantics, + ...(Object.keys(payloadBytes).length ? { payloadBytes } : {}), + ...(failedAttempts.length ? { failedAttempts } : {}), + }; + } + const contextEnvelope = pickRecord(observation.contextEnvelope, [ + "conversationId", + "latestHandoffId", + "estimatedTokenCount", + ]); + if (Object.keys(contextEnvelope).length) stub.contextEnvelope = contextEnvelope; + const retrievalReceipt = pickRecord(observation.retrievalReceipt, ["receiptId", "summary"]); + if (Object.keys(retrievalReceipt).length) stub.retrievalReceipt = retrievalReceipt; + const capturePolicy = pickRecord(observation.capturePolicy, [ + "environment", + "redactionLevel", + "retentionClass", + "structuredInspectionMode", + "rawCaptureAvailable", + "structuredInspectionAvailable", + ]); + if (Object.keys(capturePolicy).length) stub.capturePolicy = capturePolicy; + const privacyReceipt = pickRecord(observation.privacyReceipt, [ + "samplingRate", + "retentionTtlHours", + "retainUntil", + ]); + if (Object.keys(privacyReceipt).length) stub.privacyReceipt = privacyReceipt; + const taxonomyDimensions = pickRecord(observation.taxonomyDimensions, [ + "taxonomy_group_id", + "taxonomy_role_id", + "taxonomy_task_type", + "taxonomy_task_variant", + "taxonomy_capability_ids", + "taxonomy_modality_ids", + "taxonomy_tool_class_ids", + ]); + if (Object.keys(taxonomyDimensions).length) stub.taxonomyDimensions = taxonomyDimensions; + const providerEvidence = pickRecord(observation.providerEvidence, [ + "endpointId", + "modelId", + "status", + "attemptIds", + ]); + if (Object.keys(providerEvidence).length) stub.providerEvidence = providerEvidence; + const graphEvidence = pickRecord(observation.graphEvidence, [ + "rootArtifactId", + "messageNodeIds", + "responseNodeId", + "edgeCount", + ]); + if (Object.keys(graphEvidence).length) stub.graphEvidence = graphEvidence; + const run88Correlation = pickRecord(observation.run88Correlation, [ + "schemaVersion", + "correlationId", + "requestId", + "routingDecisionId", + "endpointId", + "releaseId", + "sourceId", + "deploymentId", + "scope", + ]); + if (Object.keys(run88Correlation).length) stub.run88Correlation = run88Correlation; + // SAFETY: observation is a persisted-bundle JSON payload at the storage I/O boundary; + // observedPerformance is an optional object-shaped field of that payload. + const observed = observation.observedPerformance as Record | undefined; + const sample = pickRecord(observed?.sample, [ + "endpoint_id", + "endpoint_version", + "model_id", + "source_type", + "timestamp_ms", + "latency_ms", + "success", + "input_tokens", + "output_tokens", + ]); + const profile = pickRecord(observed?.profile, [ + "measured_at_ms", + "sample_count", + "success_rate", + "latency_ms", + "throughput_tokens_per_sec", + "quality_score", + ]); + const endpointVersion = observed?.endpointVersion; + if (Object.keys(sample).length || Object.keys(profile).length || endpointVersion !== undefined) { + const compactObserved: Record = {}; + if (endpointVersion !== undefined) compactObserved.endpointVersion = endpointVersion; + if (Object.keys(sample).length) compactObserved.sample = sample; + if (Object.keys(profile).length) compactObserved.profile = profile; + stub.observedPerformance = compactObserved; + } + return stub; +} + export function resolveRuntimeObservationStoragePayload(input: { readonly databasePath: string; readonly observation: Readonly>; readonly artifactRef?: GraphArtifactReference; }): string { - const database = open(input.databasePath, true); - let state: LegacyMigrationState; - try { - state = currentState(database); - } finally { - database.close(); - } - if (state === "graph_primary" || state === "legacy_read_hold" || state === "legacy_retired") { - if (!input.artifactRef) throw new Error("graph artifact reference required after cutover"); - const requestId = input.observation.requestId; - if (typeof requestId !== "string" || requestId.length === 0) { - throw new Error("runtime observation request ID required"); + if (!input.artifactRef) { + // Fail closed: content we are about to drop from the inline stub must already + // be graph-externalized. Callers with rich observations must provide the + // artifact reference (or write a bounded degradation stub instead). + const fullJson = JSON.stringify(input.observation); + if (Buffer.byteLength(fullJson, "utf8") > LEGACY_INLINE_CAP_BYTES) { + throw new Error( + "rich runtime observation requires graph externalization; no graph artifact reference available", + ); } - return JSON.stringify({ requestId, artifactRef: input.artifactRef, graphPrimary: true }); } - return JSON.stringify(input.observation); + const stub = buildCompactRuntimeObservationStub(input.observation); + if (input.artifactRef) { + stub.artifactRef = input.artifactRef; + stub.graphPrimary = true; + } + const json = JSON.stringify(stub); + if (Buffer.byteLength(json, "utf8") > LEGACY_INLINE_CAP_BYTES) { + throw new Error( + `runtime observation compact stub exceeds ${LEGACY_INLINE_CAP_BYTES} bytes; rich content must be graph-externalized`, + ); + } + return json; } export function readRuntimeObservationStorageState(databasePath: string): LegacyMigrationState { @@ -444,6 +706,37 @@ export class LegacySqliteMigration { const database = open(this.#databasePath, true); try { const proof = sourceProof(database); + const rows = database + .prepare( + tableExists(database, "legacy_graph_migration_refs") + ? `SELECT observations.request_id, observations.observation_json, + refs.source_id AS imported_source_id + FROM runtime_observations AS observations + LEFT JOIN legacy_graph_migration_refs AS refs + ON refs.source_table='runtime_observations' AND refs.source_id=observations.request_id + ORDER BY observations.request_id ASC` + : `SELECT request_id, observation_json, NULL AS imported_source_id + FROM runtime_observations ORDER BY request_id ASC`, + ) + .all() as Array<{ + request_id: string; + observation_json: string; + imported_source_id: string | null; + }>; + const quarantined = rows.flatMap((row) => + row.imported_source_id + ? [] + : classifyLegacyRow(row.observation_json).kind === "quarantine" + ? [row] + : [], + ); + const persistedQuarantine = readQuarantine(database); + const quarantinedIds = [ + ...new Set([ + ...quarantined.map((row) => row.request_id), + ...persistedQuarantine.map((row) => row.source_id), + ]), + ].sort(); return { state: currentState(database), sourceRowCount: proof.count, @@ -458,6 +751,12 @@ export class LegacySqliteMigration { ).count, ), inlineCapBytes: LEGACY_INLINE_CAP_BYTES, + // Run 94 SP6: pointer-shaped rows without a matching migration ref are + // unclassified residue — quarantined, reported, and excluded from parity + // inputs; they are never assumed migrated. + quarantinedPointerRows: quarantined.length, + quarantinedRows: quarantinedIds.length, + quarantinedRequestIds: quarantinedIds, }; } finally { database.close(); @@ -515,6 +814,7 @@ export class LegacySqliteMigration { "utf8", ); database.exec(migrationSql); + ensureQuarantineTable(database); const postcondition = database.prepare(entry.postconditionQuery).get() as | { valid?: number } | undefined; @@ -536,11 +836,28 @@ export class LegacySqliteMigration { .prepare( `SELECT request_id, observation_json FROM runtime_observations WHERE request_id NOT IN (SELECT source_id FROM legacy_graph_migration_refs) + AND request_id NOT IN (SELECT source_id FROM legacy_migration_quarantine WHERE source_table='runtime_observations') ORDER BY request_id ASC LIMIT ?`, ) .all(input.batchSize) as Array<{ request_id: string; observation_json: string }>; let migratedCount = 0; for (const row of rows) { + const classification = classifyLegacyRow(row.observation_json); + if (classification.kind === "quarantine") { + database + .prepare( + `INSERT OR REPLACE INTO legacy_migration_quarantine + (source_table, source_id, source_hash, reason, created_at_ms) VALUES (?, ?, ?, ?, ?)`, + ) + .run( + "runtime_observations", + row.request_id, + sha256(row.observation_json), + classification.reason, + this.#now(), + ); + continue; + } const contentHash = sha256(row.observation_json); const artifact = this.#artifactWriter({ scopeId: input.scopeId, @@ -602,7 +919,9 @@ export class LegacySqliteMigration { ( database .prepare( - "SELECT COUNT(*) AS count FROM runtime_observations WHERE request_id NOT IN (SELECT source_id FROM legacy_graph_migration_refs)", + `SELECT COUNT(*) AS count FROM runtime_observations + WHERE request_id NOT IN (SELECT source_id FROM legacy_graph_migration_refs) + AND request_id NOT IN (SELECT source_id FROM legacy_migration_quarantine WHERE source_table='runtime_observations')`, ) .get() as { count: number } ).count, @@ -628,11 +947,16 @@ export class LegacySqliteMigration { const pending = ( database .prepare( - "SELECT COUNT(*) AS count FROM runtime_observations WHERE request_id NOT IN (SELECT source_id FROM legacy_graph_migration_refs)", + `SELECT COUNT(*) AS count FROM runtime_observations + WHERE request_id NOT IN (SELECT source_id FROM legacy_graph_migration_refs) + AND request_id NOT IN (SELECT source_id FROM legacy_migration_quarantine WHERE source_table='runtime_observations')`, ) .get() as { count: number } ).count; + const quarantined = readQuarantine(database); if (pending !== 0) throw new Error("backfill remains incomplete"); + if (quarantined.length > 0) + throw new Error("legacy migration quarantine blocks shadow mirror"); // Persist the live dual-write window so a restart cannot silently extend it. this.#setState(database, "shadow_mirror", { holdUntilMs: input.deadlineMs }); } finally { @@ -652,6 +976,8 @@ export class LegacySqliteMigration { try { if (currentState(database) !== "shadow_mirror") throw new Error("shadow mirror required before parity"); + const quarantined = readQuarantine(database); + if (quarantined.length > 0) throw new Error("legacy migration quarantine blocks parity"); const journal = readLegacyMigrationJournal(this.#databasePath); if (journal.holdUntilMs === null || this.#now() > journal.holdUntilMs) { throw new Error("shadow mirror deadline expired; restart backfill before parity"); diff --git a/role-model-router/packages/sqlite-memory/test/index.test.ts b/role-model-router/packages/sqlite-memory/test/index.test.ts index a7620194..c3b5c2de 100644 --- a/role-model-router/packages/sqlite-memory/test/index.test.ts +++ b/role-model-router/packages/sqlite-memory/test/index.test.ts @@ -1298,7 +1298,7 @@ describe("initializeSqliteMemory", () => { "conversation-projected", 1000, "client-projected", - "not-json-and-intentionally-large".repeat(100_000), + "not-json-and-intentionally-large".repeat(500), ); database.close(); diff --git a/role-model-router/packages/sqlite-memory/test/legacy-migration.test.ts b/role-model-router/packages/sqlite-memory/test/legacy-migration.test.ts index c1df8192..93ff974e 100644 --- a/role-model-router/packages/sqlite-memory/test/legacy-migration.test.ts +++ b/role-model-router/packages/sqlite-memory/test/legacy-migration.test.ts @@ -28,6 +28,10 @@ function fixture() { }); const backupPath = path.join(root, "backup.sqlite"); const database = new DatabaseSync(initialized.databasePath); + // Run 94 (SP2): the fixture simulates a legacy_primary database created before the + // compact-stub enforcement trigger existed. Drop the trigger so pre-invariant rich + // rows can be planted exactly as they exist in installed runtimes. + database.exec("DROP TRIGGER IF EXISTS runtime_observations_compact_stub_enforcement"); const rich = JSON.stringify({ requestId: "request-1", endpointId: "endpoint-1", @@ -312,32 +316,35 @@ describe("TB04 real SQLite legacy migration", () => { }); migration.cutover(); + // Run 94 (SP2): the inline invariant is now state-independent. Rich content without + // a graph artifact reference fails closed on size, and the compact stub never + // carries non-identity fields. expect(() => resolveRuntimeObservationStoragePayload({ databasePath, - observation: { requestId: "request-new", providerBody: "secret-rich-body" }, + observation: { requestId: "request-new", providerBody: "x".repeat(20_000) }, }), - ).toThrow("graph artifact reference required"); + ).toThrow(/graph externalization/i); expect( - resolveRuntimeObservationStoragePayload({ - databasePath, - observation: { requestId: "request-new", providerBody: "secret-rich-body" }, - artifactRef: { - scopeId: "scope-1", - artifactId: "artifact-new", - contentHash: "content-hash-new", - }, - }), - ).toEqual( - JSON.stringify({ - requestId: "request-new", - artifactRef: { - scopeId: "scope-1", - artifactId: "artifact-new", - contentHash: "content-hash-new", - }, - graphPrimary: true, - }), - ); + JSON.parse( + resolveRuntimeObservationStoragePayload({ + databasePath, + observation: { requestId: "request-new", providerBody: "secret-rich-body" }, + artifactRef: { + scopeId: "scope-1", + artifactId: "artifact-new", + contentHash: "content-hash-new", + }, + }), + ), + ).toEqual({ + requestId: "request-new", + artifactRef: { + scopeId: "scope-1", + artifactId: "artifact-new", + contentHash: "content-hash-new", + }, + graphPrimary: true, + }); }); }); diff --git a/role-model-router/packages/sqlite-memory/test/run94-compact-stub.test.ts b/role-model-router/packages/sqlite-memory/test/run94-compact-stub.test.ts new file mode 100644 index 00000000..eb587d68 --- /dev/null +++ b/role-model-router/packages/sqlite-memory/test/run94-compact-stub.test.ts @@ -0,0 +1,253 @@ +import { mkdtemp } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { expect, test } from "vitest"; + +import { + initializeSqliteMemory, + persistRuntimeObservationBundle, + persistRuntimeTelemetryFailure, +} from "../src/index.js"; + +function buildRichObservation(overrides: Record = {}): Record { + return { + requestId: "req-run94-rich", + routingDecisionId: "decision-run94-rich", + endpointId: "endpoint-run94-rich", + conversationId: "conversation-run94-rich", + reasoningEffort: "medium", + effortSource: "variant", + usageEvent: { + timestamp_ms: Date.now(), + request_id: "req-run94-rich", + routing_decision_id: "decision-run94-rich", + endpoint_id: "endpoint-run94-rich", + model_id: "model/run94", + provider_kind: "remote", + tokens_in: 4, + tokens_out: 3, + latency_ms: 900, + cost_actual: 0.02, + currency: "USD", + error_class: undefined, + }, + observedPerformance: { + endpointVersion: "v1", + sample: { + endpoint_id: "endpoint-run94-rich", + endpoint_version: "v1", + source_type: "live_request", + timestamp_ms: Date.now(), + latency_ms: 900, + success: true, + }, + history: [ + { endpoint_id: "endpoint-run94-rich", latency_ms: 800, timestamp_ms: Date.now() - 1 }, + ], + profile: { measured_at_ms: Date.now(), sampleCount: 2 }, + }, + decision: { + chosen_endpoint_id: "endpoint-run94-rich", + candidate_snapshots: [{ big: "x".repeat(20000) }], + }, + trace: { spans: ["s".repeat(20000)] }, + diagnostics: { execution: { preview: "y".repeat(20000) } }, + tooling: { executions: [{ toolName: "shell", output: "z".repeat(20000) }] }, + inspection: { + request: { + requestId: "req-run94-rich", + requestCapture: { messages: [{ role: "user", content: "m".repeat(30000) }] }, + responseCapture: { outputText: "o".repeat(30000) }, + }, + endpoint: { + endpointId: "endpoint-run94-rich", + recentSamples: [{ latency_ms: 1 }], + }, + }, + executionSemantics: { + sourceClient: "openai.responses", + adapterFamily: "ai-sdk-openai-compatible", + payloadBytes: { providerResponse: 12345 }, + }, + capturePolicy: { environment: "development", redactionLevel: "strict" }, + privacyReceipt: { + samplingRate: 1, + retentionTtlHours: 720, + retainUntil: Date.now() + 720 * 3600 * 1000, + }, + taxonomyDimensions: { taxonomy_role_id: "coder", taxonomy_task_type: "edit" }, + providerEvidence: { + endpointId: "endpoint-run94-rich", + modelId: "model/run94", + status: "ok", + attemptIds: ["req-run94-rich:attempt:1"], + }, + graphEvidence: { + rootArtifactId: "root-run94-rich", + messageNodeIds: ["message-run94-rich"], + responseNodeId: "response-run94-rich", + edgeCount: 2, + }, + run88Correlation: { + schemaVersion: "run88-correlation.v1", + correlationId: "correlation-run94-rich", + requestId: "req-run94-rich", + routingDecisionId: "decision-run94-rich", + endpointId: "endpoint-run94-rich", + releaseId: `sha256:${"a".repeat(64)}`, + sourceId: "b".repeat(40), + deploymentId: `local-development:${"c".repeat(64)}`, + scope: "tenant:run94", + }, + ...overrides, + }; +} + +test("run94 F1: persistRuntimeObservationBundle writes a <=16KiB compact stub with no journal present when graph content is externalized", async () => { + const runtimeStateRoot = await mkdtemp(path.join(os.tmpdir(), "run94-compact-stub-")); + const initialized = initializeSqliteMemory({ + runtimeStateRoot, + scopeId: "run94-compact", + channel: "development", + }); + const artifactRef = { + scopeId: "tenant:run94", + artifactId: "artifact-req-run94-rich", + contentHash: "sha256:rich", + }; + persistRuntimeObservationBundle({ + databasePath: initialized.databasePath, + channel: "development", + observation: buildRichObservation() as never, + artifactRef, + }); + const database = new DatabaseSync(initialized.databasePath); + const row = database + .prepare("SELECT observation_json FROM runtime_observations WHERE request_id=?") + .get("req-run94-rich") as { observation_json: string }; + database.close(); + expect(Buffer.byteLength(row.observation_json, "utf8")).toBeLessThanOrEqual(16 * 1024); + const stub = JSON.parse(row.observation_json) as Record; + expect(stub.requestId).toBe("req-run94-rich"); + expect(stub.artifactRef).toEqual(artifactRef); + expect(stub.graphPrimary).toBe(true); + expect(stub.usageEvent.tokens_in).toBe(4); + expect(row.observation_json).not.toContain("recentSamples"); + expect(row.observation_json).not.toContain("history"); + expect(row.observation_json).not.toContain("requestCapture"); + // payloadBytes.providerResponse is a bounded byte count and must survive as compact telemetry. + expect(stub.executionSemantics.payloadBytes.providerResponse).toBe(12345); + expect(stub.providerEvidence).toEqual({ + endpointId: "endpoint-run94-rich", + modelId: "model/run94", + status: "ok", + attemptIds: ["req-run94-rich:attempt:1"], + }); + expect(stub.graphEvidence).toEqual({ + rootArtifactId: "root-run94-rich", + messageNodeIds: ["message-run94-rich"], + responseNodeId: "response-run94-rich", + edgeCount: 2, + }); + expect(stub.run88Correlation).toMatchObject({ correlationId: "correlation-run94-rich" }); + expect(row.observation_json).not.toContain("m".repeat(30000)); + expect(stub.observedPerformance).not.toHaveProperty("history"); + expect(stub.inspection).toBeUndefined(); + expect(stub.tooling).toBeUndefined(); +}); + +test("run94 F1: schema guard rejects accidental rich inline persistence beyond 16 KiB", async () => { + const runtimeStateRoot = await mkdtemp(path.join(os.tmpdir(), "run94-compact-guard-")); + const initialized = initializeSqliteMemory({ + runtimeStateRoot, + scopeId: "run94-compact-guard", + channel: "development", + }); + const database = new DatabaseSync(initialized.databasePath); + expect(() => + database + .prepare( + "INSERT INTO runtime_observations (request_id, routing_decision_id, endpoint_id, conversation_id, created_at_ms, observation_json) VALUES (?, ?, ?, ?, ?, ?)", + ) + .run( + "req-guard", + "decision-guard", + "endpoint-guard", + "conversation-guard", + Date.now(), + JSON.stringify({ payload: "x".repeat(20000) }), + ), + ).toThrow(/16 KiB|compact|limit/i); + database.close(); +}); + +test("run94 F1: persistRuntimeObservationBundle fails closed instead of writing rich content inline without graph externalization", async () => { + const runtimeStateRoot = await mkdtemp(path.join(os.tmpdir(), "run94-compact-failclosed-")); + const initialized = initializeSqliteMemory({ + runtimeStateRoot, + scopeId: "run94-compact-failclosed", + channel: "development", + }); + expect(() => + persistRuntimeObservationBundle({ + databasePath: initialized.databasePath, + channel: "development", + observation: buildRichObservation() as never, + }), + ).toThrow(/graph|artifact|compact|16 KiB/i); +}); + +test("run94 F1: persistRuntimeTelemetryFailure writes a bounded classification stub, never the original observation", async () => { + const runtimeStateRoot = await mkdtemp(path.join(os.tmpdir(), "run94-compact-telemetry-")); + const initialized = initializeSqliteMemory({ + runtimeStateRoot, + scopeId: "run94-compact-telemetry", + channel: "development", + }); + persistRuntimeTelemetryFailure({ + databasePath: initialized.databasePath, + requestId: "req-run94-failure", + routingDecisionId: "decision-run94-failure", + endpointId: "endpoint-run94-failure", + statusCode: 500, + errorClass: "provider_timeout", + latencyMs: 1200, + requestClass: "live_request", + observation: { + requestId: "req-run94-failure", + routingDecisionId: "decision-run94-failure", + endpointId: "endpoint-run94-failure", + conversationId: "conversation-main", + statusFamily: "failure", + usageEvent: { + request_id: "req-run94-failure", + timestamp_ms: Date.now(), + tokens_in: 0, + tokens_out: 0, + latency_ms: 1200, + error_class: "provider_timeout", + }, + providerResponse: "f".repeat(30000), + messages: [{ role: "user", content: "g".repeat(30000) }], + executionSemantics: { + adapterFamily: "ai-sdk-openai-compatible", + failedAttempts: [ + { failedEndpointId: "endpoint-run94-failure", failureClass: "provider_timeout" }, + ], + }, + }, + }); + const database = new DatabaseSync(initialized.databasePath); + const row = database + .prepare("SELECT observation_json FROM runtime_observations WHERE request_id=?") + .get("req-run94-failure") as { observation_json: string }; + database.close(); + expect(Buffer.byteLength(row.observation_json, "utf8")).toBeLessThanOrEqual(16 * 1024); + expect(row.observation_json).not.toContain("providerResponse"); + expect(row.observation_json).not.toContain("messages"); + const stub = JSON.parse(row.observation_json) as Record; + expect(stub.requestId).toBe("req-run94-failure"); + expect(stub.statusFamily).toBe("failure"); + expect(stub.failure).toMatchObject({ statusCode: 500, errorClass: "provider_timeout" }); +}); diff --git a/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts b/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts new file mode 100644 index 00000000..5c367da3 --- /dev/null +++ b/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts @@ -0,0 +1,115 @@ +import { mkdtempSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, test } from "vitest"; + +import { + LegacySqliteMigration, + initializeSqliteMemory, + persistRuntimeTelemetryFailure, + readLegacyMigrationJournal, +} from "../src/index.js"; + +function newDatabase() { + const root = mkdtempSync(path.join(os.tmpdir(), "run94-public-corrections-")); + const initialized = initializeSqliteMemory({ + runtimeStateRoot: root, + scopeId: "run94-public-corrections", + channel: "development", + }); + return { root, databasePath: initialized.databasePath }; +} + +function insertObservation(databasePath: string, requestId: string, observationJson: string) { + const database = new DatabaseSync(databasePath); + database + .prepare( + `INSERT INTO runtime_observations + (request_id, routing_decision_id, endpoint_id, conversation_id, created_at_ms, observation_json) + VALUES (?, ?, ?, ?, ?, ?)`, + ) + .run( + requestId, + `decision-${requestId}`, + "endpoint-run94", + "conversation-run94", + 1, + observationJson, + ); + database.close(); +} + +describe("Run 94 public storage corrections", () => { + test("RED: enforces the 16 KiB cap by UTF-8 bytes on UPDATE, not SQLite characters", () => { + const { databasePath } = newDatabase(); + insertObservation(databasePath, "req-utf8", JSON.stringify({ requestId: "req-utf8" })); + const database = new DatabaseSync(databasePath); + expect(() => + database + .prepare("UPDATE runtime_observations SET observation_json=? WHERE request_id=?") + .run(JSON.stringify({ requestId: "req-utf8", payload: "🙂".repeat(8_500) }), "req-utf8"), + ).toThrow(/16 KiB|compact|limit/i); + database.close(); + }); + + test("RED: classifies malformed legacy rows as quarantine and blocks shadow cutover", () => { + const { databasePath, root } = newDatabase(); + const database = new DatabaseSync(databasePath); + database.exec("DROP TRIGGER IF EXISTS runtime_observations_compact_stub_enforcement"); + database.close(); + insertObservation(databasePath, "req-malformed", "{not-json"); + + const migration = new LegacySqliteMigration({ + databasePath, + backupPath: path.join(root, "backup.sqlite"), + artifactWriter: ({ sourceId, contentHash }) => ({ + artifactId: `artifact-${sourceId}`, + artifactPath: `artifact://${sourceId}`, + contentHash, + }), + }); + + migration.backfill({ scopeId: "run94-public-corrections", batchSize: 10 }); + expect(migration.audit().quarantinedRequestIds).toContain("req-malformed"); + expect(readLegacyMigrationJournal(databasePath).state).toBe("backfill"); + expect(() => migration.enterShadowMirror({ deadlineMs: Date.now() + 10_000 })).toThrow( + /quarantine|classified|parity/i, + ); + }); + + test("failure classification keeps routing fields but never persists raw diagnostics or response bodies", () => { + const { databasePath } = newDatabase(); + persistRuntimeTelemetryFailure({ + databasePath, + requestId: "req-failure-redaction", + statusCode: 502, + errorClass: "provider_error", + observation: { + requestId: "req-failure-redaction", + routingDecisionId: "decision-failure-redaction", + endpointId: "endpoint-run94", + executionSemantics: { + adapterFamily: "openai-compatible", + failedAttempts: [ + { + failedEndpointId: "endpoint-run94", + failureClass: "provider_error", + errorPreview: { message: "safe classification" }, + }, + ], + }, + diagnostics: { secret: "do-not-store" }, + inspection: { request: { responseCapture: { body: "raw-provider-body" } } }, + }, + }); + const database = new DatabaseSync(databasePath); + const row = database + .prepare("SELECT observation_json FROM runtime_observations WHERE request_id=?") + .get("req-failure-redaction") as { observation_json: string }; + database.close(); + expect(row.observation_json).toContain("failedEndpointId"); + expect(row.observation_json).not.toContain("do-not-store"); + expect(row.observation_json).not.toContain("raw-provider-body"); + }); +}); diff --git a/scripts/track-b/storage-audit.mjs b/scripts/track-b/storage-audit.mjs new file mode 100644 index 00000000..3605f838 --- /dev/null +++ b/scripts/track-b/storage-audit.mjs @@ -0,0 +1,34 @@ +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +// Public forwarder for the run 94 productized read-only storage audit. +// The audit implementation lives in the paired internal worktree (same convention +// as run-command-test.mjs): resolve ROLE_MODEL_INTERNAL_ROOT or infer the sibling +// .worktrees/ checkout, then forward all arguments unchanged. +const publicRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const runName = path.basename(publicRoot); +const inferredInternalRoot = path.resolve( + publicRoot, + "../../..", + "role-model-internal", + ".worktrees", + runName, +); +const internalRoot = process.env.ROLE_MODEL_INTERNAL_ROOT + ? path.resolve(process.env.ROLE_MODEL_INTERNAL_ROOT) + : inferredInternalRoot; +const audit = path.join(internalRoot, "scripts", "track-b", "storage-audit.mjs"); +if (!existsSync(audit)) + throw new Error( + `Run 94 internal storage audit not found. Set ROLE_MODEL_INTERNAL_ROOT (resolved: ${internalRoot})`, + ); +const args = process.argv.slice(2).filter((arg) => arg !== "--"); +const result = spawnSync(process.execPath, [audit, ...args], { + cwd: internalRoot, + stdio: "inherit", + env: process.env, +}); +if (result.error) throw result.error; +process.exit(result.status ?? 1); From 7709f83d947e5e4418e3d44b9d83d1c58c081cb9 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:18:08 +0800 Subject: [PATCH 02/40] Fix shadow rollout scoring provenance --- .../apps/runtime-host-bridge/src/track-b-runtime.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts index 9903ed8e..682f856b 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts @@ -1879,12 +1879,13 @@ export async function runTrackBShadowPipeline( })), }); const rolloutRows = [sourceRollout, ...counterfactualRollouts]; - const positive = rolloutRows.filter( - (rollout) => (rollout.outcome as Record | undefined)?.status === "success", - ); - const negative = rolloutRows.filter( - (rollout) => (rollout.outcome as Record | undefined)?.status !== "success", - ); + const scoredRollouts = rolloutRows.map((rollout) => ({ + ...rollout, + score: + (rollout.outcome as Record | undefined)?.status === "success" ? 1 : 0, + })); + const positive = scoredRollouts.filter((rollout) => rollout.score === 1); + const negative = scoredRollouts.filter((rollout) => rollout.score === 0); if (!positive.length || !negative.length) throw new Error( "R14_INSUFFICIENT_ROLLOUT_EVIDENCE: positive and negative rollout evidence is required", From 08df5b56a093a62c263e7676017274c0fa01a515 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:56:14 +0800 Subject: [PATCH 03/40] Preserve runtime channel in extension status --- .../apps/runtime-host-bridge/src/index.ts | 1 + .../src/track-b-operations.ts | 4 ++- .../recursive-87-registry-lifecycle.test.ts | 26 +++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index 9d96fb1c..e654041d 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -16480,6 +16480,7 @@ export async function createRuntimeBridgeBackend( const createTrackBOperations = (input: Parameters[0]) => createTrackBOperationsFromState({ ...input, + runtimeChannel, operationsEndpoint: options.trackBOperationsEndpoint, operationsToken: options.trackBOperationsToken, }); diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index 16183f2a..a09d7ac0 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -777,12 +777,14 @@ export function buildVerifiersLiveExport(input: { export function createTrackBOperations({ statePath, catalog, + runtimeChannel = "development", operationsEndpoint = process.env.ROLE_MODEL_TRACK_B_OPERATIONS_URL?.trim(), operationsToken = process.env.ROLE_MODEL_TRACK_B_OPERATIONS_TOKEN, extensionRuntime, }: { readonly statePath: string; readonly catalog: readonly Record[]; + readonly runtimeChannel?: "development" | "stage" | "production"; readonly operationsEndpoint?: string; readonly operationsToken?: string; readonly extensionRuntime?: { @@ -848,7 +850,7 @@ export function createTrackBOperations({ installed: true, enabled, enabledMode: enabled ? (id === "knowledge-worker" ? "shadow" : "active") : "disabled", - channel: "development", + channel: runtimeChannel, scope: "global", authorizationEpoch: 1, health: { diff --git a/role-model-router/apps/runtime-host-bridge/test/recursive-87-registry-lifecycle.test.ts b/role-model-router/apps/runtime-host-bridge/test/recursive-87-registry-lifecycle.test.ts index 8208a05a..fea296a2 100644 --- a/role-model-router/apps/runtime-host-bridge/test/recursive-87-registry-lifecycle.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/recursive-87-registry-lifecycle.test.ts @@ -222,6 +222,32 @@ describe("recursive run 87 SP0 registry and lifecycle authority", () => { expect(receipt).toMatchObject({ state: { pid: 8702, revision: 5 } }); }); + test("operations API preserves the packaged runtime channel for supervised extensions", async () => { + const stateRoot = path.join(os.tmpdir(), `run94-stage-extension-channel-${Date.now()}`); + roots.push(stateRoot); + const operations = createTrackBOperations({ + statePath: path.join(stateRoot, "bridge-state.json"), + catalog: [{ id: "artifact-store" }], + runtimeChannel: "stage", + extensionRuntime: { + listExtensions: async () => [ + { + id: "artifact-store", + lifecycle: "ready", + desiredState: "enabled", + pid: 9401, + revision: 1, + }, + ], + mutateExtension: async () => ({}), + }, + }); + + await expect(operations.listExtensions()).resolves.toEqual([ + expect.objectContaining({ id: "artifact-store", channel: "stage" }), + ]); + }); + test("Phase 3.5 journal replay preserves stopped state and never resurrects removed extensions", async () => { const stateRoot = path.join(os.tmpdir(), `run87-journal-replay-${Date.now()}`); roots.push(stateRoot); From 15d234e0983a066706787acf498157f722e93aab Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 01:21:28 +0800 Subject: [PATCH 04/40] Probe provider-native model identifiers --- .../apps/runtime-host-bridge/src/remote-health-probe.ts | 4 +++- .../test/remote-endpoint-admission-probe.test.ts | 2 +- .../test/remote-health-bootstrap.test.ts | 2 +- role-model-router/packages/provider-openai/src/index.ts | 6 +++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/remote-health-probe.ts b/role-model-router/apps/runtime-host-bridge/src/remote-health-probe.ts index 3e977b8f..dd749c25 100644 --- a/role-model-router/apps/runtime-host-bridge/src/remote-health-probe.ts +++ b/role-model-router/apps/runtime-host-bridge/src/remote-health-probe.ts @@ -1,3 +1,5 @@ +import { resolveOpenAIProviderUpstreamModelId } from "@role-model-router/provider-openai"; + export type RemoteHealthProbeReason = | "healthy" | "auth" @@ -337,7 +339,7 @@ export async function probeRemoteEndpointAdmission( const probeUrl = buildChatCompletionsProbeUrl(context.apiBase); const body = { - model: context.modelId, + model: resolveOpenAIProviderUpstreamModelId(context.modelId), messages: [{ role: "user", content: "role-model admission readiness probe" }], max_tokens: 1, stream: false, diff --git a/role-model-router/apps/runtime-host-bridge/test/remote-endpoint-admission-probe.test.ts b/role-model-router/apps/runtime-host-bridge/test/remote-endpoint-admission-probe.test.ts index 59713d91..562c5f51 100644 --- a/role-model-router/apps/runtime-host-bridge/test/remote-endpoint-admission-probe.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/remote-endpoint-admission-probe.test.ts @@ -41,7 +41,7 @@ describe("remote endpoint admission probes", () => { }), ); expect(JSON.parse(String(calls[0]?.init?.body))).toEqual({ - model: "deepseek/deepseek-v4-flash", + model: "deepseek-v4-flash", messages: [{ role: "user", content: "role-model admission readiness probe" }], max_tokens: 1, stream: false, diff --git a/role-model-router/apps/runtime-host-bridge/test/remote-health-bootstrap.test.ts b/role-model-router/apps/runtime-host-bridge/test/remote-health-bootstrap.test.ts index 6f4509b5..28a5c115 100644 --- a/role-model-router/apps/runtime-host-bridge/test/remote-health-bootstrap.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/remote-health-bootstrap.test.ts @@ -328,7 +328,7 @@ describe("remote health bootstrap", () => { const admissionCall = calls.find((call) => call.url.endsWith("/chat/completions")); expect(JSON.parse(String(admissionCall?.init?.body))).toEqual( expect.objectContaining({ - model: "deepseek/deepseek-v4-flash", + model: "deepseek-v4-flash", reasoning_effort: "high", }), ); diff --git a/role-model-router/packages/provider-openai/src/index.ts b/role-model-router/packages/provider-openai/src/index.ts index 156d7787..25e2f839 100644 --- a/role-model-router/packages/provider-openai/src/index.ts +++ b/role-model-router/packages/provider-openai/src/index.ts @@ -69,7 +69,7 @@ function getOpenAICapabilities( }; } -function resolveProviderLocalModelId(modelId: string): string { +export function resolveOpenAIProviderUpstreamModelId(modelId: string): string { const override = OPENAI_MODEL_REQUEST_POLICIES[modelId]?.upstreamModelId; if (override) { return override; @@ -920,7 +920,7 @@ export function buildOpenAIRequest( url: `${input.target.apiBase}/chat/completions`, headers, body: { - model: resolveProviderLocalModelId(input.target.modelId), + model: resolveOpenAIProviderUpstreamModelId(input.target.modelId), messages: toOpenAIInput(input.executionRequest.messages), ...(typeof input.executionRequest.temperature === "number" && !shouldOmitChatCompletionsBodyKey(input.target.modelId, "temperature") @@ -983,7 +983,7 @@ export function buildOpenAIRequest( "OpenAI-Beta": "responses=v1", }, body: { - model: resolveProviderLocalModelId(input.target.modelId), + model: resolveOpenAIProviderUpstreamModelId(input.target.modelId), input: hasResponsesToolReplayHistory(input.executionRequest.messages) ? toOpenAIResponsesInput(input.executionRequest.messages) : toOpenAIInput(input.executionRequest.messages), From f14a8a6bf816d6161153ec0587b57a440187cc98 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 01:46:40 +0800 Subject: [PATCH 05/40] Expose durable extension readback evidence --- .../apps/runtime-host-bridge/src/track-b-runtime.ts | 10 ++++++++++ .../runtime-host-bridge/test/run94-sp5-sp10.test.ts | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts index 682f856b..70839ada 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts @@ -1097,8 +1097,13 @@ export async function verifyTrackBExtensionClosureAfterRestart( const states = await runtime.listExtensions(); const results: Array<{ extensionId: string; + capability: string; durableOutputId: string; readbackOutputId: string; + durableLocator: unknown; + evidenceRef: string | null; + readCapability: string | null; + resultDigest: string; preRestartPid: number; postRestartPid: number; }> = []; @@ -1158,8 +1163,13 @@ export async function verifyTrackBExtensionClosureAfterRestart( throw new Error(`extension ${extensionId} durable output readback mismatch`); results.push({ extensionId, + capability: output.capability, durableOutputId: output.durableOutputId, readbackOutputId, + durableLocator: output.durableLocator, + evidenceRef: output.evidenceRef, + readCapability: output.readCapability, + resultDigest: output.resultDigest, preRestartPid, postRestartPid, }); diff --git a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts index 1a68f279..e9ff83dd 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts @@ -277,6 +277,15 @@ test("GREEN: real process output closure covers every canonical registry key and }); expect(readback.outputs.every((row) => row.readbackOutputId === row.durableOutputId)).toBe(true); expect(readback.outputs.every((row) => row.preRestartPid !== row.postRestartPid)).toBe(true); + expect( + readback.outputs.every( + (row) => + typeof row.capability === "string" && + row.capability.length > 0 && + row.durableLocator !== null && + row.resultDigest === row.durableOutputId, + ), + ).toBe(true); }); test("GREEN: rejects missing, health-only, and duplicate durable extension outputs", async () => { From 5e06d4af1fbc007e06ba98fdfc0e6b1bf7f6f389 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 02:09:02 +0800 Subject: [PATCH 06/40] Expose durable tool graph evidence --- .../runtime-host-bridge/src/track-b-operations.ts | 4 ++++ .../test/track-b-operations-api.test.ts | 13 +++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index a09d7ac0..83b4c0bf 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -672,6 +672,7 @@ export function buildGraphEvidenceFromCapture( capture: Readonly>, ): Readonly> { const messages = Array.isArray(capture.messages) ? capture.messages : []; + const tools = Array.isArray(capture.tools) ? capture.tools : []; const response = recordValue(capture.response); const edgeCount = Number(capture.edgeCount); if (messages.length < 1 || !Number.isSafeInteger(edgeCount) || edgeCount < 1) @@ -682,6 +683,9 @@ export function buildGraphEvidenceFromCapture( boundedIdentity(recordValue(message).nodeId, `graph message node ${index + 1}`), ), responseNodeId: boundedIdentity(response.nodeId, "graph response node id"), + toolExecutionNodeIds: tools.map((tool, index) => + boundedIdentity(recordValue(tool).nodeId, `graph tool execution node ${index + 1}`), + ), edgeCount, }); } diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 6e34e870..1b097182 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -14,6 +14,7 @@ import { LegacySqliteMigration } from "../../../packages/sqlite-memory/src/legac import { applyRecommendationServiceLauncherConfig } from "../src/cli.js"; import { createRuntimeBridgeBackend, startBridgeServer } from "../src/index.js"; import { + buildGraphEvidenceFromCapture, buildProviderEvidenceFromObservation, buildVerifiersLiveExport, createTrackBOperations, @@ -75,9 +76,16 @@ describe("Track B operations APIs", () => { { nodeId: "node-user-94", role: "user", content: "route this" }, ], response: { nodeId: "node-response-94", role: "assistant", content: "routed" }, - tools: [], - edgeCount: 3, + tools: [{ nodeId: "node-tool-94", toolName: "bash", output: "run94-tool-ok" }], + edgeCount: 4, }; + expect(buildGraphEvidenceFromCapture(capture)).toEqual({ + rootArtifactId: "root-export-94", + messageNodeIds: ["node-system-94", "node-user-94"], + responseNodeId: "node-response-94", + toolExecutionNodeIds: ["node-tool-94"], + edgeCount: 4, + }); const exported = buildVerifiersLiveExport({ channel: "development", request: { @@ -713,6 +721,7 @@ describe("Track B operations APIs", () => { rootArtifactId: "artifact-route-capture", messageNodeIds: ["message-route-capture-0"], responseNodeId: "response-route-capture", + toolExecutionNodeIds: [], edgeCount: 2, }); const correlationId = String( From d4299b6d1755a5596b6284249d8faa912ba18cec Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 02:35:46 +0800 Subject: [PATCH 07/40] Prove live tool graph and capture budgets --- .../apps/runtime-host-bridge/src/index.ts | 26 +++++- .../src/track-b-operations.ts | 59 +++++++++++++- .../test/track-b-operations-api.test.ts | 79 +++++++++++++++++-- 3 files changed, 152 insertions(+), 12 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index e654041d..0193d269 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -3387,6 +3387,10 @@ interface RuntimeBridgeSummary { endpointCount: number; scopeId: string; runtimeStateRoot: string; + processMetrics: { + pid: number; + rssBytes: number; + }; readinessSummary: { pendingDeviceAuthorizationCount: number; credentialsMissingAccountCount: number; @@ -24806,6 +24810,7 @@ export async function createRuntimeBridgeBackend( endpointCount: currentRegistry.endpoints.length, scopeId: options.scopeId, runtimeStateRoot: options.runtimeStateRoot, + processMetrics: { pid: process.pid, rssBytes: process.memoryUsage().rss }, readinessSummary: { pendingDeviceAuthorizationCount: credentialLifecycle.counts.pendingAuthorization, credentialsMissingAccountCount: credentialLifecycle.counts.credentialsMissing, @@ -27257,6 +27262,7 @@ export async function createRuntimeBridgeBackend( ): Promise => { let providerEvidence: Readonly> | undefined; let graphEvidence: Readonly> | undefined; + let liveBudgetEvidence: Readonly> | undefined; try { providerEvidence = buildProviderEvidenceFromObservation( value as unknown as Readonly>, @@ -27266,7 +27272,24 @@ export async function createRuntimeBridgeBackend( } try { const capture = await readExactRouteCapture(requestId); - if (capture) graphEvidence = buildGraphEvidenceFromCapture(capture); + if (capture) { + graphEvidence = buildGraphEvidenceFromCapture(capture); + const compactRecord = readRuntimeObservationStorageRecord({ + databasePath: initialization.databasePath, + requestId, + }); + const captureMetrics = + graphEvidence.captureMetrics && typeof graphEvidence.captureMetrics === "object" + ? (graphEvidence.captureMetrics as Readonly>) + : {}; + liveBudgetEvidence = Object.freeze({ + compactObservationBytes: compactRecord + ? Buffer.byteLength(JSON.stringify(compactRecord)) + : null, + runtimeRssBytes: process.memoryUsage().rss, + ...captureMetrics, + }); + } } catch { // Rich capture is best effort. Absence remains visible as missing graph evidence. } @@ -27274,6 +27297,7 @@ export async function createRuntimeBridgeBackend( ...value, ...(providerEvidence ? { providerEvidence } : {}), ...(graphEvidence ? { graphEvidence } : {}), + ...(liveBudgetEvidence ? { liveBudgetEvidence } : {}), } as BridgeRequestObservation; }; const telemetryRecord = (() => { diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index 83b4c0bf..7ddae46c 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -642,6 +642,12 @@ function recordValue(value: unknown): Readonly> { : {}; } +function finiteNonNegative(value: unknown, label: string): number { + const number = Number(value); + if (!Number.isFinite(number) || number < 0) throw new Error(`${label} must be finite and non-negative`); + return number; +} + export function buildProviderEvidenceFromObservation( observation: Readonly>, ): Readonly> { @@ -673,6 +679,24 @@ export function buildGraphEvidenceFromCapture( ): Readonly> { const messages = Array.isArray(capture.messages) ? capture.messages : []; const tools = Array.isArray(capture.tools) ? capture.tools : []; + const toolRows = tools.map((tool) => recordValue(tool)); + const rawCaptureMetrics = recordValue(capture.captureMetrics); + const captureMetrics = Object.keys(rawCaptureMetrics).length > 0 + ? Object.freeze({ + captureCpuMs: finiteNonNegative(rawCaptureMetrics.captureCpuMs, "capture CPU"), + captureWallMs: finiteNonNegative(rawCaptureMetrics.captureWallMs, "capture wall time"), + sqliteLockWaitMs: finiteNonNegative(rawCaptureMetrics.sqliteLockWaitMs, "SQLite lock wait"), + queueDepthBefore: finiteNonNegative(rawCaptureMetrics.queueDepthBefore, "capture queue depth before"), + queueDepthAfter: finiteNonNegative(rawCaptureMetrics.queueDepthAfter, "capture queue depth"), + filesystemBytesBefore: finiteNonNegative(rawCaptureMetrics.filesystemBytesBefore, "filesystem bytes before"), + filesystemBytesAfter: finiteNonNegative(rawCaptureMetrics.filesystemBytesAfter, "filesystem bytes after"), + casBytesBefore: finiteNonNegative(rawCaptureMetrics.casBytesBefore, "CAS bytes before"), + casBytesAfter: finiteNonNegative(rawCaptureMetrics.casBytesAfter, "CAS bytes after"), + normalizedStateBytesBefore: finiteNonNegative(rawCaptureMetrics.normalizedStateBytesBefore, "normalized state bytes before"), + normalizedStateBytesAfter: finiteNonNegative(rawCaptureMetrics.normalizedStateBytesAfter, "normalized state bytes after"), + archiveManifestInlineContentBytes: finiteNonNegative(rawCaptureMetrics.archiveManifestInlineContentBytes, "archive manifest inline content bytes"), + }) + : null; const response = recordValue(capture.response); const edgeCount = Number(capture.edgeCount); if (messages.length < 1 || !Number.isSafeInteger(edgeCount) || edgeCount < 1) @@ -683,9 +707,18 @@ export function buildGraphEvidenceFromCapture( boundedIdentity(recordValue(message).nodeId, `graph message node ${index + 1}`), ), responseNodeId: boundedIdentity(response.nodeId, "graph response node id"), - toolExecutionNodeIds: tools.map((tool, index) => - boundedIdentity(recordValue(tool).nodeId, `graph tool execution node ${index + 1}`), - ), + toolExecutionNodeIds: toolRows + .filter((tool) => tool.kind === undefined || tool.kind === "tool_execution") + .map((tool, index) => + boundedIdentity(tool.nodeId, `graph tool execution node ${index + 1}`), + ), + toolCallNodeIds: toolRows + .filter((tool) => tool.kind === "tool_call") + .map((tool, index) => boundedIdentity(tool.nodeId, `graph tool call node ${index + 1}`)), + toolResultNodeIds: toolRows + .filter((tool) => tool.kind === "tool_result") + .map((tool, index) => boundedIdentity(tool.nodeId, `graph tool result node ${index + 1}`)), + ...(captureMetrics ? { captureMetrics } : {}), edgeCount, }); } @@ -728,9 +761,17 @@ export function buildVerifiersLiveExport(input: { const message = recordValue(value); const role = boundedIdentity(message.role, `Verifiers node ${index + 1} role`); if (!("content" in message)) throw new Error(`Verifiers node ${index + 1} content is required`); + const toolCalls = Array.isArray(message.toolCalls) ? message.toolCalls : null; + const toolCallId = typeof message.toolCallId === "string" ? message.toolCallId : null; return { parent: index === 0 ? null : index - 1, - message: { role, content: message.content }, + message: { + role, + content: message.content, + ...(toolCalls ? { tool_calls: toolCalls } : {}), + ...(toolCallId ? { tool_call_id: toolCallId } : {}), + ...(typeof message.name === "string" ? { name: message.name } : {}), + }, sampled: index === messages.length, token_ids: [], mask: [], @@ -770,6 +811,16 @@ export function buildVerifiersLiveExport(input: { (tool, index) => boundedIdentity(recordValue(tool).nodeId, `graph tool node ${index + 1}`), ), + roleModelToolCallNodeIds: (Array.isArray(input.capture.tools) ? input.capture.tools : []) + .filter((tool) => recordValue(tool).kind === "tool_call") + .map((tool, index) => + boundedIdentity(recordValue(tool).nodeId, `graph tool call node ${index + 1}`), + ), + roleModelToolResultNodeIds: (Array.isArray(input.capture.tools) ? input.capture.tools : []) + .filter((tool) => recordValue(tool).kind === "tool_result") + .map((tool, index) => + boundedIdentity(recordValue(tool).nodeId, `graph tool result node ${index + 1}`), + ), }, is_completed: true, stop_condition: "role_model_graph_complete", diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 1b097182..7ff4558a 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -74,17 +74,45 @@ describe("Track B operations APIs", () => { messages: [ { nodeId: "node-system-94", role: "system", content: "route safely" }, { nodeId: "node-user-94", role: "user", content: "route this" }, + { + nodeId: "node-assistant-tool-94", + role: "assistant", + content: null, + toolCalls: [{ id: "call-pi-94", type: "function", function: { name: "bash", arguments: '{"command":"printf run94-tool-ok"}' } }], + }, + { nodeId: "node-tool-message-94", role: "tool", content: "run94-tool-ok", toolCallId: "call-pi-94", name: "bash" }, ], response: { nodeId: "node-response-94", role: "assistant", content: "routed" }, - tools: [{ nodeId: "node-tool-94", toolName: "bash", output: "run94-tool-ok" }], - edgeCount: 4, + tools: [ + { nodeId: "node-tool-execution-94", kind: "tool_execution", toolName: "router-tool" }, + { nodeId: "node-tool-call-94", kind: "tool_call", toolCallId: "call-pi-94", toolName: "bash" }, + { nodeId: "node-tool-result-94", kind: "tool_result", toolCallId: "call-pi-94", toolName: "bash" }, + ], + captureMetrics: { + captureCpuMs: 4, + captureWallMs: 7, + sqliteLockWaitMs: 1, + queueDepthBefore: 0, + queueDepthAfter: 0, + filesystemBytesBefore: 300, + filesystemBytesAfter: 400, + casBytesBefore: 100, + casBytesAfter: 140, + normalizedStateBytesBefore: 200, + normalizedStateBytesAfter: 260, + archiveManifestInlineContentBytes: 0, + }, + edgeCount: 6, }; expect(buildGraphEvidenceFromCapture(capture)).toEqual({ rootArtifactId: "root-export-94", - messageNodeIds: ["node-system-94", "node-user-94"], + messageNodeIds: ["node-system-94", "node-user-94", "node-assistant-tool-94", "node-tool-message-94"], responseNodeId: "node-response-94", - toolExecutionNodeIds: ["node-tool-94"], - edgeCount: 4, + toolExecutionNodeIds: ["node-tool-execution-94"], + toolCallNodeIds: ["node-tool-call-94"], + toolResultNodeIds: ["node-tool-result-94"], + captureMetrics: capture.captureMetrics, + edgeCount: 6, }); const exported = buildVerifiersLiveExport({ channel: "development", @@ -103,13 +131,15 @@ describe("Track B operations APIs", () => { requestId: "request-export-94", correlationId: "correlation-export-94", graphRootArtifactId: "root-export-94", - responseNodeIndex: 2, + responseNodeIndex: 4, tokenExactDisposition: "refused_missing_evidence", trace: { nodes: [ { parent: null, message: { role: "system", content: "route safely" }, sampled: false }, { parent: 0, message: { role: "user", content: "route this" }, sampled: false }, - { parent: 1, message: { role: "assistant", content: "routed" }, sampled: true }, + { parent: 1, message: { role: "assistant", content: null, tool_calls: [{ id: "call-pi-94", type: "function", function: { name: "bash", arguments: '{"command":"printf run94-tool-ok"}' } }] }, sampled: false }, + { parent: 2, message: { role: "tool", content: "run94-tool-ok", tool_call_id: "call-pi-94", name: "bash" }, sampled: false }, + { parent: 3, message: { role: "assistant", content: "routed" }, sampled: true }, ], info: { routeDecisionId: "decision-export-94", @@ -594,6 +624,20 @@ describe("Track B operations APIs", () => { content: routeCapture?.outputText, }, tools: [], + captureMetrics: { + captureCpuMs: 4, + captureWallMs: 7, + sqliteLockWaitMs: 1, + queueDepthBefore: 0, + queueDepthAfter: 0, + filesystemBytesBefore: 300, + filesystemBytesAfter: 400, + casBytesBefore: 100, + casBytesAfter: 140, + normalizedStateBytesBefore: 200, + normalizedStateBytesAfter: 260, + archiveManifestInlineContentBytes: 0, + }, edgeCount: 2, } : request.url === "/capture/route" @@ -722,8 +766,29 @@ describe("Track B operations APIs", () => { messageNodeIds: ["message-route-capture-0"], responseNodeId: "response-route-capture", toolExecutionNodeIds: [], + toolCallNodeIds: [], + toolResultNodeIds: [], + captureMetrics: { + captureCpuMs: 4, + captureWallMs: 7, + sqliteLockWaitMs: 1, + queueDepthBefore: 0, + queueDepthAfter: 0, + filesystemBytesBefore: 300, + filesystemBytesAfter: 400, + casBytesBefore: 100, + casBytesAfter: 140, + normalizedStateBytesBefore: 200, + normalizedStateBytesAfter: 260, + archiveManifestInlineContentBytes: 0, + }, edgeCount: 2, }); + expect((detail as unknown as { liveBudgetEvidence?: { compactObservationBytes?: number; runtimeRssBytes?: number } }).liveBudgetEvidence).toMatchObject({ + compactObservationBytes: expect.any(Number), + runtimeRssBytes: expect.any(Number), + }); + expect((detail as unknown as { liveBudgetEvidence: { compactObservationBytes: number } }).liveBudgetEvidence.compactObservationBytes).toBeLessThanOrEqual(16 * 1024); const correlationId = String( (detail as unknown as { run88Correlation?: { correlationId?: string } })?.run88Correlation ?.correlationId, From 03d4a1d3e48e2ab5fb015fd0f085d0271534ca5d Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 03:06:46 +0800 Subject: [PATCH 08/40] Expose measured Run 94 capture baseline --- .../apps/runtime-host-bridge/src/cli.ts | 4 +++ .../apps/runtime-host-bridge/src/index.ts | 27 +++++++++++++++ .../src/track-b-operations.ts | 8 +++++ .../test/run94-sp5-sp10.test.ts | 24 ++++++++++++++ .../test/track-b-operations-api.test.ts | 33 +++++++++++++++++++ 5 files changed, 96 insertions(+) diff --git a/role-model-router/apps/runtime-host-bridge/src/cli.ts b/role-model-router/apps/runtime-host-bridge/src/cli.ts index b3f62df6..4ac3e8c6 100644 --- a/role-model-router/apps/runtime-host-bridge/src/cli.ts +++ b/role-model-router/apps/runtime-host-bridge/src/cli.ts @@ -60,6 +60,7 @@ type CliBackend = Pick< | "readTrackBQaExtensions" | "readTrackBShadowReceipts" | "readTrackBExtensionReadback" + | "measureNoRichCaptureBaseline" | "readGraphMigration" | "advanceGraphMigration" | "rollbackGraphMigration" @@ -427,6 +428,9 @@ export function createCliServerOptions( readTrackBExtensionReadback: bindBackendMethod( "readTrackBExtensionReadback", ) as StartBridgeServerOptions["readTrackBExtensionReadback"], + measureNoRichCaptureBaseline: bindBackendMethod( + "measureNoRichCaptureBaseline", + ) as StartBridgeServerOptions["measureNoRichCaptureBaseline"], readGraphMigration: bindBackendMethod( "readGraphMigration", ) as StartBridgeServerOptions["readGraphMigration"], diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index 0193d269..d21da1b3 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -2879,6 +2879,7 @@ export interface StartBridgeServerOptions { readonly readTrackBQaExtensions?: () => Promise; readonly readTrackBShadowReceipts?: () => Promise; readonly readTrackBExtensionReadback?: (body: Record) => Promise; + readonly measureNoRichCaptureBaseline?: (body: Record) => Promise; readonly readGraphMigration?: () => Promise; readonly advanceGraphMigration?: (body: Record) => Promise; readonly rollbackGraphMigration?: () => Promise; @@ -3096,6 +3097,7 @@ export interface RuntimeBridgeBackend { readTrackBQaExtensions(): Promise; readTrackBShadowReceipts(): Promise; readTrackBExtensionReadback(body: Record): Promise; + measureNoRichCaptureBaseline(body: Record): Promise; readGraphMigration(): Promise; advanceGraphMigration(body: Record): Promise; rollbackGraphMigration(): Promise; @@ -14845,6 +14847,28 @@ function createRequestHandler(options: StartBridgeServerOptions) { return; } + if ( + request.method === "POST" && + url.pathname === "/api/role-model/track-b/performance-baseline" + ) { + if (!options.measureNoRichCaptureBaseline) { + writeJson(response, 404, { error: "not found" }); + return; + } + try { + writeJson( + response, + 200, + await options.measureNoRichCaptureBaseline(await readJsonBody(request)), + ); + } catch (error) { + writeJson(response, 409, { + error: error instanceof Error ? error.message : String(error), + }); + } + return; + } + if (request.method === "POST" && url.pathname === "/api/role-model/track-b/verifiers-export") { if (!options.exportVerifiersTrace) { writeJson(response, 404, { error: "not found" }); @@ -25360,6 +25384,9 @@ export async function createRuntimeBridgeBackend( } return options.readTrackBExtensionReadback(body); }, + async measureNoRichCaptureBaseline(body: Record): Promise { + return runtimeTrackBOperations.measureNoRichCaptureBaseline(body); + }, async readGraphMigration(): Promise { return createTrackBOperations({ statePath: path.join( diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index 7ddae46c..02767b99 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -1595,6 +1595,14 @@ export function createTrackBOperations({ throw new Error("local route capture requires a loopback operations boundary"); return requestPrivate("capture/route", { method: "POST", body: input }); }, + async measureNoRichCaptureBaseline(input: Record): Promise { + if (!operationsEndpoint) + throw new Error("private operations endpoint is required for no-rich capture baseline measurement"); + const url = new URL(operationsEndpoint); + if (!["127.0.0.1", "localhost", "::1", "[::1]"].includes(url.hostname)) + throw new Error("no-rich capture baseline requires a loopback operations boundary"); + return requestPrivate("capture/performance-baseline", { method: "POST", body: input }); + }, async readLocalRouteCapture(input: Record): Promise { if (!operationsEndpoint) throw new Error("private operations endpoint is required for exact route capture readback"); diff --git a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts index e9ff83dd..f35c7117 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts @@ -375,6 +375,15 @@ test("GREEN: exposes extension readback through the Track B HTTP surface", async executeChatCompletions: backend.executeChatCompletions, executeResponses: backend.executeResponses, readTrackBExtensionReadback: backend.readTrackBExtensionReadback, + measureNoRichCaptureBaseline: async (body) => ({ + schemaVersion: "role-model.no-rich-capture-baseline-channel.v1", + sourceMode: "measured_capture_disabled_packaged_runtime", + channel: "development", + sampleCount: body.sampleCount, + captureCpuP95Ms: 12, + providerPathLatencyP95Ms: 18, + sqliteLockWaitP95Ms: 2, + }), }); try { const response = await fetch( @@ -390,6 +399,21 @@ test("GREEN: exposes extension readback through the Track B HTTP surface", async requestId: "run94-extension-closure", schemaVersion: "role-model.track-b-extension-readback.v1", }); + const baselineResponse = await fetch( + `http://127.0.0.1:${server.port}/api/role-model/track-b/performance-baseline`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ sampleCount: 5 }), + }, + ); + expect(baselineResponse.status).toBe(200); + expect(await baselineResponse.json()).toMatchObject({ + schemaVersion: "role-model.no-rich-capture-baseline-channel.v1", + sourceMode: "measured_capture_disabled_packaged_runtime", + channel: "development", + sampleCount: 5, + }); } finally { await server.close(); await backend.shutdown(); diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 7ff4558a..ee251608 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -221,6 +221,39 @@ describe("Track B operations APIs", () => { } }); + test("reads the measured no-rich baseline through the authenticated loopback sidecar", async () => { + const received: Array<{ path: string; authorization?: string; body: unknown }> = []; + const operations = createServer(async (request, response) => { + let body = ""; + for await (const chunk of request) body += chunk; + received.push({ path: request.url ?? "", authorization: request.headers.authorization, body: JSON.parse(body) }); + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ schemaVersion: "role-model.no-rich-capture-baseline-channel.v1", sourceMode: "measured_capture_disabled_packaged_runtime", channel: "development", sampleCount: 5, captureCpuP95Ms: 12, providerPathLatencyP95Ms: 18, sqliteLockWaitP95Ms: 2 })); + }); + await new Promise((resolve, reject) => { + operations.once("error", reject); + operations.listen(0, "127.0.0.1", resolve); + }); + try { + const address = operations.address(); + if (!address || typeof address === "string") throw new Error("operations server did not bind"); + const api = createTrackBOperations({ + statePath: path.join(os.tmpdir(), `run94-no-rich-baseline-${Date.now()}.json`), + catalog: [], + operationsEndpoint: `http://127.0.0.1:${address.port}`, + operationsToken: "run94-baseline-token-0001", + }); + await expect(api.measureNoRichCaptureBaseline({ sampleCount: 5 })).resolves.toMatchObject({ + schemaVersion: "role-model.no-rich-capture-baseline-channel.v1", + channel: "development", + sampleCount: 5, + }); + expect(received).toEqual([{ path: "/capture/performance-baseline", authorization: "Bearer run94-baseline-token-0001", body: { sampleCount: 5 } }]); + } finally { + await new Promise((resolve, reject) => operations.close((error) => error ? reject(error) : resolve())); + } + }); + test("fails closed instead of issuing unauthenticated calls to an owned operations endpoint", async () => { const runtimeStateRoot = path.join(os.tmpdir(), `track-b-operations-auth-${Date.now()}`); roots.push(runtimeStateRoot); From 3a1ccc76b21e9aa4a991259cb008e06e82158800 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 03:33:23 +0800 Subject: [PATCH 09/40] Preserve API-authored runtime aliases --- .../src/unified-runtime-config.ts | 7 +++++ .../test/unified-runtime-config.test.ts | 31 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/role-model-router/apps/runtime-host-bridge/src/unified-runtime-config.ts b/role-model-router/apps/runtime-host-bridge/src/unified-runtime-config.ts index a515500f..eb742bd6 100644 --- a/role-model-router/apps/runtime-host-bridge/src/unified-runtime-config.ts +++ b/role-model-router/apps/runtime-host-bridge/src/unified-runtime-config.ts @@ -1982,6 +1982,13 @@ function normalizeRuntimeConfigPatchDocument( normalized.execution_mode = normalized.executionMode; delete normalized.executionMode; } + if ( + Object.prototype.hasOwnProperty.call(normalized, "modelAliases") && + !Object.prototype.hasOwnProperty.call(normalized, "model_aliases") + ) { + normalized.model_aliases = normalized.modelAliases; + delete normalized.modelAliases; + } return normalized; } diff --git a/role-model-router/apps/runtime-host-bridge/test/unified-runtime-config.test.ts b/role-model-router/apps/runtime-host-bridge/test/unified-runtime-config.test.ts index 755f1ba9..dafc514c 100644 --- a/role-model-router/apps/runtime-host-bridge/test/unified-runtime-config.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/unified-runtime-config.test.ts @@ -1055,6 +1055,37 @@ observed_data: expect(merged.observedData?.throughputSla.enabled).toBe(true); }); + test("merges API camelCase model aliases without retaining the stale YAML alias map", () => { + const merged = mergeUnifiedRuntimeConfigDocuments( + { + version: "1.0", + execution_mode: "remote_only", + model_aliases: { + "default.remote-only": { mode: "basic", model_ids: ["deepseek/model"] }, + }, + }, + { + modelAliases: [ + { + aliasId: "run94.success", + mode: "basic", + modelIds: ["deepseek/model"], + endpointIds: ["deepseek.valid"], + }, + ], + }, + ); + + expect(merged.modelAliases).toEqual([ + { + aliasId: "run94.success", + mode: "basic", + modelIds: ["deepseek/model"], + endpointIds: ["deepseek.valid"], + }, + ]); + }); + test("accepts routing_strategy alias in partial runtime config patches", () => { const merged = mergeUnifiedRuntimeConfigDocuments( { From 6fc45ce514a706d935074b9c9f8e8d85b117f58b Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 04:35:36 +0800 Subject: [PATCH 10/40] Route env-backed providers through runtime --- .../apps/runtime-host-bridge/src/index.ts | 14 +- .../runtime-host-bridge/test/index.test.ts | 144 ++++++++++++++++++ 2 files changed, 151 insertions(+), 7 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index d21da1b3..61aba5a3 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -22469,13 +22469,16 @@ export async function createRuntimeBridgeBackend( requestCapture: ProviderRequestCapture; fallbackModelIds?: readonly string[]; }) => { - // File-backed credentials (OAuth, locally-saved API keys) always need direct HTTP execution - // so that OAuth tokens are correctly resolved and X-Msh-* device headers are applied. + // Router-owned credentials (env refs, OAuth, locally-saved API keys) use direct HTTP execution + // so credentials stay in the runtime and provider-specific headers are applied there. // In the unified config path, LiteLLM providers get adapterFamily "litellm-proxy", so // shouldUseLiveProviderExecution would return false for them — this flag bypasses that check. + const usesFixtureAccount = + target.providerAccountId !== null && fixtureAccountIds.has(target.providerAccountId); const useDirectExecution = target.account?.credentialRef.backend === "local-file" || - target.account?.credentialRef.backend === "local-encrypted-file"; + target.account?.credentialRef.backend === "local-encrypted-file" || + (!usesFixtureAccount && shouldUseLiveProviderExecution(target)); const capture = captures.byEndpointId[target.endpointId]; const failureContext = { providerId: target.providerId, @@ -22484,9 +22487,6 @@ export async function createRuntimeBridgeBackend( adapterFamily: target.adapterFamily, failurePhase: "provider_execution", } as const; - const usesFixtureAccount = - target.providerAccountId !== null && fixtureAccountIds.has(target.providerAccountId); - if ( !useDirectExecution && capture && @@ -22613,7 +22613,7 @@ export async function createRuntimeBridgeBackend( vendorMetadata: result.metadata, }; } - // Fall through to direct HTTP execution for file-backed credential accounts. + // Fall through to direct HTTP execution for router-owned credential accounts. } if (!useDirectExecution && !shouldUseLiveProviderExecution(target)) { diff --git a/role-model-router/apps/runtime-host-bridge/test/index.test.ts b/role-model-router/apps/runtime-host-bridge/test/index.test.ts index de29b379..511ea741 100644 --- a/role-model-router/apps/runtime-host-bridge/test/index.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/index.test.ts @@ -18290,6 +18290,150 @@ describe("runtime-host-bridge", () => { } }); + test("executes env-backed OpenAI-compatible accounts directly when unified config has no LiteLLM provider", async () => { + expect( + typeof (bridge as { createRuntimeBridgeBackend?: unknown }).createRuntimeBridgeBackend, + ).toBe("function"); + + const runtimeStateRoot = await mkdtemp( + path.join(os.tmpdir(), "role-model-unified-env-direct-execution-tests-"), + ); + const unifiedRuntimeConfigPath = path.join(runtimeStateRoot, "runtime-config.yaml"); + const providerRequests: Array<{ authorization: string; model: unknown }> = []; + await writeFile( + unifiedRuntimeConfigPath, + stringify({ + version: "1.0", + routing: { strategy: "balanced" }, + }), + "utf8", + ); + + const backend = await ( + bridge as { + createRuntimeBridgeBackend: (options: { + repoRoot: string; + fixtureRoot: string; + runtimeStateRoot: string; + scopeId: string; + unifiedRuntimeConfigPath: string; + networkFetcher: typeof fetch; + providerCredentialEnvironment: Readonly>; + }) => Promise<{ + upsertProviderAccount: (input: Record) => Promise; + activateEndpoint: (input: { + providerAccountId: string; + modelId: string; + region: string; + }) => Promise<{ endpointId: string }>; + executeChatCompletions: ( + body: Record, + requestId: string, + ) => Promise<{ endpointId: string; outputText: string }>; + shutdown: () => Promise; + }>; + } + ).createRuntimeBridgeBackend({ + repoRoot, + fixtureRoot: testFixtureRoot, + runtimeStateRoot, + scopeId: "runtime-host-unified-env-direct-execution-tests", + unifiedRuntimeConfigPath, + providerCredentialEnvironment: { + RUN94_DEEPSEEK_API_KEY: "router-owned-test-secret", + }, + networkFetcher: async (input, init) => { + const url = + typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + if (isAdmissionReadinessProbe(init)) { + return successfulAdmissionReadinessProbe(); + } + if (url === "https://api.deepseek.example/v1/chat/completions") { + const body = init?.body ? JSON.parse(String(init.body)) : {}; + providerRequests.push({ + authorization: (init?.headers as Record)?.authorization ?? "", + model: body.model, + }); + return new Response( + JSON.stringify({ + id: "chatcmpl-unified-env-direct", + object: "chat.completion", + model: "deepseek/deepseek-v4-flash", + choices: [ + { + index: 0, + message: { role: "assistant", content: "direct provider execution" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 8, completion_tokens: 3, total_tokens: 11 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + } + throw new Error(`Unexpected network request: ${url}`); + }, + }); + + try { + await backend.upsertProviderAccount({ + providerAccountId: "deepseek.personal.run94", + providerId: "deepseek", + providerKind: "provider-openai", + orgScope: "personal", + accountScope: "workspace-default", + credentialRef: { backend: "env", ref: "RUN94_DEEPSEEK_API_KEY" }, + authMode: "api-key-static", + regionPolicy: { mode: "prefer", regions: ["global"] }, + baseUrlOverride: "https://api.deepseek.example/v1", + allowedModels: ["deepseek/deepseek-v4-flash"], + modelRoleBindings: [ + { + modelId: "deepseek/deepseek-v4-flash", + roleAssignmentMode: "all", + roleIds: [], + enabledRoleIds: [], + disabledRoleIds: [], + }, + ], + deniedModels: [], + entitlementTags: ["chat"], + budgetPolicyRef: "budget.default", + quotaPolicyRef: "quota.default", + status: "active", + healthStatus: "healthy", + rotationState: "stable", + }); + const endpoint = await backend.activateEndpoint({ + providerAccountId: "deepseek.personal.run94", + modelId: "deepseek/deepseek-v4-flash", + region: "global", + }); + + await expect( + backend.executeChatCompletions( + { + model: endpoint.endpointId, + messages: [{ role: "user", content: "Route directly." }], + }, + "req-runtime-bridge-unified-env-direct-001", + ), + ).resolves.toMatchObject({ + endpointId: endpoint.endpointId, + outputText: "direct provider execution", + }); + expect(providerRequests).toEqual([ + { + authorization: "Bearer router-owned-test-secret", + model: "deepseek-v4-flash", + }, + ]); + } finally { + await backend.shutdown(); + await rm(runtimeStateRoot, { recursive: true, force: true }); + } + }); + test("registers configured local OpenAI-compatible peers as execution-ready model endpoints", async () => { expect( typeof (bridge as { createRuntimeBridgeBackend?: unknown }).createRuntimeBridgeBackend, From 51357dd57487cc4865ec13a36407626d2c674ef6 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 04:58:21 +0800 Subject: [PATCH 11/40] Persist supervised extension outputs --- packages/extension-host/index.mjs | 8 +- packages/extension-host/worker-runtime.mjs | 125 +++++++++++++++++- .../recursive-87-registry-lifecycle.test.ts | 57 ++++++++ 3 files changed, 188 insertions(+), 2 deletions(-) diff --git a/packages/extension-host/index.mjs b/packages/extension-host/index.mjs index 336b39ad..e531c5b3 100644 --- a/packages/extension-host/index.mjs +++ b/packages/extension-host/index.mjs @@ -472,7 +472,13 @@ export class ExtensionHost { return Promise.reject( new Error("oversized payload requires a channel-local transfer artifact"), ); - if (envelope.capability && !registered.descriptor.capabilities.includes(envelope.capability)) + const hostReadCapability = + registered.kind === "process" && envelope.capability === "extension-output:read"; + if ( + envelope.capability && + !hostReadCapability && + !registered.descriptor.capabilities.includes(envelope.capability) + ) return Promise.reject(new Error("capability denied")); if (envelope.signal?.aborted) { this.#record(id, "cancelled", envelope); diff --git a/packages/extension-host/worker-runtime.mjs b/packages/extension-host/worker-runtime.mjs index e71557d2..828ee7f1 100644 --- a/packages/extension-host/worker-runtime.mjs +++ b/packages/extension-host/worker-runtime.mjs @@ -1,4 +1,8 @@ +import { createHash } from "node:crypto"; import { once } from "node:events"; +import { mkdirSync } from "node:fs"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { pathToFileURL } from "node:url"; import { encodeFrame, extractFrames } from "../extension-sdk/index.mjs"; @@ -7,6 +11,115 @@ if (!moduleRef) throw new Error("worker module URL required"); const extension = await import(moduleRef.includes(":") ? moduleRef : pathToFileURL(moduleRef).href); if (typeof extension.run !== "function") throw new Error("worker module must export run(envelope)"); +const extensionId = process.env.ROLE_MODEL_EXTENSION_ID ?? "unknown-extension"; +const stateRoot = process.env.ROLE_MODEL_EXTENSION_STATE_ROOT; +if (stateRoot) mkdirSync(stateRoot, { recursive: true }); +const outputDatabase = new DatabaseSync( + stateRoot ? join(stateRoot, "durable-output.sqlite") : ":memory:", +); +outputDatabase.exec(` + PRAGMA journal_mode=WAL; + PRAGMA synchronous=FULL; + CREATE TABLE IF NOT EXISTS durable_extension_outputs ( + output_key TEXT PRIMARY KEY, + request_id TEXT NOT NULL, + capability TEXT NOT NULL, + channel TEXT NOT NULL, + scope_id TEXT NOT NULL, + result_hash TEXT NOT NULL, + byte_length INTEGER NOT NULL, + result_json TEXT, + created_at TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS durable_extension_outputs_created + ON durable_extension_outputs(created_at, output_key); +`); +const MAX_INLINE_OUTPUT_BYTES = 16 * 1024; +const MAX_DURABLE_OUTPUT_ROWS = 512; + +function persistBusinessOutput(envelope, result) { + const resultJson = JSON.stringify(result ?? null); + const byteLength = Buffer.byteLength(resultJson, "utf8"); + const resultHash = `sha256:${createHash("sha256").update(resultJson).digest("hex")}`; + const outputKey = `sha256:${createHash("sha256") + .update(`${extensionId}\0${envelope.requestId}\0${envelope.capability}\0${resultHash}`) + .digest("hex")}`; + outputDatabase + .prepare("INSERT OR IGNORE INTO durable_extension_outputs VALUES (?,?,?,?,?,?,?,?,?)") + .run( + outputKey, + envelope.requestId, + envelope.capability, + envelope.channel, + envelope.scope, + resultHash, + byteLength, + byteLength <= MAX_INLINE_OUTPUT_BYTES ? resultJson : null, + new Date().toISOString(), + ); + outputDatabase + .prepare(`DELETE FROM durable_extension_outputs WHERE output_key IN ( + SELECT output_key FROM durable_extension_outputs + ORDER BY created_at DESC, output_key DESC LIMIT -1 OFFSET ? + )`) + .run(MAX_DURABLE_OUTPUT_ROWS); + const durableLocator = Object.freeze({ + extensionId, + requestId: envelope.requestId, + capability: envelope.capability, + channel: envelope.channel, + scope: envelope.scope, + outputKey, + resultHash, + byteLength, + }); + const businessOutput = + result && typeof result === "object" && !Array.isArray(result) ? result : { value: result }; + return { + ...businessOutput, + businessOutput, + durableLocator, + evidenceRef: `extension-output:${outputKey}`, + readCapability: "extension-output:read", + }; +} + +function readBusinessOutput(envelope) { + const locator = envelope.payload?.durableLocator; + const durableOutputId = envelope.payload?.durableOutputId; + if ( + !locator || + locator.extensionId !== extensionId || + locator.channel !== envelope.channel || + locator.scope !== envelope.scope || + !locator.outputKey || + !durableOutputId + ) { + throw new Error("durable extension output read identity is incomplete"); + } + const row = outputDatabase + .prepare("SELECT * FROM durable_extension_outputs WHERE output_key=?") + .get(locator.outputKey); + if ( + !row || + row.request_id !== locator.requestId || + row.capability !== locator.capability || + row.channel !== locator.channel || + row.scope_id !== locator.scope || + row.result_hash !== locator.resultHash || + row.byte_length !== locator.byteLength + ) { + throw new Error("durable extension output locator does not match stored evidence"); + } + return { + durableLocator: locator, + evidenceRef: `extension-output:${locator.outputKey}`, + readbackOutputId: durableOutputId, + resultHash: row.result_hash, + byteLength: row.byte_length, + }; +} + const retained = new Map(); let input = Buffer.alloc(0); const send = async (value) => { @@ -24,12 +137,22 @@ process.stdin.on("data", async (chunk) => { continue; } if (message.type === "shutdown") { + outputDatabase.close(); await send({ type: "shutdown-ack" }); process.exit(0); } if (message.type !== "invoke") continue; try { - const result = await extension.run(message.envelope); + let result; + if (message.envelope.capability === "extension-output:read") { + result = readBusinessOutput(message.envelope); + } else { + const value = await extension.run(message.envelope); + result = + message.envelope.capability === "health:probe" + ? value + : persistBusinessOutput(message.envelope, value); + } const response = { type: "result", requestId: message.requestId, result }; retained.set(message.requestId, response); await send(response); diff --git a/role-model-router/apps/runtime-host-bridge/test/recursive-87-registry-lifecycle.test.ts b/role-model-router/apps/runtime-host-bridge/test/recursive-87-registry-lifecycle.test.ts index fea296a2..44aeebd3 100644 --- a/role-model-router/apps/runtime-host-bridge/test/recursive-87-registry-lifecycle.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/recursive-87-registry-lifecycle.test.ts @@ -66,6 +66,63 @@ describe("recursive run 87 SP0 registry and lifecycle authority", () => { ).resolves.toMatchObject({ echoed: { value: 87 }, requestId: "run87:synthetic:echo" }); }); + test("process extension business output survives a supervised worker restart", async () => { + const stateRoot = path.join(os.tmpdir(), `run94-durable-extension-output-${Date.now()}`); + roots.push(stateRoot); + const runtime = await trackBRuntime.createExtensionRuntime({ + stateRoot, + authorizationEpoch: 94, + repoRoot, + extensions: await syntheticExtensions(1), + }); + runtimes.push(runtime); + + const output = await runtime.invoke("canonical-01", { + requestId: "run94:durable-output", + protocolVersion: "1.1.0", + channel: "development", + scope: "tenant:run94", + authorizationEpoch: 94, + capability: "fixture:echo", + payload: { value: 94 }, + }); + expect(output).toMatchObject({ + readCapability: "extension-output:read", + businessOutput: { echoed: { value: 94 }, requestId: "run94:durable-output" }, + durableLocator: { + extensionId: "canonical-01", + requestId: "run94:durable-output", + capability: "fixture:echo", + }, + }); + + const before = runtime.listExtensions()[0]; + const restarted = await runtime.mutateExtension({ + id: "canonical-01", + action: "restart", + mutationId: "run94:restart:durable-output", + expectedRevision: before.revision, + }); + expect(restarted.state.pid).not.toBe(before.pid); + await expect( + runtime.invoke("canonical-01", { + requestId: "run94:durable-output:readback", + protocolVersion: "1.1.0", + channel: "development", + scope: "tenant:run94", + authorizationEpoch: 94, + capability: "extension-output:read", + payload: { + durableLocator: output.durableLocator, + durableOutputId: "sha256:host-bound-output", + }, + }), + ).resolves.toMatchObject({ + readbackOutputId: "sha256:host-bound-output", + durableLocator: output.durableLocator, + }); + }); + test("the production constructor keeps thirteen release extensions while admitting an explicit QA extension", async () => { const stateRoot = path.join(os.tmpdir(), `run87-packaged-qa-runtime-${Date.now()}`); roots.push(stateRoot); From 12e85e565aa4edd237dfdbadd9070bca816c5cb6 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 05:03:48 +0800 Subject: [PATCH 12/40] Preserve bounded post-observation receipts --- .../src/track-b-runtime.ts | 59 ++++++++++++++++--- .../test/run94-sp5-sp10.test.ts | 5 ++ 2 files changed, 57 insertions(+), 7 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts index 70839ada..4cb76529 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { createInterface } from "node:readline"; import { DatabaseSync } from "node:sqlite"; import { pathToFileURL } from "node:url"; +import { gunzipSync, gzipSync } from "node:zlib"; import type { RuntimeEffortSource } from "@role-model-router/runtime-observability"; import { @@ -1184,19 +1185,64 @@ export async function verifyTrackBExtensionClosureAfterRestart( const TRACK_B_OUTBOX_SCHEMA_VERSION = "role-model.track-b-post-observation-outbox.v3" as const; const TRACK_B_OUTBOX_RECEIPT_CAP_BYTES = 16 * 1024; +const TRACK_B_OUTBOX_RECEIPT_RAW_CAP_BYTES = 256 * 1024; const TRACK_B_OUTBOX_SQLITE_HEADER = "SQLite format 3"; function boundedJson(value: unknown, capBytes = TRACK_B_OUTBOX_RECEIPT_CAP_BYTES): string { const json = JSON.stringify(value ?? null); if (Buffer.byteLength(json, "utf8") <= capBytes) return json; - const digest = createHash("sha256").update(json).digest("hex"); + const bytes = Buffer.from(json, "utf8"); + const digest = createHash("sha256").update(bytes).digest("hex"); + if (bytes.byteLength <= TRACK_B_OUTBOX_RECEIPT_RAW_CAP_BYTES) { + const compressed = JSON.stringify({ + status: "compressed_receipt", + encoding: "gzip-base64", + byteLength: bytes.byteLength, + sha256: `sha256:${digest}`, + payload: gzipSync(bytes, { level: 9 }).toString("base64"), + }); + if (Buffer.byteLength(compressed, "utf8") <= capBytes) return compressed; + } return JSON.stringify({ status: "bounded_receipt", - byteLength: Buffer.byteLength(json, "utf8"), + byteLength: bytes.byteLength, sha256: `sha256:${digest}`, }); } +function parseBoundedJson(json: string): unknown { + const value = JSON.parse(json) as Record | unknown; + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + (value as Record).status !== "compressed_receipt" + ) { + return value; + } + const record = value as Record; + if ( + record.encoding !== "gzip-base64" || + typeof record.payload !== "string" || + !Number.isSafeInteger(record.byteLength) || + Number(record.byteLength) < 0 || + Number(record.byteLength) > TRACK_B_OUTBOX_RECEIPT_RAW_CAP_BYTES || + !/^sha256:[a-f0-9]{64}$/.test(String(record.sha256 ?? "")) + ) { + throw new Error("compressed receipt identity is invalid"); + } + const bytes = gunzipSync(Buffer.from(record.payload, "base64"), { + maxOutputLength: TRACK_B_OUTBOX_RECEIPT_RAW_CAP_BYTES, + }); + if ( + bytes.byteLength !== record.byteLength || + `sha256:${createHash("sha256").update(bytes).digest("hex")}` !== record.sha256 + ) { + throw new Error("compressed receipt integrity verification failed"); + } + return JSON.parse(bytes.toString("utf8")); +} + function outboxSchema(database: DatabaseSync): void { database.exec(` CREATE TABLE IF NOT EXISTS track_b_post_observation_pending ( @@ -1587,7 +1633,7 @@ export function createTrackBPostObservationOutbox({ ...(row.reasoning_effort !== null ? { reasoningEffort: row.reasoning_effort } : {}), ...(row.effort_source !== null ? { effortSource: row.effort_source } : {}), ...(row.run88_correlation_json - ? { run88Correlation: JSON.parse(row.run88_correlation_json) } + ? { run88Correlation: parseBoundedJson(row.run88_correlation_json) } : {}), ...(row.legacy_identity_missing ? { legacyIdentityMissing: true as const } : {}), } as TrackBPostObservationWorkItem; @@ -1649,7 +1695,7 @@ export function createTrackBPostObservationOutbox({ receipts: rows.map((row) => ({ requestId: row.request_id, completedAt: row.completed_at, - result: JSON.parse(row.result_json), + result: parseBoundedJson(row.result_json), })), }; }); @@ -1668,7 +1714,7 @@ export function createTrackBPostObservationOutbox({ ? { requestId: row.request_id, completedAt: row.completed_at, - result: JSON.parse(row.result_json), + result: parseBoundedJson(row.result_json), } : null; }); @@ -1891,8 +1937,7 @@ export async function runTrackBShadowPipeline( const rolloutRows = [sourceRollout, ...counterfactualRollouts]; const scoredRollouts = rolloutRows.map((rollout) => ({ ...rollout, - score: - (rollout.outcome as Record | undefined)?.status === "success" ? 1 : 0, + score: (rollout.outcome as Record | undefined)?.status === "success" ? 1 : 0, })); const positive = scoredRollouts.filter((rollout) => rollout.score === 1); const negative = scoredRollouts.filter((rollout) => rollout.score === 0); diff --git a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts index f35c7117..918f5344 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts @@ -62,6 +62,11 @@ test("GREEN: post-observation outbox is a normalized SQLite authority with bound })); const receipt = await restarted.read(); expect(receipt).toMatchObject({ pendingCount: 0, receiptCount: 2 }); + const recovered = await restarted.readReceipt("queued-1"); + expect( + ((recovered?.result as Record).extensionClosure as Record) + .result, + ).toHaveLength(100_000); const afterDrain = new DatabaseSync(filePath); const row = afterDrain .prepare("SELECT length(result_json) AS bytes FROM track_b_post_observation_receipts") From bd3de1a7244a23c02732f409bc30629ce54e71b7 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 05:08:41 +0800 Subject: [PATCH 13/40] Raise structured outbox receipt bound --- .../apps/runtime-host-bridge/src/track-b-runtime.ts | 4 ++-- .../apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts index 4cb76529..f6097b94 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts @@ -1184,8 +1184,8 @@ export async function verifyTrackBExtensionClosureAfterRestart( } const TRACK_B_OUTBOX_SCHEMA_VERSION = "role-model.track-b-post-observation-outbox.v3" as const; -const TRACK_B_OUTBOX_RECEIPT_CAP_BYTES = 16 * 1024; -const TRACK_B_OUTBOX_RECEIPT_RAW_CAP_BYTES = 256 * 1024; +const TRACK_B_OUTBOX_RECEIPT_CAP_BYTES = 10 * 1024 * 1024; +const TRACK_B_OUTBOX_RECEIPT_RAW_CAP_BYTES = 10 * 1024 * 1024; const TRACK_B_OUTBOX_SQLITE_HEADER = "SQLite format 3"; function boundedJson(value: unknown, capBytes = TRACK_B_OUTBOX_RECEIPT_CAP_BYTES): string { diff --git a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts index 918f5344..9a78ccd7 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts @@ -58,7 +58,7 @@ test("GREEN: post-observation outbox is a normalized SQLite authority with bound const restarted = createTrackBPostObservationOutbox({ filePath, maxItems: 8 }); await restarted.drain(async () => ({ status: "processed", - extensionClosure: { result: "x".repeat(100_000) }, + extensionClosure: { result: "x".repeat(1_000_000) }, })); const receipt = await restarted.read(); expect(receipt).toMatchObject({ pendingCount: 0, receiptCount: 2 }); @@ -66,12 +66,12 @@ test("GREEN: post-observation outbox is a normalized SQLite authority with bound expect( ((recovered?.result as Record).extensionClosure as Record) .result, - ).toHaveLength(100_000); + ).toHaveLength(1_000_000); const afterDrain = new DatabaseSync(filePath); const row = afterDrain .prepare("SELECT length(result_json) AS bytes FROM track_b_post_observation_receipts") .get() as { bytes: number }; - expect(row.bytes).toBeLessThanOrEqual(16 * 1024); + expect(row.bytes).toBeLessThanOrEqual(10 * 1024 * 1024); afterDrain.close(); }); From 9b3cae30752ad87abea18b4113e227308561e016 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 05:16:12 +0800 Subject: [PATCH 14/40] Make extension readback repeatable --- .../runtime-host-bridge/src/track-b-runtime.ts | 2 +- .../test/run94-sp5-sp10.test.ts | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts index f6097b94..a0248280 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts @@ -1122,7 +1122,7 @@ export async function verifyTrackBExtensionClosureAfterRestart( const mutation = (await runtime.mutateExtension({ id: extensionId, action: "restart", - mutationId: `run94-readback:${closure.requestId}:${extensionId}`, + mutationId: `run94-readback:${closure.requestId}:${extensionId}:revision:${revision}`, expectedRevision: revision, })) as Record; const mutationState = extensionResultRecord(mutation.state); diff --git a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts index 9a78ccd7..590cf4d0 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts @@ -265,7 +265,7 @@ test("GREEN: real process output closure covers every canonical registry key and expect( outputs.every((output) => Number.isInteger((output as Record).workerPid)), ).toBe(true); - const readback = await verifyTrackBExtensionClosureAfterRestart(runtime, closure as never, { + const readbackOptions = { channel: "development", scope: "tenant:run94", authorizationEpoch: 94, @@ -279,7 +279,12 @@ test("GREEN: real process output closure covers every canonical registry key and capability: "artifact:read", payload: { durableLocator, durableOutputId }, }), - }); + } as const; + const readback = await verifyTrackBExtensionClosureAfterRestart( + runtime, + closure as never, + readbackOptions, + ); expect(readback.outputs.every((row) => row.readbackOutputId === row.durableOutputId)).toBe(true); expect(readback.outputs.every((row) => row.preRestartPid !== row.postRestartPid)).toBe(true); expect( @@ -291,6 +296,15 @@ test("GREEN: real process output closure covers every canonical registry key and row.resultDigest === row.durableOutputId, ), ).toBe(true); + const repeatedReadback = await verifyTrackBExtensionClosureAfterRestart( + runtime, + closure as never, + readbackOptions, + ); + expect(repeatedReadback.outputs).toHaveLength(readback.outputs.length); + expect( + repeatedReadback.outputs.every((row) => row.readbackOutputId === row.durableOutputId), + ).toBe(true); }); test("GREEN: rejects missing, health-only, and duplicate durable extension outputs", async () => { From a9f582cc8c88c17cae65e8580ee1242a62875e7e Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 05:52:53 +0800 Subject: [PATCH 15/40] Correlate runtime aggregates end to end --- .../apps/runtime-host-bridge/src/cli.ts | 59 +++++++++++++- .../apps/runtime-host-bridge/src/index.ts | 33 +++++++- .../src/track-b-operations.ts | 3 +- .../src/track-b-runtime.ts | 78 ++++++++++++++++++- .../test/run94-sp5-sp10.test.ts | 26 ++++++- .../test/track-b-operations-api.test.ts | 6 ++ 6 files changed, 194 insertions(+), 11 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/cli.ts b/role-model-router/apps/runtime-host-bridge/src/cli.ts index 4ac3e8c6..2bd8e786 100644 --- a/role-model-router/apps/runtime-host-bridge/src/cli.ts +++ b/role-model-router/apps/runtime-host-bridge/src/cli.ts @@ -20,15 +20,18 @@ import { validateRun88PrivateDistributionIdentity } from "./kw-private-loader.js import { type RuntimeChannelProfile, readPackagedRuntimeProfile } from "./runtime-channel.js"; import { migrateLegacyProductionState } from "./runtime-state-migration.js"; import { resolveRun88StageRuntimeIdentity } from "./runtime-version.js"; +import { createTrackBOperations } from "./track-b-operations.js"; import { type TrackBExtensionClosure, createOwnedTrackBSidecarSpec, createPackagedProductionRuntime, createProductionExtensionRuntime, createRun88RuntimeCorrelation, + createRuntimeRequestCorrelationId, createTrackBPostObservationOutbox, resolveManagedArtifactKeyFiles, runTrackBPostObservation, + runTrackBPostObservationWithContribution, trackBDistributionRequiresSQLiteMaintenance, validateRun88ProviderResponseObservation, verifyTrackBExtensionClosureAfterRestart, @@ -229,6 +232,11 @@ export function createRun88StagePostObservation(input: { sourceId: input.sourceId, deploymentId: `local-stage:${input.executableSha256}`, scope: input.scope, + correlationId: createRuntimeRequestCorrelationId({ + scope: input.scope, + requestId: String(input.observation.requestId ?? ""), + routingDecisionId: String(input.observation.routingDecisionId ?? ""), + }), }), }); } @@ -825,6 +833,19 @@ export async function main(): Promise { packagedManifestRecord, ); const packagedReleaseId = run88StageIdentity?.releaseId; + const packagedExecutableSha256 = String(packagedManifestRecord?.executable_sha256 ?? ""); + const aggregateCorrelationReleaseId = + run88StageIdentity?.releaseId ?? + (/^[a-f0-9]{64}$/.test(packagedExecutableSha256) + ? `sha256:${packagedExecutableSha256}` + : undefined); + const aggregateCorrelationCohortId = packagedProfile + ? packagedProfile.channel === "stage" + ? "stage-1pct" + : packagedProfile.channel === "development" + ? "development-default" + : undefined + : undefined; const loadRun88PiInvocationProvenance = run88StageIdentity ? () => readRun88PiInvocationProvenance(process.env, run88StageIdentity.releaseId) : null; @@ -967,11 +988,12 @@ export async function main(): Promise { "post-observation-outbox.json", ), }); + let postObservationOperations: ReturnType | null = null; const drainPostObservationOutbox = async ( runtime: Awaited>, ) => - postObservationOutbox.drain((observation) => - runTrackBPostObservation(runtime, observation, { + postObservationOutbox.drain((observation) => { + const processingInput = { scope: options.scopeId, channel: packagedProfile?.channel ?? "development", authorizationEpoch: 1, @@ -981,13 +1003,35 @@ export async function main(): Promise { run88Correlation: observation.run88Correlation as Record, } : {}), - }), - ); + } as const; + const operations = postObservationOperations; + return operations + ? runTrackBPostObservationWithContribution( + runtime, + observation, + processingInput, + (aggregate) => operations.recordContributionAggregate(aggregate), + ) + : runTrackBPostObservation(runtime, observation, processingInput); + }); const createBackend = async ( trackBOperationsEndpoint?: string, trackBOperationsToken?: string, runStartupSQLiteMaintenance = true, ) => { + postObservationOperations = trackBOperationsEndpoint + ? createTrackBOperations({ + statePath: path.join( + options.runtimeStateRoot, + options.scopeId, + "track-b-production-bridge.json", + ), + catalog: [], + runtimeChannel: packagedProfile?.channel ?? "development", + operationsEndpoint: trackBOperationsEndpoint, + operationsToken: trackBOperationsToken, + }) + : null; const created = await createRuntimeBridgeBackend({ fixtureRoot: resolveCliFixtureRoot(options.repoRoot, args.values["fixture-root"]), repoRoot: options.repoRoot, @@ -1190,6 +1234,13 @@ export async function main(): Promise { args.values["aggregate-ingestion-url"] ?? process.env.ROLE_MODEL_AGGREGATE_INGESTION_URL, aggregateScope: args.values["aggregate-scope"] ?? process.env.ROLE_MODEL_AGGREGATE_SCOPE, + ...(aggregateCorrelationReleaseId && aggregateCorrelationCohortId + ? { + aggregateCorrelationReleaseId, + aggregateCorrelationCohortId, + aggregateCorrelationOperationId: "aggregate.upload", + } + : {}), ...(manifest.publicRuntimeAdapter ? { sqliteDatabasePath: path.join( diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index 61aba5a3..dae07507 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -170,7 +170,11 @@ import { buildVerifiersLiveExport, createTrackBOperations as createTrackBOperationsFromState, } from "./track-b-operations.js"; -import { createRun88RuntimeCorrelation, createTrackBFileGraphStore } from "./track-b-runtime.js"; +import { + createRun88RuntimeCorrelation, + createRuntimeRequestCorrelationId, + createTrackBFileGraphStore, +} from "./track-b-runtime.js"; import { type ProviderRequestCapture, @@ -23864,6 +23868,11 @@ export async function createRuntimeBridgeBackend( } : {}), }); + const correlationId = createRuntimeRequestCorrelationId({ + scope: options.scopeId, + requestId, + routingDecisionId, + }); const run88Correlation = options.run88StageIdentity ? createRun88RuntimeCorrelation({ requestId, @@ -23873,6 +23882,7 @@ export async function createRuntimeBridgeBackend( sourceId: options.run88StageIdentity.sourceId, deploymentId: `local-${runtimeChannel}:${options.run88StageIdentity.executableSha256}`, scope: options.scopeId, + correlationId, }) : undefined; const bundle = Object.freeze({ @@ -23880,6 +23890,7 @@ export async function createRuntimeBridgeBackend( providerEvidence: buildProviderEvidenceFromObservation( baseBundle as unknown as Readonly>, ), + correlationId, ...(run88Correlation ? { run88Correlation } : {}), }); let artifactRef: @@ -24640,6 +24651,11 @@ export async function createRuntimeBridgeBackend( try { await trackBOperations.recordContributionAggregate({ requestId, + correlationId: createRuntimeRequestCorrelationId({ + scope: options.scopeId, + requestId, + routingDecisionId, + }), routingDecisionId, endpointId: execution.target.endpointId, modelId: bridgeResult.model, @@ -24804,6 +24820,11 @@ export async function createRuntimeBridgeBackend( try { await trackBOperations.recordContributionAggregate({ requestId, + correlationId: createRuntimeRequestCorrelationId({ + scope: options.scopeId, + requestId, + routingDecisionId, + }), routingDecisionId, endpointId: execution.target.endpointId, modelId: bridgeResult.model, @@ -27322,6 +27343,16 @@ export async function createRuntimeBridgeBackend( } return { ...value, + correlationId: + typeof (value as unknown as Record).correlationId === "string" + ? (value as unknown as Record).correlationId + : createRuntimeRequestCorrelationId({ + scope: options.scopeId, + requestId, + routingDecisionId: String( + (value as unknown as Record).routingDecisionId ?? "", + ), + }), ...(providerEvidence ? { providerEvidence } : {}), ...(graphEvidence ? { graphEvidence } : {}), ...(liveBudgetEvidence ? { liveBudgetEvidence } : {}), diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index 02767b99..a232c6c9 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -743,7 +743,8 @@ export function buildVerifiersLiveExport(input: { "only semantic live Verifiers export is available without token-exact evidence", ); const observationCorrelation = boundedIdentity( - recordValue(input.observation.run88Correlation).correlationId, + input.observation.correlationId ?? + recordValue(input.observation.run88Correlation).correlationId, "observation correlation id", ); if ( diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts index a0248280..ed9d5de0 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-runtime.ts @@ -552,6 +552,7 @@ export function createRun88RuntimeCorrelation(input: { readonly service?: string; readonly operation?: string; readonly outcome?: string; + readonly correlationId?: string; }): Record { for (const field of [ "requestId", @@ -570,11 +571,14 @@ export function createRun88RuntimeCorrelation(input: { const seed = `${input.releaseId}\0${input.requestId}\0${input.routingDecisionId}`; const hex = (label: string, length: number) => createHash("sha256").update(`${label}\0${seed}`).digest("hex").slice(0, length); + const correlationId = input.correlationId ?? `corr-${hex("correlation", 24)}`; + if (!/^corr-[a-f0-9]{24}$/.test(correlationId)) + throw new Error("Run 88 correlationId is invalid"); return normalizeRun88RuntimeCorrelation( { schemaVersion: "run88-correlation.v1", eventId: `evt-${hex("event", 24)}`, - correlationId: `corr-${hex("correlation", 24)}`, + correlationId, traceId: hex("trace", 32), spanId: hex("span", 16), causalParentId: input.routingDecisionId, @@ -595,6 +599,23 @@ export function createRun88RuntimeCorrelation(input: { ); } +export function createRuntimeRequestCorrelationId(input: { + readonly scope: string; + readonly requestId: string; + readonly routingDecisionId: string; +}): string { + for (const [field, value] of Object.entries(input)) { + if (typeof value !== "string" || !value.trim() || value.length > 512 || /[\r\n]/.test(value)) + throw new Error(`runtime request correlation ${field} is invalid`); + } + return `corr-${createHash("sha256") + .update( + `role-model.request-correlation.v1\0${input.scope}\0${input.requestId}\0${input.routingDecisionId}`, + ) + .digest("hex") + .slice(0, 24)}`; +} + export function validateRun88ProviderResponseObservation( observation: Readonly>, provenance: @@ -2374,6 +2395,49 @@ export async function runTrackBPostObservation( }; } +export async function runTrackBPostObservationWithContribution( + runtime: TrackBShadowPipelineRuntime, + observation: Readonly>, + input: { + readonly scope: string; + readonly channel: "development" | "stage" | "production"; + readonly authorizationEpoch: number; + readonly expectedReleaseId?: string; + readonly run88Correlation?: Record; + }, + recordContribution: (input: Record) => Promise, +) { + if (typeof recordContribution !== "function") + throw new Error("Track B contribution recorder is required"); + const result = await runTrackBPostObservation(runtime, observation, input); + const identity = normalizeTrackBVariantIdentity(observation); + const requestId = String(observation.requestId ?? ""); + const routingDecisionId = String(observation.routingDecisionId ?? ""); + const correlationId = createRuntimeRequestCorrelationId({ + scope: input.scope, + requestId, + routingDecisionId, + }); + const usageEvent = + observation.usageEvent && typeof observation.usageEvent === "object" + ? (observation.usageEvent as Record) + : {}; + const contribution = await recordContribution({ + requestId, + correlationId, + routingDecisionId, + endpointId: identity.endpointId, + modelId: identity.modelId, + reasoningEffort: identity.reasoningEffort, + effortSource: identity.effortSource, + taskType: "general.chat", + inputTokens: Number(usageEvent.tokens_in ?? 0), + outputTokens: Number(usageEvent.tokens_out ?? 0), + success: true, + }); + return { ...result, contribution }; +} + interface ExtensionRuntimeState { readonly id: string; readonly desiredState: "enabled" | "disabled"; @@ -2726,6 +2790,9 @@ export function createOwnedTrackBSidecarSpec(options: { trustMaterialFile?: string; aggregateEndpoint?: string; aggregateScope?: string; + aggregateCorrelationReleaseId?: string; + aggregateCorrelationCohortId?: string; + aggregateCorrelationOperationId?: string; sqliteDatabasePath?: string; publicRuntimeAdapterPath?: string; publicRouterRoot?: string; @@ -2773,6 +2840,15 @@ export function createOwnedTrackBSidecarSpec(options: { : []), ...(options.aggregateEndpoint ? ["--aggregate-endpoint", options.aggregateEndpoint] : []), ...(options.aggregateScope ? ["--aggregate-scope", options.aggregateScope] : []), + ...(options.aggregateCorrelationReleaseId + ? ["--aggregate-correlation-release-id", options.aggregateCorrelationReleaseId] + : []), + ...(options.aggregateCorrelationCohortId + ? ["--aggregate-correlation-cohort-id", options.aggregateCorrelationCohortId] + : []), + ...(options.aggregateCorrelationOperationId + ? ["--aggregate-correlation-operation-id", options.aggregateCorrelationOperationId] + : []), ...(options.sqliteDatabasePath ? ["--sqlite-database-path", options.sqliteDatabasePath] : []), diff --git a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts index 590cf4d0..36927add 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts @@ -11,6 +11,7 @@ import { createProductionExtensionRuntime, createTrackBPostObservationOutbox, runTrackBPostObservation, + runTrackBPostObservationWithContribution, verifyTrackBExtensionClosureAfterRestart, } from "../src/track-b-runtime.js"; @@ -240,11 +241,28 @@ test("GREEN: real process output closure covers every canonical registry key and ); roots.push(root); const runtime = await createRealCanonicalRuntime(root); - const result = await runTrackBPostObservation(runtime, observation(), { - scope: "tenant:run94", - channel: "development", - authorizationEpoch: 94, + let contributionInput: Record | null = null; + const result = await runTrackBPostObservationWithContribution( + runtime, + observation(), + { + scope: "tenant:run94", + channel: "development", + authorizationEpoch: 94, + }, + async (input) => { + contributionInput = input; + return { status: "uploaded" }; + }, + ); + expect(contributionInput).toMatchObject({ + requestId: "run94-extension-closure", + correlationId: expect.stringMatching(/^corr-[a-f0-9]{24}$/), + routingDecisionId: "decision:run94-extension-closure", + endpointId: "endpoint:run94", + modelId: "model:run94", }); + expect((result as Record).contribution).toEqual({ status: "uploaded" }); const closure = (result as Record).extensionClosure as Record; expect(Object.keys(closure.registry ?? {})).toEqual(canonicalExtensions.map(([id]) => id).sort()); const outputs = Object.values(closure.registry as Record).flatMap( diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index ee251608..a959c876 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -744,6 +744,7 @@ describe("Track B operations APIs", () => { authorization: `Bearer ${"a".repeat(64)}`, body: { requestId: "req-track-b-upload-001", + correlationId: expect.stringMatching(/^corr-[a-f0-9]{24}$/), routingDecisionId: result.routingDecisionId, endpointId: result.endpointId, modelId: "deepseek/chat-capture-v1", @@ -788,6 +789,9 @@ describe("Track B operations APIs", () => { }, }); const detail = await backend.readRequestObservation("req-track-b-upload-001"); + expect(detail).toMatchObject({ + correlationId: aggregate?.body.correlationId, + }); expect(detail?.providerEvidence).toMatchObject({ endpointId: result.endpointId, modelId: "deepseek/chat-capture-v1", @@ -928,6 +932,7 @@ describe("Track B operations APIs", () => { authorization: `Bearer ${"b".repeat(64)}`, body: { requestId: "req-track-b-responses-upload-001", + correlationId: expect.stringMatching(/^corr-[a-f0-9]{24}$/), routingDecisionId: result.routingDecisionId, endpointId: result.endpointId, modelId: "deepseek/chat-capture-v1", @@ -941,6 +946,7 @@ describe("Track B operations APIs", () => { }); const serialized = JSON.stringify(aggregate?.body); expect(Object.keys(aggregate?.body ?? {}).sort()).toEqual([ + "correlationId", "effortSource", "endpointId", "inputTokens", From fc17a457c28a13f751d8e5bb3959fba68d2869e7 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:04:17 +0800 Subject: [PATCH 16/40] Expose measured provider latency in request detail --- role-model-router/apps/runtime-host-bridge/src/index.ts | 3 +++ .../apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts | 2 +- .../test/track-b-operations-api.test.ts | 7 +++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index dae07507..5def6fed 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -2740,6 +2740,7 @@ type BridgeRequestObservation = RuntimeObservationBundle & readonly structuredInspectionAvailable: boolean; readonly reason: string; }; + readonly latencyMs?: number | null; readonly effectiveCostUsd?: number; readonly costCalculationBasis?: string; readonly costCalculationVersion?: string; @@ -27578,6 +27579,7 @@ export async function createRuntimeBridgeBackend( reason: "Raw observation retention has expired or the preserved observation bundle is unavailable; canonical request detail is reconstructed from the telemetry ledger.", }, + latencyMs: telemetryRecord.latencyMs, effectiveCostUsd: telemetryRecord.effectiveCostUsd, costCalculationBasis: telemetryRecord.costCalculationBasis, costCalculationVersion: telemetryRecord.costCalculationVersion, @@ -27603,6 +27605,7 @@ export async function createRuntimeBridgeBackend( }, ...(telemetryRecord ? { + latencyMs: telemetryRecord.latencyMs, effectiveCostUsd: telemetryRecord.effectiveCostUsd, costCalculationBasis: telemetryRecord.costCalculationBasis, costCalculationVersion: telemetryRecord.costCalculationVersion, diff --git a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts index 36927add..48f6cc2a 100644 --- a/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/run94-sp5-sp10.test.ts @@ -323,7 +323,7 @@ test("GREEN: real process output closure covers every canonical registry key and expect( repeatedReadback.outputs.every((row) => row.readbackOutputId === row.durableOutputId), ).toBe(true); -}); +}, 20_000); test("GREEN: rejects missing, health-only, and duplicate durable extension outputs", async () => { const base = async (id: string, envelope: Record) => ({ diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index a959c876..e11ba872 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -5,6 +5,7 @@ import os from "node:os"; import path from "node:path"; import { + readRuntimeTelemetryRecord, readRuntimeObservationStorageRecord, resolveSqliteMemoryLocation, } from "@role-model-router/sqlite-memory"; @@ -789,8 +790,14 @@ describe("Track B operations APIs", () => { }, }); const detail = await backend.readRequestObservation("req-track-b-upload-001"); + const telemetry = readRuntimeTelemetryRecord({ + databasePath, + requestId: "req-track-b-upload-001", + }); + expect(telemetry?.latencyMs).toEqual(expect.any(Number)); expect(detail).toMatchObject({ correlationId: aggregate?.body.correlationId, + latencyMs: telemetry?.latencyMs, }); expect(detail?.providerEvidence).toMatchObject({ endpointId: result.endpointId, From 24b5da7f52ef07740dab7a38fe10da85bb59b8ad Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:18:33 +0800 Subject: [PATCH 17/40] Preserve backend context for Verifiers export --- .../apps/runtime-host-bridge/src/index.ts | 2 +- .../test/track-b-operations-api.test.ts | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index 5def6fed..550c3e46 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -27640,7 +27640,7 @@ export async function createRuntimeBridgeBackend( if (runtimeChannel === "production") throw new Error("live Verifiers export is restricted to development and stage channels"); const requestId = typeof body.requestId === "string" ? body.requestId : ""; - const observation = await this.readRequestObservation(requestId); + const observation = await backend.readRequestObservation(requestId); if (!observation) throw new Error("runtime observation not found for Verifiers export"); const capture = await readExactRouteCapture(requestId); if (!capture) throw new Error("exact live graph is unavailable for Verifiers export"); diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index e11ba872..30009cde 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -846,6 +846,37 @@ describe("Track B operations APIs", () => { readiness: "semantic", }), ).toMatchObject({ responseNodeIndex: 1, tokenExactDisposition: "refused_missing_evidence" }); + const exportServer = await startBridgeServer({ + host: "127.0.0.1", + port: 0, + registry: backend.registry, + getRegistry: () => backend.registry, + executeChatCompletions: backend.executeChatCompletions, + executeResponses: backend.executeResponses, + exportVerifiersTrace: backend.exportVerifiersTrace, + }); + try { + const response = await fetch( + `http://127.0.0.1:${exportServer.port}/api/role-model/track-b/verifiers-export`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + requestId: "req-track-b-upload-001", + correlationId, + graphRootArtifactId: "artifact-route-capture", + readiness: "semantic", + }), + }, + ); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + requestId: "req-track-b-upload-001", + graphRootArtifactId: "artifact-route-capture", + }); + } finally { + await exportServer.close(); + } } finally { await backend.shutdown(); await new Promise((resolve, reject) => From 26c73c935e86366b57e9dc50363c63a4ac222d9a Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:29:09 +0800 Subject: [PATCH 18/40] Conform live exports to Verifiers TraceTask --- .../runtime-host-bridge/src/track-b-operations.ts | 11 ++++++++++- .../test/track-b-operations-api.test.ts | 7 +++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index a232c6c9..f05c861b 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -758,6 +758,15 @@ export function buildVerifiersLiveExport(input: { throw new Error("Verifiers export does not reference the exact live graph and router decision"); const messages = Array.isArray(input.capture.messages) ? input.capture.messages : []; const response = recordValue(input.capture.response); + const taskPromptMessage = + messages.find((value) => recordValue(value).role === "user") ?? messages[0]; + const taskPromptContent = recordValue(taskPromptMessage).content; + if (taskPromptContent === undefined) + throw new Error("Verifiers TraceTask prompt content is required"); + const taskPrompt = + typeof taskPromptContent === "string" + ? taskPromptContent + : JSON.stringify(taskPromptContent); const semanticMessages = [...messages, response].map((value, index) => { const message = recordValue(value); const role = boundedIdentity(message.role, `Verifiers node ${index + 1} role`); @@ -797,7 +806,7 @@ export function buildVerifiersLiveExport(input: { tokenExactDisposition: "refused_missing_evidence", trace: { id: traceId, - task: { type: "RoleModelTraceTask", data: { requestId } }, + task: { type: "RoleModelTraceTask", data: { idx: 0, prompt: taskPrompt } }, nodes: semanticMessages, rewards: {}, metrics: {}, diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 30009cde..0ea3aa87 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -135,6 +135,13 @@ describe("Track B operations APIs", () => { responseNodeIndex: 4, tokenExactDisposition: "refused_missing_evidence", trace: { + task: { + type: "RoleModelTraceTask", + data: { + idx: 0, + prompt: "route this", + }, + }, nodes: [ { parent: null, message: { role: "system", content: "route safely" }, sampled: false }, { parent: 0, message: { role: "user", content: "route this" }, sampled: false }, From 72f875c0ef66e9d830ea5d500ba44bde8f2968f4 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 06:40:48 +0800 Subject: [PATCH 19/40] Project tool calls into Verifiers dialect --- .../src/track-b-operations.ts | 23 ++++++++++++++++++- .../test/track-b-operations-api.test.ts | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index f05c861b..053f3731 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -772,13 +772,34 @@ export function buildVerifiersLiveExport(input: { const role = boundedIdentity(message.role, `Verifiers node ${index + 1} role`); if (!("content" in message)) throw new Error(`Verifiers node ${index + 1} content is required`); const toolCalls = Array.isArray(message.toolCalls) ? message.toolCalls : null; + const verifierToolCalls = toolCalls?.map((value, toolIndex) => { + const toolCall = recordValue(value); + const nestedFunction = recordValue(toolCall.function); + const argumentsValue = toolCall.arguments ?? nestedFunction.arguments; + if (argumentsValue === undefined) + throw new Error(`Verifiers node ${index + 1} tool call ${toolIndex + 1} arguments are required`); + return { + id: boundedIdentity( + toolCall.id, + `Verifiers node ${index + 1} tool call ${toolIndex + 1} id`, + ), + name: boundedIdentity( + toolCall.name ?? nestedFunction.name, + `Verifiers node ${index + 1} tool call ${toolIndex + 1} name`, + ), + arguments: + typeof argumentsValue === "string" + ? argumentsValue + : JSON.stringify(argumentsValue), + }; + }); const toolCallId = typeof message.toolCallId === "string" ? message.toolCallId : null; return { parent: index === 0 ? null : index - 1, message: { role, content: message.content, - ...(toolCalls ? { tool_calls: toolCalls } : {}), + ...(verifierToolCalls ? { tool_calls: verifierToolCalls } : {}), ...(toolCallId ? { tool_call_id: toolCallId } : {}), ...(typeof message.name === "string" ? { name: message.name } : {}), }, diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 0ea3aa87..404c28cc 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -145,7 +145,7 @@ describe("Track B operations APIs", () => { nodes: [ { parent: null, message: { role: "system", content: "route safely" }, sampled: false }, { parent: 0, message: { role: "user", content: "route this" }, sampled: false }, - { parent: 1, message: { role: "assistant", content: null, tool_calls: [{ id: "call-pi-94", type: "function", function: { name: "bash", arguments: '{"command":"printf run94-tool-ok"}' } }] }, sampled: false }, + { parent: 1, message: { role: "assistant", content: null, tool_calls: [{ id: "call-pi-94", name: "bash", arguments: '{"command":"printf run94-tool-ok"}' }] }, sampled: false }, { parent: 2, message: { role: "tool", content: "run94-tool-ok", tool_call_id: "call-pi-94", name: "bash" }, sampled: false }, { parent: 3, message: { role: "assistant", content: "routed" }, sampled: true }, ], From 469820bb77e9a862556f5cd0fef3740f9f7f995d Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:09:27 +0800 Subject: [PATCH 20/40] Capture provider failures in Track B graph --- .../apps/runtime-host-bridge/src/index.ts | 57 ++++++++++- .../src/track-b-operations.ts | 51 ++++++++-- .../runtime-host-bridge/test/index.test.ts | 95 +++++++++++++++++-- .../test/track-b-operations-api.test.ts | 68 ++++++++++++- 4 files changed, 249 insertions(+), 22 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index 550c3e46..770ff74f 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -22985,7 +22985,7 @@ export async function createRuntimeBridgeBackend( } return receipt; }; - const persistRoutedProviderFailure = (error: UpstreamExecutionError): void => { + const persistRoutedProviderFailure = async (error: UpstreamExecutionError): Promise => { if (executionOptions?.persistObservation === false || hasRuntimeTelemetryPersisted(error)) { return; } @@ -23132,7 +23132,7 @@ export async function createRuntimeBridgeBackend( costSavingsSupport: "partial", dimensions: selectedEndpointDimensions, } as const; - const failureObservation = { + const baseFailureObservation = { requestId, ...(executionOptions?.requestOptions?.clientRequestId ? { clientRequestId: executionOptions.requestOptions.clientRequestId } @@ -23276,6 +23276,46 @@ export async function createRuntimeBridgeBackend( }, }, }; + let routeCapture: Record | undefined; + try { + const requestBody = executionOptions?.requestBody ?? {}; + const captureInput = Array.isArray(requestBody.messages) + ? requestBody.messages + : Array.isArray(requestBody.input) + ? requestBody.input + : []; + routeCapture = (await runtimeTrackBOperations.recordLocalRouteCapture({ + requestId, + routingDecisionId, + endpointId: selectedEndpointId, + modelId: selectedModelId ?? selectedEndpointId, + reasoningEffort: selectedReasoningEffort, + effortSource: selectedEffortSource, + messages: captureInput, + failure: { + errorClass: error.errorClass, + statusCode: error.statusCode, + message: error.message, + }, + toolExecutions: [], + })) as Record; + } catch { + // Rich failure capture remains non-routing-critical and degrades to compact telemetry. + } + const graphEvidence = routeCapture + ? { + rootArtifactId: routeCapture.rootArtifactId, + messageNodeIds: Array.isArray(routeCapture.messageArtifactIds) + ? routeCapture.messageArtifactIds + : [], + responseNodeId: routeCapture.responseArtifactId, + edgeCount: routeCapture.edgeCount, + } + : undefined; + const failureObservation = Object.freeze({ + ...baseFailureObservation, + ...(graphEvidence ? { graphEvidence } : {}), + }); persistRuntimeTelemetryFailure({ databasePath: initialization.databasePath, requestId, @@ -23349,6 +23389,13 @@ export async function createRuntimeBridgeBackend( observation: failureObservation, ...(localGraphStore ? { graphStore: localGraphStore } : {}), }); + if (options.trackBPostObservation) { + try { + await options.trackBPostObservation(failureObservation); + } catch (postObservationError) { + console.error("Track B failure post-observation processing failed", postObservationError); + } + } markRuntimeTelemetryPersisted(error); emitTelemetryUpdate(requestId); }; @@ -23526,7 +23573,7 @@ export async function createRuntimeBridgeBackend( continue; } if (!error.fallbackEligible || deniedEndpointIds.includes(error.endpointId)) { - persistRoutedProviderFailure(error); + await persistRoutedProviderFailure(error); throw error; } deniedEndpointIds.push(error.endpointId); @@ -23534,11 +23581,11 @@ export async function createRuntimeBridgeBackend( try { nextRoute = routeExecutionRequest(deniedEndpointIds); } catch { - persistRoutedProviderFailure(error); + await persistRoutedProviderFailure(error); throw error; } if (nextRoute.routed.decision.chosen_endpoint_id.trim().length === 0) { - persistRoutedProviderFailure(error); + await persistRoutedProviderFailure(error); throwUnavailableExecutionTarget({ deniedEndpointIds: nextRoute.deniedEndpointIds, previousError: error, diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index 053f3731..07a471f9 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -738,6 +738,9 @@ export function buildVerifiersLiveExport(input: { input.request.graphRootArtifactId, "Verifiers export graph root artifact id", ); + const taskIndex = input.request.taskIndex; + if (!Number.isSafeInteger(taskIndex) || Number(taskIndex) < 0) + throw new Error("Verifiers TraceTask task index is required from external evaluation context"); if (input.request.readiness !== "semantic") throw new Error( "only semantic live Verifiers export is available without token-exact evidence", @@ -758,6 +761,26 @@ export function buildVerifiersLiveExport(input: { throw new Error("Verifiers export does not reference the exact live graph and router decision"); const messages = Array.isArray(input.capture.messages) ? input.capture.messages : []; const response = recordValue(input.capture.response); + const hasProviderFailure = input.capture.terminalState === "provider_error"; + const failureRecord = recordValue(input.capture.failure ?? response.failure); + const providerFailure = hasProviderFailure + ? { + errorClass: boundedIdentity(failureRecord.errorClass, "provider failure class"), + message: boundedIdentity(failureRecord.message, "provider failure message"), + statusCode: + failureRecord.statusCode === null || failureRecord.statusCode === undefined + ? null + : Number(failureRecord.statusCode), + } + : null; + if ( + providerFailure && + providerFailure.statusCode !== null && + (!Number.isInteger(providerFailure.statusCode) || + providerFailure.statusCode < 100 || + providerFailure.statusCode > 599) + ) + throw new Error("provider failure status code is invalid"); const taskPromptMessage = messages.find((value) => recordValue(value).role === "user") ?? messages[0]; const taskPromptContent = recordValue(taskPromptMessage).content; @@ -803,12 +826,14 @@ export function buildVerifiersLiveExport(input: { ...(toolCallId ? { tool_call_id: toolCallId } : {}), ...(typeof message.name === "string" ? { name: message.name } : {}), }, - sampled: index === messages.length, + sampled: index === messages.length && providerFailure === null, token_ids: [], mask: [], is_content: [], logprobs: [], - ...(index === messages.length ? { finish_reason: "stop" } : {}), + ...(index === messages.length + ? { finish_reason: providerFailure ? "error" : "stop" } + : {}), }; }); const routingDecisionId = boundedIdentity( @@ -827,17 +852,23 @@ export function buildVerifiersLiveExport(input: { tokenExactDisposition: "refused_missing_evidence", trace: { id: traceId, - task: { type: "RoleModelTraceTask", data: { idx: 0, prompt: taskPrompt } }, + task: { type: "RoleModelTraceTask", data: { idx: Number(taskIndex), prompt: taskPrompt } }, nodes: semanticMessages, rewards: {}, metrics: {}, info: { - limitations: ["semantic projection; provider-native tokens unavailable"], + limitations: [ + "semantic projection; provider-native tokens unavailable", + ...(providerFailure ? ["provider failed before a sampled completion"] : []), + ], routeDecisionId: routingDecisionId, roleModelGraphRootArtifactId: graphRootArtifactId, roleModelResponseNodeId: responseNodeId, roleModelRequestId: requestId, roleModelCorrelationId: correlationId, + ...(providerFailure && providerFailure.statusCode !== null + ? { providerStatusCode: providerFailure.statusCode } + : {}), roleModelToolNodeIds: (Array.isArray(input.capture.tools) ? input.capture.tools : []).map( (tool, index) => boundedIdentity(recordValue(tool).nodeId, `graph tool node ${index + 1}`), @@ -854,8 +885,16 @@ export function buildVerifiersLiveExport(input: { ), }, is_completed: true, - stop_condition: "role_model_graph_complete", - errors: [], + stop_condition: providerFailure ? "provider_error" : "role_model_graph_complete", + errors: providerFailure + ? [ + { + type: providerFailure.errorClass, + message: providerFailure.message, + traceback: null, + }, + ] + : [], }, }); } diff --git a/role-model-router/apps/runtime-host-bridge/test/index.test.ts b/role-model-router/apps/runtime-host-bridge/test/index.test.ts index 511ea741..dbbd29b8 100644 --- a/role-model-router/apps/runtime-host-bridge/test/index.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/index.test.ts @@ -15823,15 +15823,58 @@ describe("runtime-host-bridge", () => { process.env.DEEPSEEK_FAILURE_CAPTURE_API_KEY = "deepseek-failure-capture-key"; const requestId = "req-runtime-bridge-routed-provider-failure-001"; const seenRequestBodies: unknown[] = []; + const routeCaptures: Record[] = []; + const postObservations: Readonly>[] = []; + const graphRootArtifactId = "a".repeat(64); + const graphResponseArtifactId = "b".repeat(64); + const operationsServer = createServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { + body += chunk; + }); + request.on("end", () => { + if (request.method !== "POST" || request.url !== "/capture/route") { + response.statusCode = 404; + response.end(); + return; + } + routeCaptures.push(JSON.parse(body) as Record); + response.setHeader("content-type", "application/json"); + response.end( + JSON.stringify({ + scope: "runtime-host-failure-capture-tests", + rootArtifactId: graphRootArtifactId, + rootArtifactDigest: graphRootArtifactId, + messageArtifactIds: ["c".repeat(64)], + responseArtifactId: graphResponseArtifactId, + edgeCount: 2, + }), + ); + }); + }); + await new Promise((resolve, reject) => { + operationsServer.once("error", reject); + operationsServer.listen(0, "127.0.0.1", resolve); + }); + const operationsAddress = operationsServer.address(); + if (!operationsAddress || typeof operationsAddress === "string") { + throw new Error("failure capture operations server did not bind a TCP port"); + } const backend = await ( bridge as { createRuntimeBridgeBackend: (options: { repoRoot: string; fixtureRoot: string; - runtimeStateRoot: string; - scopeId: string; - networkFetcher?: typeof fetch; + runtimeStateRoot: string; + scopeId: string; + networkFetcher?: typeof fetch; + trackBOperationsEndpoint?: string; + trackBOperationsToken?: string; + trackBPostObservation?: ( + observation: Readonly>, + ) => Promise; }) => Promise<{ upsertProviderAccount: (body: Record) => Promise; activateEndpoint: (body: Record) => Promise<{ endpointId: string }>; @@ -15868,6 +15911,12 @@ describe("runtime-host-bridge", () => { fixtureRoot: path.join(repoRoot, "testdata", "router-runtime", "fixtures"), runtimeStateRoot, scopeId: "runtime-host-failure-capture-tests", + trackBOperationsEndpoint: `http://127.0.0.1:${operationsAddress.port}/`, + trackBOperationsToken: "failure-capture-test-token", + trackBPostObservation: async (observation) => { + postObservations.push(observation); + return { status: "processed" }; + }, networkFetcher: async (input, init) => { const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; @@ -15940,6 +15989,30 @@ describe("runtime-host-bridge", () => { ), ).rejects.toThrow(/Insufficient Balance/); expect(seenRequestBodies).toHaveLength(1); + expect(routeCaptures).toEqual([ + expect.objectContaining({ + requestId, + endpointId: endpoint.endpointId, + messages: [{ role: "user", content: "Return OK." }], + failure: { + errorClass: "quota_exhausted", + statusCode: 402, + message: "Insufficient Balance", + }, + }), + ]); + expect(postObservations).toEqual([ + expect.objectContaining({ + requestId, + endpointId: endpoint.endpointId, + graphEvidence: { + rootArtifactId: graphRootArtifactId, + messageNodeIds: ["c".repeat(64)], + responseNodeId: graphResponseArtifactId, + edgeCount: 2, + }, + }), + ]); const telemetryRows = await backend.listTelemetryRequests(); const failureRow = telemetryRows.find((row) => row.requestId === requestId); @@ -15991,13 +16064,12 @@ describe("runtime-host-bridge", () => { }), ], }), - telemetrySnapshot: expect.objectContaining({ - providerId: "deepseek", - providerAccountId: "deepseek.personal.failure-capture", - requestedModelId: "deepseek/deepseek-v4-pro", - selectedModelId: "deepseek/deepseek-v4-pro", - eligibleEndpointIds: [endpoint.endpointId], - }), + graphEvidence: { + rootArtifactId: graphRootArtifactId, + messageNodeIds: ["c".repeat(64)], + responseNodeId: graphResponseArtifactId, + edgeCount: 2, + }, }), ); } finally { @@ -16007,6 +16079,9 @@ describe("runtime-host-bridge", () => { process.env.DEEPSEEK_FAILURE_CAPTURE_API_KEY = originalApiKey; } await backend.shutdown(); + await new Promise((resolve, reject) => + operationsServer.close((error) => (error ? reject(error) : resolve())), + ); await rm(runtimeStateRoot, { recursive: true, force: true }); } }); diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 404c28cc..51e0d956 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -122,6 +122,7 @@ describe("Track B operations APIs", () => { correlationId: "correlation-export-94", graphRootArtifactId: "root-export-94", readiness: "semantic", + taskIndex: 7, }, observation, capture, @@ -138,7 +139,7 @@ describe("Track B operations APIs", () => { task: { type: "RoleModelTraceTask", data: { - idx: 0, + idx: 7, prompt: "route this", }, }, @@ -164,11 +165,74 @@ describe("Track B operations APIs", () => { correlationId: "correlation-export-94", graphRootArtifactId: "wrong-root", readiness: "semantic", + taskIndex: 7, }, observation, capture, }), ).toThrow(/exact live graph/i); + expect(() => + buildVerifiersLiveExport({ + channel: "development", + request: { + requestId: "request-export-94", + correlationId: "correlation-export-94", + graphRootArtifactId: "root-export-94", + readiness: "semantic", + }, + observation, + capture, + }), + ).toThrow(/task index/i); + const failureExport = buildVerifiersLiveExport({ + channel: "development", + request: { + requestId: "request-export-94", + correlationId: "correlation-export-94", + graphRootArtifactId: "root-export-94", + readiness: "semantic", + taskIndex: 8, + }, + observation, + capture: { + ...capture, + terminalState: "provider_error", + failure: { + errorClass: "provider_unavailable", + statusCode: 503, + message: "provider unavailable", + }, + response: { + nodeId: "node-response-94", + role: "assistant", + content: null, + failure: { + errorClass: "provider_unavailable", + statusCode: 503, + message: "provider unavailable", + }, + }, + }, + }); + expect(failureExport).toMatchObject({ + trace: { + task: { data: { idx: 8, prompt: "route this" } }, + nodes: expect.arrayContaining([ + expect.objectContaining({ + message: { role: "assistant", content: null }, + sampled: false, + finish_reason: "error", + }), + ]), + is_completed: true, + stop_condition: "provider_error", + errors: [{ type: "provider_unavailable", message: "provider unavailable", traceback: null }], + info: expect.objectContaining({ + providerStatusCode: 503, + limitations: expect.arrayContaining(["provider failed before a sampled completion"]), + }), + }, + }); }); test("reads one exact graph capture through the authenticated loopback sidecar", async () => { @@ -851,6 +915,7 @@ describe("Track B operations APIs", () => { correlationId, graphRootArtifactId: "artifact-route-capture", readiness: "semantic", + taskIndex: 3, }), ).toMatchObject({ responseNodeIndex: 1, tokenExactDisposition: "refused_missing_evidence" }); const exportServer = await startBridgeServer({ @@ -873,6 +938,7 @@ describe("Track B operations APIs", () => { correlationId, graphRootArtifactId: "artifact-route-capture", readiness: "semantic", + taskIndex: 3, }), }, ); From 66e160bf389d3ba403d16888d7697753b3aad2f8 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:53:28 +0800 Subject: [PATCH 21/40] Recover consumed provider failure evidence --- .../apps/runtime-host-bridge/src/cli.ts | 4 + .../apps/runtime-host-bridge/src/index.ts | 112 ++++++++++++++++ .../src/track-b-operations.ts | 63 ++++++++- .../test/track-b-operations-api.test.ts | 126 ++++++++++++++++++ .../sqlite-memory/src/legacy-migration.ts | 26 +++- .../run94-storage-corrections-red.test.ts | 17 ++- 6 files changed, 345 insertions(+), 3 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/cli.ts b/role-model-router/apps/runtime-host-bridge/src/cli.ts index 2bd8e786..f5669a0f 100644 --- a/role-model-router/apps/runtime-host-bridge/src/cli.ts +++ b/role-model-router/apps/runtime-host-bridge/src/cli.ts @@ -106,6 +106,7 @@ type CliBackend = Pick< | "listRecentRequestObservations" | "readRequestObservation" | "exportVerifiersTrace" + | "recoverLegacyTerminalFailure" | "readEndpointProfile" | "readBenchmarkSuite" | "runBenchmark" @@ -559,6 +560,9 @@ export function createCliServerOptions( exportVerifiersTrace: bindBackendMethod( "exportVerifiersTrace", ) as StartBridgeServerOptions["exportVerifiersTrace"], + recoverLegacyTerminalFailure: bindBackendMethod( + "recoverLegacyTerminalFailure", + ) as StartBridgeServerOptions["recoverLegacyTerminalFailure"], readEndpointProfile: bindBackendMethod( "readEndpointProfile", ) as StartBridgeServerOptions["readEndpointProfile"], diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index 770ff74f..c52d6bb7 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -166,6 +166,7 @@ import { readPackagedRuntimeProfile, resolveRuntimeChannelProfile } from "./runt import { type RuntimeVersionInfoRecord, resolveRuntimeVersionInfo } from "./runtime-version.js"; import { buildGraphEvidenceFromCapture, + buildLegacyTerminalFailureRecoveryCapture, buildProviderEvidenceFromObservation, buildVerifiersLiveExport, createTrackBOperations as createTrackBOperationsFromState, @@ -2948,6 +2949,7 @@ export interface StartBridgeServerOptions { readonly subscribeTelemetry?: (listener: (event: RuntimeBridgeStreamEvent) => void) => () => void; readonly readRequestObservation?: (requestId: string) => Promise; readonly exportVerifiersTrace?: (body: Record) => Promise; + readonly recoverLegacyTerminalFailure?: (body: Record) => Promise; readonly readEndpointProfile?: (endpointId: string) => Promise; readonly readBenchmarkSuite?: () => Promise; readonly runBenchmark?: (body: Record) => Promise; @@ -3204,6 +3206,7 @@ export interface RuntimeBridgeBackend { subscribeTelemetry(listener: (event: RuntimeBridgeStreamEvent) => void): () => void; readRequestObservation(requestId: string): Promise; exportVerifiersTrace(body: Record): Promise; + recoverLegacyTerminalFailure(body: Record): Promise; readEndpointProfile(endpointId: string): Promise<{ endpointId: string; latestProfile: ReturnType; @@ -14889,6 +14892,28 @@ function createRequestHandler(options: StartBridgeServerOptions) { return; } + if ( + request.method === "POST" && + url.pathname === "/api/role-model/track-b/recover-terminal-failure" + ) { + if (!options.recoverLegacyTerminalFailure) { + writeJson(response, 404, { error: "not found" }); + return; + } + try { + writeJson( + response, + 200, + await options.recoverLegacyTerminalFailure(await readJsonBody(request)), + ); + } catch (error) { + writeJson(response, 409, { + error: error instanceof Error ? error.message : String(error), + }); + } + return; + } + if (request.method === "GET" && url.pathname === "/api/role-model/graph-migration") { if (!options.readGraphMigration) { writeJson(response, 404, { error: "not found" }); @@ -27683,6 +27708,93 @@ export async function createRuntimeBridgeBackend( } return attachLiveEvidence(requestDetail); }, + async recoverLegacyTerminalFailure(body: Record): Promise { + if (runtimeChannel === "production") + throw new Error("legacy terminal failure recovery is restricted to development and stage channels"); + const allowedKeys = new Set(["requestId", "acknowledgeMetadataOnly"]); + const unexpectedKeys = Object.keys(body).filter((key) => !allowedKeys.has(key)); + if (unexpectedKeys.length > 0) + throw new Error( + `legacy terminal failure recovery refuses caller-supplied content: ${unexpectedKeys.join(", ")}`, + ); + if (body.acknowledgeMetadataOnly !== true) + throw new Error("metadata-only recovery requires explicit acknowledgement"); + const requestId = typeof body.requestId === "string" ? body.requestId.trim() : ""; + if (!requestId) throw new Error("legacy terminal failure recovery request id is required"); + const observation = await backend.readRequestObservation(requestId); + if (!observation) throw new Error("persisted runtime observation not found for recovery"); + + let existingCapture: Record | null = null; + try { + existingCapture = await readExactRouteCapture(requestId); + } catch { + // An unknown capture is the expected precondition for this bounded upgrade path. + } + if (existingCapture) { + const existingRecovery = + existingCapture.recovery && typeof existingCapture.recovery === "object" + ? (existingCapture.recovery as Record) + : {}; + if ( + existingCapture.projectionCompleteness !== "metadata_only" || + existingCapture.terminalState !== "provider_error" || + existingRecovery.kind !== "legacy_terminal_failure" || + existingRecovery.source !== "persisted_runtime_observation" + ) + throw new Error("legacy terminal failure recovery refused because graph evidence already exists"); + return Object.freeze({ + schemaVersion: "role-model.legacy-terminal-failure-recovery.v1", + status: "already_recovered", + requestId, + routingDecisionId: observation.routingDecisionId, + correlationId: (observation as unknown as Record).correlationId, + graphRootArtifactId: existingCapture.rootArtifactId, + projectionCompleteness: "metadata_only", + }); + } + + const captureInput = buildLegacyTerminalFailureRecoveryCapture( + observation as unknown as Readonly>, + ); + await runtimeTrackBOperations.recordLocalRouteCapture(captureInput as Record); + const capture = await readExactRouteCapture(requestId); + if (!capture) throw new Error("metadata-only recovery did not commit durable graph evidence"); + const graphEvidence = buildGraphEvidenceFromCapture(capture); + const recoveredObservation = Object.freeze({ + ...(observation as unknown as Record), + graphEvidence, + recovery: Object.freeze({ + kind: "legacy_terminal_failure", + source: "persisted_runtime_observation", + }), + }); + const extensionReceipts = options.trackBPostObservation + ? await options.trackBPostObservation(recoveredObservation) + : null; + const correlationId = (observation as unknown as Record).correlationId; + const receiptIdentity = JSON.stringify({ + requestId, + routingDecisionId: observation.routingDecisionId, + correlationId, + graphRootArtifactId: graphEvidence.rootArtifactId, + }); + return Object.freeze({ + schemaVersion: "role-model.legacy-terminal-failure-recovery.v1", + receiptId: `legacy-recovery-${createHash("sha256").update(receiptIdentity).digest("hex")}`, + status: "recovered", + requestId, + routingDecisionId: observation.routingDecisionId, + correlationId, + graphRootArtifactId: graphEvidence.rootArtifactId, + responseNodeId: graphEvidence.responseNodeId, + projectionCompleteness: "metadata_only", + recovery: Object.freeze({ + kind: "legacy_terminal_failure", + source: "persisted_runtime_observation", + }), + extensionProcessing: extensionReceipts === null ? "not_configured" : "completed", + }); + }, async exportVerifiersTrace(body: Record): Promise { if (runtimeChannel === "production") throw new Error("live Verifiers export is restricted to development and stage channels"); diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index 07a471f9..5ec18e47 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -674,6 +674,47 @@ export function buildProviderEvidenceFromObservation( return Object.freeze({ endpointId, modelId, status: failed ? "error" : "ok", attemptIds }); } +export function buildLegacyTerminalFailureRecoveryCapture( + observation: Readonly>, +): Readonly> { + const failure = recordValue(observation.failure); + if (observation.statusFamily !== "failure" || Object.keys(failure).length === 0) + throw new Error("legacy graph recovery requires a persisted terminal failure"); + const errorClass = boundedIdentity(failure.errorClass, "terminal failure class"); + const statusCode = Number(failure.statusCode); + if (!Number.isInteger(statusCode) || statusCode < 100 || statusCode > 599) + throw new Error("legacy graph recovery requires a persisted terminal failure status code"); + return Object.freeze({ + requestId: boundedIdentity(observation.requestId, "terminal failure request id"), + routingDecisionId: boundedIdentity( + observation.routingDecisionId, + "terminal failure route decision id", + ), + endpointId: boundedIdentity(observation.endpointId, "terminal failure endpoint id"), + modelId: boundedIdentity( + recordValue(observation.usageEvent).model_id ?? observation.modelId, + "terminal failure model id", + ), + reasoningEffort: observation.reasoningEffort ?? null, + effortSource: boundedIdentity( + observation.effortSource ?? "none", + "terminal failure effort source", + ), + messages: [], + projectionCompleteness: "metadata_only", + recovery: Object.freeze({ + kind: "legacy_terminal_failure", + source: "persisted_runtime_observation", + }), + failure: Object.freeze({ + errorClass, + statusCode, + message: `Persisted runtime observation recorded ${errorClass} (HTTP ${statusCode}).`, + }), + toolExecutions: [], + }); +} + export function buildGraphEvidenceFromCapture( capture: Readonly>, ): Readonly> { @@ -699,7 +740,18 @@ export function buildGraphEvidenceFromCapture( : null; const response = recordValue(capture.response); const edgeCount = Number(capture.edgeCount); - if (messages.length < 1 || !Number.isSafeInteger(edgeCount) || edgeCount < 1) + const metadataOnly = capture.projectionCompleteness === "metadata_only"; + const recovery = recordValue(capture.recovery); + if ( + !Number.isSafeInteger(edgeCount) || + edgeCount < 1 || + (messages.length < 1 && !metadataOnly) || + (metadataOnly && + (messages.length !== 0 || + capture.terminalState !== "provider_error" || + recovery.kind !== "legacy_terminal_failure" || + recovery.source !== "persisted_runtime_observation")) + ) throw new Error("exact live graph is incomplete"); return Object.freeze({ rootArtifactId: boundedIdentity(capture.rootArtifactId, "graph root artifact id"), @@ -718,6 +770,15 @@ export function buildGraphEvidenceFromCapture( toolResultNodeIds: toolRows .filter((tool) => tool.kind === "tool_result") .map((tool, index) => boundedIdentity(tool.nodeId, `graph tool result node ${index + 1}`)), + ...(metadataOnly + ? { + projectionCompleteness: "metadata_only", + recovery: Object.freeze({ + kind: "legacy_terminal_failure", + source: "persisted_runtime_observation", + }), + } + : {}), ...(captureMetrics ? { captureMetrics } : {}), edgeCount, }); diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 51e0d956..59124002 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -16,6 +16,7 @@ import { applyRecommendationServiceLauncherConfig } from "../src/cli.js"; import { createRuntimeBridgeBackend, startBridgeServer } from "../src/index.js"; import { buildGraphEvidenceFromCapture, + buildLegacyTerminalFailureRecoveryCapture, buildProviderEvidenceFromObservation, buildVerifiersLiveExport, createTrackBOperations, @@ -235,6 +236,112 @@ describe("Track B operations APIs", () => { }); }); + test("preserves metadata-only legacy failure recovery and refuses to invent a Verifiers prompt", () => { + const capture = { + schemaVersion: "role-model.route-capture-read.v1", + requestId: "request-legacy-failure-94", + routingDecisionId: "decision-legacy-failure-94", + rootArtifactId: "root-legacy-failure-94", + projectionCompleteness: "metadata_only", + recovery: { + kind: "legacy_terminal_failure", + source: "persisted_runtime_observation", + }, + messages: [], + response: { + nodeId: "response-legacy-failure-94", + role: "assistant", + content: null, + failure: { + errorClass: "provider_auth_error", + statusCode: 401, + message: "provider rejected router-owned credentials", + }, + }, + terminalState: "provider_error", + failure: { + errorClass: "provider_auth_error", + statusCode: 401, + message: "provider rejected router-owned credentials", + }, + tools: [], + edgeCount: 1, + }; + expect(buildGraphEvidenceFromCapture(capture)).toEqual({ + rootArtifactId: "root-legacy-failure-94", + messageNodeIds: [], + responseNodeId: "response-legacy-failure-94", + toolExecutionNodeIds: [], + toolCallNodeIds: [], + toolResultNodeIds: [], + projectionCompleteness: "metadata_only", + recovery: { + kind: "legacy_terminal_failure", + source: "persisted_runtime_observation", + }, + edgeCount: 1, + }); + expect(() => + buildVerifiersLiveExport({ + channel: "development", + request: { + requestId: "request-legacy-failure-94", + correlationId: "correlation-legacy-failure-94", + graphRootArtifactId: "root-legacy-failure-94", + readiness: "semantic", + taskIndex: 2, + }, + observation: { + requestId: "request-legacy-failure-94", + routingDecisionId: "decision-legacy-failure-94", + correlationId: "correlation-legacy-failure-94", + }, + capture, + }), + ).toThrow(/prompt content/i); + }); + + test("derives metadata-only recovery solely from a persisted terminal failure", () => { + const observation = { + requestId: "request-legacy-failure-94", + routingDecisionId: "decision-legacy-failure-94", + endpointId: "endpoint-legacy-failure-94", + reasoningEffort: "high", + effortSource: "variant", + statusFamily: "failure", + correlationId: "correlation-legacy-failure-94", + usageEvent: { model_id: "provider/model-legacy-failure-94" }, + failure: { errorClass: "provider_auth_error", statusCode: 401, latencyMs: 17 }, + inspection: { request: { requestCapture: { body: "must-not-be-recovered" } } }, + }; + expect(buildLegacyTerminalFailureRecoveryCapture(observation)).toEqual({ + requestId: "request-legacy-failure-94", + routingDecisionId: "decision-legacy-failure-94", + endpointId: "endpoint-legacy-failure-94", + modelId: "provider/model-legacy-failure-94", + reasoningEffort: "high", + effortSource: "variant", + messages: [], + projectionCompleteness: "metadata_only", + recovery: { + kind: "legacy_terminal_failure", + source: "persisted_runtime_observation", + }, + failure: { + errorClass: "provider_auth_error", + statusCode: 401, + message: "Persisted runtime observation recorded provider_auth_error (HTTP 401).", + }, + toolExecutions: [], + }); + expect(() => + buildLegacyTerminalFailureRecoveryCapture({ ...observation, statusFamily: "success" }), + ).toThrow(/terminal failure/i); + expect(() => + buildLegacyTerminalFailureRecoveryCapture({ ...observation, failure: undefined }), + ).toThrow(/terminal failure/i); + }); + test("reads one exact graph capture through the authenticated loopback sidecar", async () => { const received: Array<{ path: string; authorization?: string; body: unknown }> = []; const operations = createServer(async (request, response) => { @@ -441,6 +548,11 @@ describe("Track B operations APIs", () => { requestId: body.requestId, graphRootArtifactId: body.graphRootArtifactId, }), + recoverLegacyTerminalFailure: async (body) => ({ + schemaVersion: "role-model.legacy-terminal-failure-recovery.v1", + requestId: body.requestId, + status: "recovered", + }), }); try { const base = `http://127.0.0.1:${server.port}`; @@ -528,6 +640,20 @@ describe("Track B operations APIs", () => { requestId: "request-http-export-94", graphRootArtifactId: "root-http-export-94", }); + const recovered = await fetch(`${base}/api/role-model/track-b/recover-terminal-failure`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + requestId: "request-http-recovery-94", + acknowledgeMetadataOnly: true, + }), + }); + expect(recovered.status).toBe(200); + expect(await recovered.json()).toEqual({ + schemaVersion: "role-model.legacy-terminal-failure-recovery.v1", + requestId: "request-http-recovery-94", + status: "recovered", + }); } finally { await server.close(); await backend.shutdown(); diff --git a/role-model-router/packages/sqlite-memory/src/legacy-migration.ts b/role-model-router/packages/sqlite-memory/src/legacy-migration.ts index 328bfc0a..8a1f1f7b 100644 --- a/role-model-router/packages/sqlite-memory/src/legacy-migration.ts +++ b/role-model-router/packages/sqlite-memory/src/legacy-migration.ts @@ -175,6 +175,30 @@ export interface SqliteMigrationRegistry { readonly entries: readonly SqliteMigrationRegistryEntry[]; } +/** + * Resolves source-checkout migration assets without relying on module URL metadata. + * Packaged runtimes must pass their verified staged router root explicitly. + */ +export function resolveLegacyMigrationRouterRoot(startDirectory = process.cwd()): string { + const start = path.resolve(startDirectory); + const initialCandidates = [start, path.join(start, "role-model-router")]; + for (const candidate of initialCandidates) { + if (existsSync(path.join(candidate, "migrations", "registry.json"))) return candidate; + } + + let candidate = path.dirname(start); + while (true) { + if (existsSync(path.join(candidate, "migrations", "registry.json"))) return candidate; + const parent = path.dirname(candidate); + if (parent === candidate) break; + candidate = parent; + } + + throw new Error( + "SQLite migration registry was not found; packaged runtimes must provide an explicit routerRoot", + ); +} + function sha256(value: string | Buffer): string { return createHash("sha256").update(value).digest("hex"); } @@ -699,7 +723,7 @@ export class LegacySqliteMigration { this.#artifactWriter = input.artifactWriter; this.#artifactRollback = input.artifactRollback; this.#now = input.now ?? Date.now; - this.#routerRoot = input.routerRoot ?? path.resolve(import.meta.dirname, "../../.."); + this.#routerRoot = input.routerRoot ?? resolveLegacyMigrationRouterRoot(); } audit(): LegacyStorageAudit { diff --git a/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts b/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts index 5c367da3..64fe27bb 100644 --- a/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts +++ b/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync } from "node:fs"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { DatabaseSync } from "node:sqlite"; @@ -9,6 +9,7 @@ import { initializeSqliteMemory, persistRuntimeTelemetryFailure, readLegacyMigrationJournal, + resolveLegacyMigrationRouterRoot, } from "../src/index.js"; function newDatabase() { @@ -41,6 +42,20 @@ function insertObservation(databasePath: string, requestId: string, observationJ } describe("Run 94 public storage corrections", () => { + test("discovers the migration router root without import.meta bundle semantics", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "run94-router-root-")); + const routerRoot = path.join(root, "role-model-router"); + const nestedWorkingDirectory = path.join(routerRoot, "packages", "sqlite-memory"); + mkdirSync(path.join(routerRoot, "migrations"), { recursive: true }); + mkdirSync(nestedWorkingDirectory, { recursive: true }); + writeFileSync(path.join(routerRoot, "migrations", "registry.json"), "{}", "utf8"); + + expect(resolveLegacyMigrationRouterRoot(nestedWorkingDirectory)).toBe(routerRoot); + expect(() => resolveLegacyMigrationRouterRoot(path.join(root, "unrelated"))).toThrow( + /explicit routerRoot|migration registry/i, + ); + }); + test("RED: enforces the 16 KiB cap by UTF-8 bytes on UPDATE, not SQLite characters", () => { const { databasePath } = newDatabase(); insertObservation(databasePath, "req-utf8", JSON.stringify({ requestId: "req-utf8" })); From 47a4264e7797f6fe8170d673de3fe5da0d50be2d Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:10:02 +0800 Subject: [PATCH 22/40] Harden Track B recovery and migration identity --- .../apps/runtime-host-bridge/src/index.ts | 37 ++++-- .../src/track-b-operations.ts | 26 +++- .../test/track-b-operations-api.test.ts | 111 +++++++++++++++++- .../sqlite-memory/src/legacy-migration.ts | 23 +++- .../run94-storage-corrections-red.test.ts | 21 +++- 5 files changed, 203 insertions(+), 15 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index c52d6bb7..67a2188c 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -27700,7 +27700,7 @@ export async function createRuntimeBridgeBackend( const isFailureObservation = observationRecord.statusFamily === "failure" || Boolean(observationRecord.failure) || - telemetryRecord?.errorClass !== null; + telemetryRecord?.errorClass != null; if (!isFailureObservation) { const { inspection: _inspection, ...boundedRequestDetail } = requestDetail as BridgeRequestObservation & { inspection?: unknown }; @@ -27724,12 +27724,7 @@ export async function createRuntimeBridgeBackend( const observation = await backend.readRequestObservation(requestId); if (!observation) throw new Error("persisted runtime observation not found for recovery"); - let existingCapture: Record | null = null; - try { - existingCapture = await readExactRouteCapture(requestId); - } catch { - // An unknown capture is the expected precondition for this bounded upgrade path. - } + const existingCapture = await readExactRouteCapture(requestId); if (existingCapture) { const existingRecovery = existingCapture.recovery && typeof existingCapture.recovery === "object" @@ -27742,14 +27737,40 @@ export async function createRuntimeBridgeBackend( existingRecovery.source !== "persisted_runtime_observation" ) throw new Error("legacy terminal failure recovery refused because graph evidence already exists"); + const graphEvidence = buildGraphEvidenceFromCapture(existingCapture); + const recoveredObservation = Object.freeze({ + ...(observation as unknown as Record), + graphEvidence, + recovery: Object.freeze({ + kind: "legacy_terminal_failure", + source: "persisted_runtime_observation", + }), + }); + const extensionReceipts = options.trackBPostObservation + ? await options.trackBPostObservation(recoveredObservation) + : null; + const correlationId = (observation as unknown as Record).correlationId; + const receiptIdentity = JSON.stringify({ + requestId, + routingDecisionId: observation.routingDecisionId, + correlationId, + graphRootArtifactId: graphEvidence.rootArtifactId, + }); return Object.freeze({ schemaVersion: "role-model.legacy-terminal-failure-recovery.v1", + receiptId: `legacy-recovery-${createHash("sha256").update(receiptIdentity).digest("hex")}`, status: "already_recovered", requestId, routingDecisionId: observation.routingDecisionId, - correlationId: (observation as unknown as Record).correlationId, + correlationId, graphRootArtifactId: existingCapture.rootArtifactId, + responseNodeId: graphEvidence.responseNodeId, projectionCompleteness: "metadata_only", + recovery: Object.freeze({ + kind: "legacy_terminal_failure", + source: "persisted_runtime_observation", + }), + extensionProcessing: extensionReceipts === null ? "not_configured" : "completed", }); } diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index 5ec18e47..44fe7878 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -600,6 +600,16 @@ const runtimePlan = (state: BridgeState, sourceRevision: number): RetentionPlan }; }; +class TrackBPrivateOperationError extends Error { + readonly status: number; + + constructor(status: number, message: string) { + super(message); + this.name = "TrackBPrivateOperationError"; + this.status = status; + } +} + const privateRetentionRequest = async ( endpoint: string | undefined, token: string | undefined, @@ -622,7 +632,8 @@ const privateRetentionRequest = async ( }); const result = (await response.json().catch(() => ({}))) as { readonly error?: unknown }; if (!response.ok) - throw new Error( + throw new TrackBPrivateOperationError( + response.status, typeof result.error === "string" ? result.error : `private Track B operation failed with ${response.status}`, @@ -1740,7 +1751,18 @@ export function createTrackBOperations({ const url = new URL(operationsEndpoint); if (!["127.0.0.1", "localhost", "::1", "[::1]"].includes(url.hostname)) throw new Error("local route capture readback requires a loopback operations boundary"); - return requestPrivate("capture/read", { method: "POST", body: input }); + try { + return await requestPrivate("capture/read", { method: "POST", body: input }); + } catch (error) { + const requestId = String(input.requestId ?? ""); + if ( + error instanceof TrackBPrivateOperationError && + error.status === 409 && + error.message === `unknown route capture ${requestId}` + ) + return null; + throw error; + } }, async listRecommendations(): Promise { return (await readState(statePath)).recommendations ?? []; diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 59124002..4aee2b37 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -3,10 +3,11 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { createServer } from "node:http"; import os from "node:os"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { - readRuntimeTelemetryRecord, readRuntimeObservationStorageRecord, + readRuntimeTelemetryRecord, resolveSqliteMemoryLocation, } from "@role-model-router/sqlite-memory"; import { afterEach, describe, expect, test, vi } from "vitest"; @@ -1030,6 +1031,14 @@ describe("Track B operations APIs", () => { runtimeRssBytes: expect.any(Number), }); expect((detail as unknown as { liveBudgetEvidence: { compactObservationBytes: number } }).liveBudgetEvidence.compactObservationBytes).toBeLessThanOrEqual(16 * 1024); + const telemetryDatabase = new DatabaseSync(databasePath); + telemetryDatabase + .prepare("DELETE FROM runtime_telemetry_records WHERE request_id=?") + .run("req-track-b-upload-001"); + telemetryDatabase.close(); + expect(await backend.readRequestObservation("req-track-b-upload-001")).not.toHaveProperty( + "inspection", + ); const correlationId = String( (detail as unknown as { run88Correlation?: { correlationId?: string } })?.run88Correlation ?.correlationId, @@ -1084,6 +1093,106 @@ describe("Track B operations APIs", () => { } }); + test("legacy failure recovery retries extension closure after graph commit and propagates sidecar auth failures", async () => { + const runtimeStateRoot = path.join(os.tmpdir(), `track-b-recovery-retry-${Date.now()}`); + roots.push(runtimeStateRoot); + const scopeId = "track-b-recovery-retry"; + const databasePath = resolveSqliteMemoryLocation({ runtimeStateRoot, scopeId }); + let captureInput: Record | null = null; + let extensionAttempts = 0; + let rejectReadsAsUnauthorized = false; + const operations = createServer(async (request, response) => { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record; + if (request.url === "/capture/read" && rejectReadsAsUnauthorized) { + response.writeHead(401, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: "sidecar authorization refused" })); + return; + } + if (request.url === "/capture/read" && !captureInput) { + response.writeHead(409, { "content-type": "application/json" }); + response.end(JSON.stringify({ error: `unknown route capture ${body.requestId}` })); + return; + } + if (request.url === "/capture/route") { + captureInput = body; + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ status: "captured" })); + return; + } + const failure = captureInput?.failure as Record; + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ + schemaVersion: "role-model.route-capture-read.v1", + requestId: captureInput?.requestId, + routingDecisionId: captureInput?.routingDecisionId, + rootArtifactId: "root-recovered-94", + projectionCompleteness: "metadata_only", + recovery: captureInput?.recovery, + messages: [], + response: { nodeId: "response-recovered-94", role: "assistant", content: null, failure }, + terminalState: "provider_error", + failure, + tools: [], + edgeCount: 1, + })); + }); + await new Promise((resolve, reject) => { + operations.once("error", reject); + operations.listen(0, "127.0.0.1", resolve); + }); + const address = operations.address(); + if (!address || typeof address === "string") throw new Error("operations server did not bind"); + const backend = await createRuntimeBridgeBackend({ + repoRoot, + fixtureRoot, + runtimeStateRoot, + scopeId, + runtimeChannel: "development", + trackBOperationsEndpoint: `http://127.0.0.1:${address.port}`, + trackBOperationsToken: "r".repeat(64), + trackBPostObservation: async () => { + extensionAttempts += 1; + if (extensionAttempts === 1) throw new Error("extension closure interrupted"); + return { status: "processed" }; + }, + }); + try { + await expect( + backend.executeChatCompletions( + { + model: "nonexistent/run94-recovery-provider", + messages: [{ role: "user", content: "exercise bounded recovery" }], + }, + "request-recovery-retry-94", + ), + ).rejects.toThrow(); + const input = { + requestId: "request-recovery-retry-94", + acknowledgeMetadataOnly: true, + }; + await expect(backend.recoverLegacyTerminalFailure(input)).rejects.toThrow( + /extension closure interrupted/i, + ); + await expect(backend.recoverLegacyTerminalFailure(input)).resolves.toMatchObject({ + status: "already_recovered", + extensionProcessing: "completed", + }); + expect(extensionAttempts).toBe(2); + + rejectReadsAsUnauthorized = true; + await expect(backend.recoverLegacyTerminalFailure(input)).rejects.toThrow( + /authorization refused/i, + ); + } finally { + await backend.shutdown(); + await new Promise((resolve, reject) => + operations.close((error) => (error ? reject(error) : resolve())), + ); + } + }); + test("production Responses requests report aggregate metrics through the same private operations boundary", async () => { const runtimeStateRoot = path.join( os.tmpdir(), diff --git a/role-model-router/packages/sqlite-memory/src/legacy-migration.ts b/role-model-router/packages/sqlite-memory/src/legacy-migration.ts index 8a1f1f7b..0c834857 100644 --- a/role-model-router/packages/sqlite-memory/src/legacy-migration.ts +++ b/role-model-router/packages/sqlite-memory/src/legacy-migration.ts @@ -175,6 +175,23 @@ export interface SqliteMigrationRegistry { readonly entries: readonly SqliteMigrationRegistryEntry[]; } +function hasMigrationRegistryIdentity(routerRoot: string): boolean { + const registryPath = path.join(routerRoot, "migrations", "registry.json"); + if (!existsSync(registryPath)) return false; + try { + const parsed = JSON.parse(readFileSync(registryPath, "utf8")) as { + readonly schemaVersion?: unknown; + readonly entries?: unknown; + }; + return ( + parsed.schemaVersion === "role-model.sqlite-migration-registry.v1" && + Array.isArray(parsed.entries) + ); + } catch { + return false; + } +} + /** * Resolves source-checkout migration assets without relying on module URL metadata. * Packaged runtimes must pass their verified staged router root explicitly. @@ -183,19 +200,19 @@ export function resolveLegacyMigrationRouterRoot(startDirectory = process.cwd()) const start = path.resolve(startDirectory); const initialCandidates = [start, path.join(start, "role-model-router")]; for (const candidate of initialCandidates) { - if (existsSync(path.join(candidate, "migrations", "registry.json"))) return candidate; + if (hasMigrationRegistryIdentity(candidate)) return candidate; } let candidate = path.dirname(start); while (true) { - if (existsSync(path.join(candidate, "migrations", "registry.json"))) return candidate; + if (hasMigrationRegistryIdentity(candidate)) return candidate; const parent = path.dirname(candidate); if (parent === candidate) break; candidate = parent; } throw new Error( - "SQLite migration registry was not found; packaged runtimes must provide an explicit routerRoot", + "A valid SQLite migration registry identity was not found; packaged runtimes must provide an explicit routerRoot", ); } diff --git a/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts b/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts index 64fe27bb..2641e372 100644 --- a/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts +++ b/role-model-router/packages/sqlite-memory/test/run94-storage-corrections-red.test.ts @@ -48,7 +48,14 @@ describe("Run 94 public storage corrections", () => { const nestedWorkingDirectory = path.join(routerRoot, "packages", "sqlite-memory"); mkdirSync(path.join(routerRoot, "migrations"), { recursive: true }); mkdirSync(nestedWorkingDirectory, { recursive: true }); - writeFileSync(path.join(routerRoot, "migrations", "registry.json"), "{}", "utf8"); + writeFileSync( + path.join(routerRoot, "migrations", "registry.json"), + JSON.stringify({ + schemaVersion: "role-model.sqlite-migration-registry.v1", + entries: [], + }), + "utf8", + ); expect(resolveLegacyMigrationRouterRoot(nestedWorkingDirectory)).toBe(routerRoot); expect(() => resolveLegacyMigrationRouterRoot(path.join(root, "unrelated"))).toThrow( @@ -56,6 +63,18 @@ describe("Run 94 public storage corrections", () => { ); }); + test("rejects an unrelated ancestor that merely contains migrations/registry.json", () => { + const root = mkdtempSync(path.join(os.tmpdir(), "run94-unrelated-router-root-")); + const nestedWorkingDirectory = path.join(root, "unrelated", "packages", "sqlite-memory"); + mkdirSync(path.join(root, "migrations"), { recursive: true }); + mkdirSync(nestedWorkingDirectory, { recursive: true }); + writeFileSync(path.join(root, "migrations", "registry.json"), "{}", "utf8"); + + expect(() => resolveLegacyMigrationRouterRoot(nestedWorkingDirectory)).toThrow( + /explicit routerRoot|valid migration registry|identity/i, + ); + }); + test("RED: enforces the 16 KiB cap by UTF-8 bytes on UPDATE, not SQLite characters", () => { const { databasePath } = newDatabase(); insertObservation(databasePath, "req-utf8", JSON.stringify({ requestId: "req-utf8" })); From 7d89285cdb2ae65acd1e44603e666256bf47af66 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:26:19 +0800 Subject: [PATCH 23/40] Normalize packaged extension health probes --- packages/extension-host/worker-runtime.mjs | 7 +- .../apps/runtime-host-bridge/src/index.ts | 12 +- .../src/track-b-operations.ts | 84 +++++++---- .../test/track-b-operations-api.test.ts | 140 ++++++++++++++---- 4 files changed, 177 insertions(+), 66 deletions(-) diff --git a/packages/extension-host/worker-runtime.mjs b/packages/extension-host/worker-runtime.mjs index 828ee7f1..1b4674e2 100644 --- a/packages/extension-host/worker-runtime.mjs +++ b/packages/extension-host/worker-runtime.mjs @@ -38,18 +38,19 @@ const MAX_INLINE_OUTPUT_BYTES = 16 * 1024; const MAX_DURABLE_OUTPUT_ROWS = 512; function persistBusinessOutput(envelope, result) { + const capability = envelope.capability ?? "health:probe"; const resultJson = JSON.stringify(result ?? null); const byteLength = Buffer.byteLength(resultJson, "utf8"); const resultHash = `sha256:${createHash("sha256").update(resultJson).digest("hex")}`; const outputKey = `sha256:${createHash("sha256") - .update(`${extensionId}\0${envelope.requestId}\0${envelope.capability}\0${resultHash}`) + .update(`${extensionId}\0${envelope.requestId}\0${capability}\0${resultHash}`) .digest("hex")}`; outputDatabase .prepare("INSERT OR IGNORE INTO durable_extension_outputs VALUES (?,?,?,?,?,?,?,?,?)") .run( outputKey, envelope.requestId, - envelope.capability, + capability, envelope.channel, envelope.scope, resultHash, @@ -66,7 +67,7 @@ function persistBusinessOutput(envelope, result) { const durableLocator = Object.freeze({ extensionId, requestId: envelope.requestId, - capability: envelope.capability, + capability, channel: envelope.channel, scope: envelope.scope, outputKey, diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index 67a2188c..924a1ea8 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -27710,7 +27710,9 @@ export async function createRuntimeBridgeBackend( }, async recoverLegacyTerminalFailure(body: Record): Promise { if (runtimeChannel === "production") - throw new Error("legacy terminal failure recovery is restricted to development and stage channels"); + throw new Error( + "legacy terminal failure recovery is restricted to development and stage channels", + ); const allowedKeys = new Set(["requestId", "acknowledgeMetadataOnly"]); const unexpectedKeys = Object.keys(body).filter((key) => !allowedKeys.has(key)); if (unexpectedKeys.length > 0) @@ -27736,7 +27738,9 @@ export async function createRuntimeBridgeBackend( existingRecovery.kind !== "legacy_terminal_failure" || existingRecovery.source !== "persisted_runtime_observation" ) - throw new Error("legacy terminal failure recovery refused because graph evidence already exists"); + throw new Error( + "legacy terminal failure recovery refused because graph evidence already exists", + ); const graphEvidence = buildGraphEvidenceFromCapture(existingCapture); const recoveredObservation = Object.freeze({ ...(observation as unknown as Record), @@ -27777,7 +27781,9 @@ export async function createRuntimeBridgeBackend( const captureInput = buildLegacyTerminalFailureRecoveryCapture( observation as unknown as Readonly>, ); - await runtimeTrackBOperations.recordLocalRouteCapture(captureInput as Record); + await runtimeTrackBOperations.recordLocalRouteCapture( + captureInput as Record, + ); const capture = await readExactRouteCapture(requestId); if (!capture) throw new Error("metadata-only recovery did not commit durable graph evidence"); const graphEvidence = buildGraphEvidenceFromCapture(capture); diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index 44fe7878..f025985c 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -655,7 +655,8 @@ function recordValue(value: unknown): Readonly> { function finiteNonNegative(value: unknown, label: string): number { const number = Number(value); - if (!Number.isFinite(number) || number < 0) throw new Error(`${label} must be finite and non-negative`); + if (!Number.isFinite(number) || number < 0) + throw new Error(`${label} must be finite and non-negative`); return number; } @@ -733,22 +734,47 @@ export function buildGraphEvidenceFromCapture( const tools = Array.isArray(capture.tools) ? capture.tools : []; const toolRows = tools.map((tool) => recordValue(tool)); const rawCaptureMetrics = recordValue(capture.captureMetrics); - const captureMetrics = Object.keys(rawCaptureMetrics).length > 0 - ? Object.freeze({ - captureCpuMs: finiteNonNegative(rawCaptureMetrics.captureCpuMs, "capture CPU"), - captureWallMs: finiteNonNegative(rawCaptureMetrics.captureWallMs, "capture wall time"), - sqliteLockWaitMs: finiteNonNegative(rawCaptureMetrics.sqliteLockWaitMs, "SQLite lock wait"), - queueDepthBefore: finiteNonNegative(rawCaptureMetrics.queueDepthBefore, "capture queue depth before"), - queueDepthAfter: finiteNonNegative(rawCaptureMetrics.queueDepthAfter, "capture queue depth"), - filesystemBytesBefore: finiteNonNegative(rawCaptureMetrics.filesystemBytesBefore, "filesystem bytes before"), - filesystemBytesAfter: finiteNonNegative(rawCaptureMetrics.filesystemBytesAfter, "filesystem bytes after"), - casBytesBefore: finiteNonNegative(rawCaptureMetrics.casBytesBefore, "CAS bytes before"), - casBytesAfter: finiteNonNegative(rawCaptureMetrics.casBytesAfter, "CAS bytes after"), - normalizedStateBytesBefore: finiteNonNegative(rawCaptureMetrics.normalizedStateBytesBefore, "normalized state bytes before"), - normalizedStateBytesAfter: finiteNonNegative(rawCaptureMetrics.normalizedStateBytesAfter, "normalized state bytes after"), - archiveManifestInlineContentBytes: finiteNonNegative(rawCaptureMetrics.archiveManifestInlineContentBytes, "archive manifest inline content bytes"), - }) - : null; + const captureMetrics = + Object.keys(rawCaptureMetrics).length > 0 + ? Object.freeze({ + captureCpuMs: finiteNonNegative(rawCaptureMetrics.captureCpuMs, "capture CPU"), + captureWallMs: finiteNonNegative(rawCaptureMetrics.captureWallMs, "capture wall time"), + sqliteLockWaitMs: finiteNonNegative( + rawCaptureMetrics.sqliteLockWaitMs, + "SQLite lock wait", + ), + queueDepthBefore: finiteNonNegative( + rawCaptureMetrics.queueDepthBefore, + "capture queue depth before", + ), + queueDepthAfter: finiteNonNegative( + rawCaptureMetrics.queueDepthAfter, + "capture queue depth", + ), + filesystemBytesBefore: finiteNonNegative( + rawCaptureMetrics.filesystemBytesBefore, + "filesystem bytes before", + ), + filesystemBytesAfter: finiteNonNegative( + rawCaptureMetrics.filesystemBytesAfter, + "filesystem bytes after", + ), + casBytesBefore: finiteNonNegative(rawCaptureMetrics.casBytesBefore, "CAS bytes before"), + casBytesAfter: finiteNonNegative(rawCaptureMetrics.casBytesAfter, "CAS bytes after"), + normalizedStateBytesBefore: finiteNonNegative( + rawCaptureMetrics.normalizedStateBytesBefore, + "normalized state bytes before", + ), + normalizedStateBytesAfter: finiteNonNegative( + rawCaptureMetrics.normalizedStateBytesAfter, + "normalized state bytes after", + ), + archiveManifestInlineContentBytes: finiteNonNegative( + rawCaptureMetrics.archiveManifestInlineContentBytes, + "archive manifest inline content bytes", + ), + }) + : null; const response = recordValue(capture.response); const edgeCount = Number(capture.edgeCount); const metadataOnly = capture.projectionCompleteness === "metadata_only"; @@ -772,9 +798,7 @@ export function buildGraphEvidenceFromCapture( responseNodeId: boundedIdentity(response.nodeId, "graph response node id"), toolExecutionNodeIds: toolRows .filter((tool) => tool.kind === undefined || tool.kind === "tool_execution") - .map((tool, index) => - boundedIdentity(tool.nodeId, `graph tool execution node ${index + 1}`), - ), + .map((tool, index) => boundedIdentity(tool.nodeId, `graph tool execution node ${index + 1}`)), toolCallNodeIds: toolRows .filter((tool) => tool.kind === "tool_call") .map((tool, index) => boundedIdentity(tool.nodeId, `graph tool call node ${index + 1}`)), @@ -859,9 +883,7 @@ export function buildVerifiersLiveExport(input: { if (taskPromptContent === undefined) throw new Error("Verifiers TraceTask prompt content is required"); const taskPrompt = - typeof taskPromptContent === "string" - ? taskPromptContent - : JSON.stringify(taskPromptContent); + typeof taskPromptContent === "string" ? taskPromptContent : JSON.stringify(taskPromptContent); const semanticMessages = [...messages, response].map((value, index) => { const message = recordValue(value); const role = boundedIdentity(message.role, `Verifiers node ${index + 1} role`); @@ -872,7 +894,9 @@ export function buildVerifiersLiveExport(input: { const nestedFunction = recordValue(toolCall.function); const argumentsValue = toolCall.arguments ?? nestedFunction.arguments; if (argumentsValue === undefined) - throw new Error(`Verifiers node ${index + 1} tool call ${toolIndex + 1} arguments are required`); + throw new Error( + `Verifiers node ${index + 1} tool call ${toolIndex + 1} arguments are required`, + ); return { id: boundedIdentity( toolCall.id, @@ -883,9 +907,7 @@ export function buildVerifiersLiveExport(input: { `Verifiers node ${index + 1} tool call ${toolIndex + 1} name`, ), arguments: - typeof argumentsValue === "string" - ? argumentsValue - : JSON.stringify(argumentsValue), + typeof argumentsValue === "string" ? argumentsValue : JSON.stringify(argumentsValue), }; }); const toolCallId = typeof message.toolCallId === "string" ? message.toolCallId : null; @@ -903,9 +925,7 @@ export function buildVerifiersLiveExport(input: { mask: [], is_content: [], logprobs: [], - ...(index === messages.length - ? { finish_reason: providerFailure ? "error" : "stop" } - : {}), + ...(index === messages.length ? { finish_reason: providerFailure ? "error" : "stop" } : {}), }; }); const routingDecisionId = boundedIdentity( @@ -1739,7 +1759,9 @@ export function createTrackBOperations({ }, async measureNoRichCaptureBaseline(input: Record): Promise { if (!operationsEndpoint) - throw new Error("private operations endpoint is required for no-rich capture baseline measurement"); + throw new Error( + "private operations endpoint is required for no-rich capture baseline measurement", + ); const url = new URL(operationsEndpoint); if (!["127.0.0.1", "localhost", "::1", "[::1]"].includes(url.hostname)) throw new Error("no-rich capture baseline requires a loopback operations boundary"); diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 4aee2b37..4e8e5d90 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -81,15 +81,37 @@ describe("Track B operations APIs", () => { nodeId: "node-assistant-tool-94", role: "assistant", content: null, - toolCalls: [{ id: "call-pi-94", type: "function", function: { name: "bash", arguments: '{"command":"printf run94-tool-ok"}' } }], + toolCalls: [ + { + id: "call-pi-94", + type: "function", + function: { name: "bash", arguments: '{"command":"printf run94-tool-ok"}' }, + }, + ], + }, + { + nodeId: "node-tool-message-94", + role: "tool", + content: "run94-tool-ok", + toolCallId: "call-pi-94", + name: "bash", }, - { nodeId: "node-tool-message-94", role: "tool", content: "run94-tool-ok", toolCallId: "call-pi-94", name: "bash" }, ], response: { nodeId: "node-response-94", role: "assistant", content: "routed" }, tools: [ { nodeId: "node-tool-execution-94", kind: "tool_execution", toolName: "router-tool" }, - { nodeId: "node-tool-call-94", kind: "tool_call", toolCallId: "call-pi-94", toolName: "bash" }, - { nodeId: "node-tool-result-94", kind: "tool_result", toolCallId: "call-pi-94", toolName: "bash" }, + { + nodeId: "node-tool-call-94", + kind: "tool_call", + toolCallId: "call-pi-94", + toolName: "bash", + }, + { + nodeId: "node-tool-result-94", + kind: "tool_result", + toolCallId: "call-pi-94", + toolName: "bash", + }, ], captureMetrics: { captureCpuMs: 4, @@ -109,7 +131,12 @@ describe("Track B operations APIs", () => { }; expect(buildGraphEvidenceFromCapture(capture)).toEqual({ rootArtifactId: "root-export-94", - messageNodeIds: ["node-system-94", "node-user-94", "node-assistant-tool-94", "node-tool-message-94"], + messageNodeIds: [ + "node-system-94", + "node-user-94", + "node-assistant-tool-94", + "node-tool-message-94", + ], responseNodeId: "node-response-94", toolExecutionNodeIds: ["node-tool-execution-94"], toolCallNodeIds: ["node-tool-call-94"], @@ -148,8 +175,27 @@ describe("Track B operations APIs", () => { nodes: [ { parent: null, message: { role: "system", content: "route safely" }, sampled: false }, { parent: 0, message: { role: "user", content: "route this" }, sampled: false }, - { parent: 1, message: { role: "assistant", content: null, tool_calls: [{ id: "call-pi-94", name: "bash", arguments: '{"command":"printf run94-tool-ok"}' }] }, sampled: false }, - { parent: 2, message: { role: "tool", content: "run94-tool-ok", tool_call_id: "call-pi-94", name: "bash" }, sampled: false }, + { + parent: 1, + message: { + role: "assistant", + content: null, + tool_calls: [ + { id: "call-pi-94", name: "bash", arguments: '{"command":"printf run94-tool-ok"}' }, + ], + }, + sampled: false, + }, + { + parent: 2, + message: { + role: "tool", + content: "run94-tool-ok", + tool_call_id: "call-pi-94", + name: "bash", + }, + sampled: false, + }, { parent: 3, message: { role: "assistant", content: "routed" }, sampled: true }, ], info: { @@ -228,7 +274,9 @@ describe("Track B operations APIs", () => { ]), is_completed: true, stop_condition: "provider_error", - errors: [{ type: "provider_unavailable", message: "provider unavailable", traceback: null }], + errors: [ + { type: "provider_unavailable", message: "provider unavailable", traceback: null }, + ], info: expect.objectContaining({ providerStatusCode: 503, limitations: expect.arrayContaining(["provider failed before a sampled completion"]), @@ -406,9 +454,23 @@ describe("Track B operations APIs", () => { const operations = createServer(async (request, response) => { let body = ""; for await (const chunk of request) body += chunk; - received.push({ path: request.url ?? "", authorization: request.headers.authorization, body: JSON.parse(body) }); + received.push({ + path: request.url ?? "", + authorization: request.headers.authorization, + body: JSON.parse(body), + }); response.writeHead(200, { "content-type": "application/json" }); - response.end(JSON.stringify({ schemaVersion: "role-model.no-rich-capture-baseline-channel.v1", sourceMode: "measured_capture_disabled_packaged_runtime", channel: "development", sampleCount: 5, captureCpuP95Ms: 12, providerPathLatencyP95Ms: 18, sqliteLockWaitP95Ms: 2 })); + response.end( + JSON.stringify({ + schemaVersion: "role-model.no-rich-capture-baseline-channel.v1", + sourceMode: "measured_capture_disabled_packaged_runtime", + channel: "development", + sampleCount: 5, + captureCpuP95Ms: 12, + providerPathLatencyP95Ms: 18, + sqliteLockWaitP95Ms: 2, + }), + ); }); await new Promise((resolve, reject) => { operations.once("error", reject); @@ -416,7 +478,8 @@ describe("Track B operations APIs", () => { }); try { const address = operations.address(); - if (!address || typeof address === "string") throw new Error("operations server did not bind"); + if (!address || typeof address === "string") + throw new Error("operations server did not bind"); const api = createTrackBOperations({ statePath: path.join(os.tmpdir(), `run94-no-rich-baseline-${Date.now()}.json`), catalog: [], @@ -428,9 +491,17 @@ describe("Track B operations APIs", () => { channel: "development", sampleCount: 5, }); - expect(received).toEqual([{ path: "/capture/performance-baseline", authorization: "Bearer run94-baseline-token-0001", body: { sampleCount: 5 } }]); + expect(received).toEqual([ + { + path: "/capture/performance-baseline", + authorization: "Bearer run94-baseline-token-0001", + body: { sampleCount: 5 }, + }, + ]); } finally { - await new Promise((resolve, reject) => operations.close((error) => error ? reject(error) : resolve())); + await new Promise((resolve, reject) => + operations.close((error) => (error ? reject(error) : resolve())), + ); } }); @@ -1026,11 +1097,20 @@ describe("Track B operations APIs", () => { }, edgeCount: 2, }); - expect((detail as unknown as { liveBudgetEvidence?: { compactObservationBytes?: number; runtimeRssBytes?: number } }).liveBudgetEvidence).toMatchObject({ + expect( + ( + detail as unknown as { + liveBudgetEvidence?: { compactObservationBytes?: number; runtimeRssBytes?: number }; + } + ).liveBudgetEvidence, + ).toMatchObject({ compactObservationBytes: expect.any(Number), runtimeRssBytes: expect.any(Number), }); - expect((detail as unknown as { liveBudgetEvidence: { compactObservationBytes: number } }).liveBudgetEvidence.compactObservationBytes).toBeLessThanOrEqual(16 * 1024); + expect( + (detail as unknown as { liveBudgetEvidence: { compactObservationBytes: number } }) + .liveBudgetEvidence.compactObservationBytes, + ).toBeLessThanOrEqual(16 * 1024); const telemetryDatabase = new DatabaseSync(databasePath); telemetryDatabase .prepare("DELETE FROM runtime_telemetry_records WHERE request_id=?") @@ -1123,20 +1203,22 @@ describe("Track B operations APIs", () => { } const failure = captureInput?.failure as Record; response.writeHead(200, { "content-type": "application/json" }); - response.end(JSON.stringify({ - schemaVersion: "role-model.route-capture-read.v1", - requestId: captureInput?.requestId, - routingDecisionId: captureInput?.routingDecisionId, - rootArtifactId: "root-recovered-94", - projectionCompleteness: "metadata_only", - recovery: captureInput?.recovery, - messages: [], - response: { nodeId: "response-recovered-94", role: "assistant", content: null, failure }, - terminalState: "provider_error", - failure, - tools: [], - edgeCount: 1, - })); + response.end( + JSON.stringify({ + schemaVersion: "role-model.route-capture-read.v1", + requestId: captureInput?.requestId, + routingDecisionId: captureInput?.routingDecisionId, + rootArtifactId: "root-recovered-94", + projectionCompleteness: "metadata_only", + recovery: captureInput?.recovery, + messages: [], + response: { nodeId: "response-recovered-94", role: "assistant", content: null, failure }, + terminalState: "provider_error", + failure, + tools: [], + edgeCount: 1, + }), + ); }); await new Promise((resolve, reject) => { operations.once("error", reject); From 3a72480ac8758ec42b44082f2c59ff60c78f0f59 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:34:19 +0800 Subject: [PATCH 24/40] Format provider failure regression --- .../apps/runtime-host-bridge/test/index.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/test/index.test.ts b/role-model-router/apps/runtime-host-bridge/test/index.test.ts index dbbd29b8..630681d9 100644 --- a/role-model-router/apps/runtime-host-bridge/test/index.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/index.test.ts @@ -15867,14 +15867,14 @@ describe("runtime-host-bridge", () => { createRuntimeBridgeBackend: (options: { repoRoot: string; fixtureRoot: string; - runtimeStateRoot: string; - scopeId: string; - networkFetcher?: typeof fetch; - trackBOperationsEndpoint?: string; - trackBOperationsToken?: string; - trackBPostObservation?: ( - observation: Readonly>, - ) => Promise; + runtimeStateRoot: string; + scopeId: string; + networkFetcher?: typeof fetch; + trackBOperationsEndpoint?: string; + trackBOperationsToken?: string; + trackBPostObservation?: ( + observation: Readonly>, + ) => Promise; }) => Promise<{ upsertProviderAccount: (body: Record) => Promise; activateEndpoint: (body: Record) => Promise<{ endpointId: string }>; From 9f7db7a59148bc4251da4a3ab9c8aeac025d3f2a Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:47:22 +0800 Subject: [PATCH 25/40] fix(runtime): separate recommendation lookup scope --- role-model-router/apps/runtime-host-bridge/src/cli.ts | 10 ++++++++++ .../apps/runtime-host-bridge/src/index.ts | 4 ++-- .../test/track-b-operations-api.test.ts | 6 +++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/cli.ts b/role-model-router/apps/runtime-host-bridge/src/cli.ts index f5669a0f..c267ce74 100644 --- a/role-model-router/apps/runtime-host-bridge/src/cli.ts +++ b/role-model-router/apps/runtime-host-bridge/src/cli.ts @@ -703,6 +703,7 @@ export function applyRecommendationServiceLauncherConfig(values: LauncherConfigV const serviceToken = readLauncherString(values, "recommendation-service-token"); const materialFile = readLauncherString(values, "recommendation-material-file"); const aggregateScope = readLauncherString(values, "aggregate-scope"); + const recommendationScope = readLauncherString(values, "recommendation-scope"); if (serviceUrl) { process.env.ROLE_MODEL_RECOMMENDATION_SERVICE_URL = serviceUrl; @@ -722,6 +723,12 @@ export function applyRecommendationServiceLauncherConfig(values: LauncherConfigV } process.env.ROLE_MODEL_AGGREGATE_SCOPE = aggregateScope; } + if (recommendationScope) { + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,511}$/.test(recommendationScope)) { + throw new Error("recommendation scope is invalid"); + } + process.env.ROLE_MODEL_RECOMMENDATION_SCOPE = recommendationScope; + } if (!materialFile) { return; } @@ -795,6 +802,9 @@ export async function main(): Promise { "aggregate-scope": { type: "string", }, + "recommendation-scope": { + type: "string", + }, "recommendation-service-url": { type: "string", }, diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index 924a1ea8..dd6708a4 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -25609,9 +25609,9 @@ export async function createRuntimeBridgeBackend( const baseUrl = serviceUrl.endsWith("/") ? serviceUrl : `${serviceUrl}/`; const channel = process.env.ROLE_MODEL_RECOMMENDATION_CHANNEL ?? "production"; const recommendationScopeId = - process.env.ROLE_MODEL_AGGREGATE_SCOPE?.trim() || options.scopeId; + process.env.ROLE_MODEL_RECOMMENDATION_SCOPE?.trim() || options.scopeId; if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,511}$/.test(recommendationScopeId)) - throw new Error("recommendation contribution scope is invalid"); + throw new Error("recommendation scope is invalid"); const statePath = path.join( options.runtimeStateRoot, options.scopeId, diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 4e8e5d90..05b44ed0 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -45,6 +45,7 @@ afterEach(async () => { delete process.env.ROLE_MODEL_RECOMMENDATION_VERIFICATION_KEY; delete process.env.ROLE_MODEL_RECOMMENDATION_SERVICE_TOKEN; delete process.env.ROLE_MODEL_RECOMMENDATION_CHANNEL; + delete process.env.ROLE_MODEL_RECOMMENDATION_SCOPE; delete process.env.ROLE_MODEL_AGGREGATE_SCOPE; delete process.env.ROLE_MODEL_TRACK_B_OPERATIONS_URL; delete process.env.ROLE_MODEL_TRACK_B_OPERATIONS_TOKEN; @@ -1654,6 +1655,7 @@ describe("Track B operations APIs", () => { process.env.ROLE_MODEL_RECOMMENDATION_SERVICE_TOKEN = "service-token"; process.env.ROLE_MODEL_RECOMMENDATION_CHANNEL = "development"; process.env.ROLE_MODEL_AGGREGATE_SCOPE = "tenant:run91-live-cohort"; + process.env.ROLE_MODEL_RECOMMENDATION_SCOPE = "public:deepseek-high"; vi.stubGlobal( "fetch", vi.fn(async (input, init) => { @@ -1662,7 +1664,7 @@ describe("Track B operations APIs", () => { expect(init?.method).toBe("POST"); expect(new Headers(init?.headers).get("authorization")).toBe("Bearer service-token"); expect(JSON.parse(String(init?.body))).toMatchObject({ - scopeId: "tenant:run91-live-cohort", + scopeId: "public:deepseek-high", }); return new Response( JSON.stringify({ @@ -1762,6 +1764,7 @@ describe("Track B operations APIs", () => { "recommendation-material-file": materialPath, "recommendation-channel": "development", "aggregate-scope": "tenant:run91-live-cohort", + "recommendation-scope": "public:deepseek-high", }); expect(process.env.ROLE_MODEL_RECOMMENDATION_SERVICE_URL).toBe( @@ -1769,6 +1772,7 @@ describe("Track B operations APIs", () => { ); expect(process.env.ROLE_MODEL_RECOMMENDATION_CHANNEL).toBe("development"); expect(process.env.ROLE_MODEL_AGGREGATE_SCOPE).toBe("tenant:run91-live-cohort"); + expect(process.env.ROLE_MODEL_RECOMMENDATION_SCOPE).toBe("public:deepseek-high"); expect(process.env.ROLE_MODEL_RECOMMENDATION_VERIFICATION_KEY).toBe("public-spki-fixture"); expect(process.env.ROLE_MODEL_RECOMMENDATION_SERVICE_TOKEN).toBe("service-token-fixture"); }); From f9a0e0ba93317b7a180ab05fe743a4816bba2a46 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 14:56:10 +0800 Subject: [PATCH 26/40] fix(runtime): reject stale disclosure authorization --- .../src/track-b-operations.ts | 2 + .../test/track-b-operations-api.test.ts | 48 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index f025985c..7f3a8cf3 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -1705,6 +1705,8 @@ export function createTrackBOperations({ const action = String(input.action ?? ""); if (current.managed && action !== "complete_disclosure") throw new Error("contribution is controlled by managed policy"); + if (action === "complete_disclosure" && current.authorizationState !== "pending_disclosure") + throw new Error("contribution disclosure is not pending; re-enable before authorizing"); let next: ContributionState; if (action === "opt_out") next = { diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 05b44ed0..926d4f31 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -2279,6 +2279,54 @@ describe("Track B operations APIs", () => { expect(applied.activePack.id).toBe("pack-optout"); }); + test("run94 contribution opt-out rejects stale disclosure while preserving recommendation access", async () => { + const runtimeStateRoot = path.join(os.tmpdir(), `track-b-run94-optout-${Date.now()}`); + roots.push(runtimeStateRoot); + const statePath = path.join(runtimeStateRoot, "track-b-production-bridge.json"); + await mkdir(runtimeStateRoot, { recursive: true }); + await writeFile( + statePath, + JSON.stringify({ + schemaVersion: "role-model.track-b-production-bridge.v1", + protocolVersion: "1.0", + revision: 1, + generatedAt: new Date().toISOString(), + extensions: [], + storageServices: [], + retention: { managedPolicy: false, receipts: [], activeJob: null }, + contribution: { + mode: "contributor", + contributionTier: "advanced", + recommendationTier: "advanced", + recommendationAccess: "preview_and_apply", + allowCloudUpload: true, + authorizationState: "active", + revocationEpoch: 4, + queuedCount: 0, + managed: false, + disclosureId: "disc-run94-old", + }, + recommendations: [], + recommendationRevision: 0, + activePack: null, + }), + ); + const ops = createTrackBOperations({ statePath, catalog: [] }); + const optedOut = (await ops.updateContributionState({ action: "opt_out" })) as { + recommendationAccess: string; + authorizationState: string; + revocationEpoch: number; + }; + expect(optedOut).toMatchObject({ + recommendationAccess: "preview_and_apply", + authorizationState: "revoked", + revocationEpoch: 5, + }); + await expect( + ops.updateContributionState({ action: "complete_disclosure", disclosureId: "disc-run94-old" }), + ).rejects.toThrow(/pending|stale|revoked/i); + }); + test("run79 HTTP exposes extensions mutate and recommendations dismiss routes", async () => { const runtimeStateRoot = path.join(os.tmpdir(), `track-b-run79-http-${Date.now()}`); roots.push(runtimeStateRoot); From a3a7395dddb9fa3c52fbb8ab9ecfb87e49e9b466 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:29:18 +0800 Subject: [PATCH 27/40] feat(runtime): support ordered alias endpoint preferences --- .../apps/runtime-host-bridge/src/index.ts | 57 ++++++++++++++----- .../src/unified-runtime-config.ts | 44 ++++++++++++++ .../runtime-host-bridge/test/index.test.ts | 50 ++++++++++++++++ .../test/unified-runtime-config.test.ts | 47 +++++++++++++++ 4 files changed, 183 insertions(+), 15 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index dd6708a4..a8c302ba 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -8213,6 +8213,7 @@ function resolveRequestedModelPool( inventory: RoutableInventory | null = null, ): { readonly allowEndpoints: readonly string[]; + readonly preferredEndpointIds: readonly string[]; readonly routingDiagnostics?: Pick; } { // A selectable endpoint row is also a valid OpenAI `model` value. Resolve @@ -8224,13 +8225,14 @@ function resolveRequestedModelPool( toLegacyCredentializedEndpointId(endpoint.identity.endpoint_id) === requestedModel, ); if (exactEndpoint) { - return { allowEndpoints: [exactEndpoint.identity.endpoint_id] }; + return { allowEndpoints: [exactEndpoint.identity.endpoint_id], preferredEndpointIds: [] }; } const alias = modelAliases.find((entry) => entry.aliasId === requestedModel); if (!alias) { return { allowEndpoints: collectAllowedEndpointIds(registry, [requestedModel]), + preferredEndpointIds: [], }; } @@ -8238,6 +8240,9 @@ function resolveRequestedModelPool( const resolution = resolveAliasAllowEndpoints(alias, inventory, registry); return { allowEndpoints: resolution.allowEndpoints, + preferredEndpointIds: (alias.preferredEndpointIds ?? []).filter((endpointId) => + resolution.allowEndpoints.includes(endpointId), + ), routingDiagnostics: { aliasResolution: { requestedModel, @@ -8267,6 +8272,9 @@ function resolveRequestedModelPool( ); return { allowEndpoints, + preferredEndpointIds: (alias.preferredEndpointIds ?? []).filter((endpointId) => + allowEndpoints.includes(endpointId), + ), routingDiagnostics: { aliasResolution: { requestedModel, @@ -8279,6 +8287,17 @@ function resolveRequestedModelPool( }; } +function resolveAliasRoutingModel( + preferredEndpointIds: readonly string[], + allowEndpoints: readonly string[], +): RoutingModelSelection | undefined { + const allowed = new Set(allowEndpoints); + const preferred = preferredEndpointIds.filter((endpointId) => allowed.has(endpointId)); + return preferred.length > 0 + ? { endpointId: preferred[0], preferredEndpointIds: preferred } + : undefined; +} + function filterRequestedModelPoolByReasoningEffort(input: { readonly registry: EndpointRegistryResult; readonly requestedModel: string; @@ -8920,12 +8939,11 @@ export function mapChatCompletionsRequest( const roleModelIntent = readRoleModelIntentFromRequestBody( body as unknown as Record, ); - const { allowEndpoints: modelAllowEndpoints, routingDiagnostics } = resolveRequestedModelPool( - registry, - body.model, - modelAliases, - inventory, - ); + const { + allowEndpoints: modelAllowEndpoints, + preferredEndpointIds: aliasPreferredEndpointIds, + routingDiagnostics, + } = resolveRequestedModelPool(registry, body.model, modelAliases, inventory); const allowEndpoints = filterRequestedModelPoolByReasoningEffort({ registry, requestedModel: body.model, @@ -9041,6 +9059,11 @@ export function mapChatCompletionsRequest( rolePolicyExecution.executionRequest.messages, ); const sessionAffinity = buildBridgeExecutionSessionAffinity(requestOptions); + const aliasRoutingModel = resolveAliasRoutingModel( + aliasPreferredEndpointIds, + rolePolicyExecution.routingRequest.allowEndpoints ?? [], + ); + const effectiveRoutingModel = controllerRouting.routingModel ?? aliasRoutingModel; return { routingRequest: rolePolicyExecution.routingRequest, executionRequest: { @@ -9061,7 +9084,7 @@ export function mapChatCompletionsRequest( ...(typeof body.max_tokens === "number" ? { maxOutputTokens: body.max_tokens } : {}), ...(typeof body.temperature === "number" ? { temperature: body.temperature } : {}), }, - ...(controllerRouting.routingModel ? { routingModel: controllerRouting.routingModel } : {}), + ...(effectiveRoutingModel ? { routingModel: effectiveRoutingModel } : {}), ...(rolePolicyExecution.routingDiagnostics ? { routingDiagnostics: rolePolicyExecution.routingDiagnostics } : {}), @@ -9090,12 +9113,11 @@ export function mapResponsesRequest( const capabilityRequirements = inferResponsesCapabilityRequirements( body as unknown as Record, ); - const { allowEndpoints: modelAllowEndpoints, routingDiagnostics } = resolveRequestedModelPool( - registry, - body.model, - modelAliases, - inventory, - ); + const { + allowEndpoints: modelAllowEndpoints, + preferredEndpointIds: aliasPreferredEndpointIds, + routingDiagnostics, + } = resolveRequestedModelPool(registry, body.model, modelAliases, inventory); const allowEndpoints = filterRequestedModelPoolByReasoningEffort({ registry, requestedModel: body.model, @@ -9219,6 +9241,11 @@ export function mapResponsesRequest( ); const continuation = readResponsesContinuationRequest(body); const sessionAffinity = buildBridgeExecutionSessionAffinity(requestOptions); + const aliasRoutingModel = resolveAliasRoutingModel( + aliasPreferredEndpointIds, + rolePolicyExecution.routingRequest.allowEndpoints ?? [], + ); + const effectiveRoutingModel = controllerRouting.routingModel ?? aliasRoutingModel; return { routingRequest: rolePolicyExecution.routingRequest, @@ -9243,7 +9270,7 @@ export function mapResponsesRequest( : {}), ...(typeof body.temperature === "number" ? { temperature: body.temperature } : {}), }, - ...(controllerRouting.routingModel ? { routingModel: controllerRouting.routingModel } : {}), + ...(effectiveRoutingModel ? { routingModel: effectiveRoutingModel } : {}), ...(rolePolicyExecution.routingDiagnostics ? { routingDiagnostics: rolePolicyExecution.routingDiagnostics } : {}), diff --git a/role-model-router/apps/runtime-host-bridge/src/unified-runtime-config.ts b/role-model-router/apps/runtime-host-bridge/src/unified-runtime-config.ts index eb742bd6..1442e7a6 100644 --- a/role-model-router/apps/runtime-host-bridge/src/unified-runtime-config.ts +++ b/role-model-router/apps/runtime-host-bridge/src/unified-runtime-config.ts @@ -73,6 +73,8 @@ export interface UnifiedRuntimeModelAliasConfig { readonly modelIds: readonly string[]; /** Optional exact endpoint-instance allowlist; omitted means all expanded siblings. */ readonly endpointIds?: readonly string[]; + /** Ordered routing preference; every entry must also be in endpointIds. */ + readonly preferredEndpointIds?: readonly string[]; } export interface UnifiedRuntimeDifficultyClassifierConfig { @@ -308,6 +310,7 @@ interface RawUnifiedRuntimeConfig { readonly mode?: string; readonly model_ids?: readonly string[]; readonly endpoint_ids?: readonly string[]; + readonly preferred_endpoint_ids?: readonly string[]; } > >; @@ -842,6 +845,31 @@ function normalizeModelAliasInput( .filter(Boolean), ), ]; + const preferredEndpointIdsInput = + "preferredEndpointIds" in value + ? value.preferredEndpointIds + : "preferred_endpoint_ids" in value + ? value.preferred_endpoint_ids + : undefined; + const preferredEndpointIds = + preferredEndpointIdsInput === undefined + ? undefined + : [ + ...new Set( + readStringArray(preferredEndpointIdsInput) + .map((entry) => entry.trim()) + .filter(Boolean), + ), + ]; + if ( + preferredEndpointIds !== undefined && + (endpointIds === undefined || + preferredEndpointIds.some((endpointId) => !endpointIds.includes(endpointId))) + ) { + throw new Error( + `${prefix}.${aliasId}.preferred_endpoint_ids must be a subset of endpoint_ids.`, + ); + } return { aliasId, mode: readAliasRoutingMode( @@ -850,6 +878,7 @@ function normalizeModelAliasInput( ), modelIds, ...(endpointIds !== undefined ? { endpointIds } : {}), + ...(preferredEndpointIds !== undefined ? { preferredEndpointIds } : {}), }; } @@ -1563,11 +1592,17 @@ function mergeCanonicalAliasEntries( : alias.endpointIds === undefined ? existing.endpointIds : [...new Set([...existing.endpointIds, ...alias.endpointIds])]; + const mergedPreferredEndpointIds = [ + ...new Set([...(existing.preferredEndpointIds ?? []), ...(alias.preferredEndpointIds ?? [])]), + ]; merged.set(alias.aliasId, { aliasId: alias.aliasId, mode: existing.mode ?? alias.mode ?? null, modelIds: [...new Set([...existing.modelIds, ...alias.modelIds])], ...(mergedEndpointIds !== undefined ? { endpointIds: mergedEndpointIds } : {}), + ...(mergedPreferredEndpointIds.length > 0 + ? { preferredEndpointIds: mergedPreferredEndpointIds } + : {}), }); } return [...merged.values()]; @@ -1591,6 +1626,12 @@ function sameCanonicalAliasList( (alias.endpointIds ?? []).every( (endpointId, endpointIndex) => endpointId === (nextAlias.endpointIds ?? [])[endpointIndex], + ) && + (alias.preferredEndpointIds ?? []).length === + (nextAlias.preferredEndpointIds ?? []).length && + (alias.preferredEndpointIds ?? []).every( + (endpointId, endpointIndex) => + endpointId === (nextAlias.preferredEndpointIds ?? [])[endpointIndex], ) ); }) @@ -1885,6 +1926,9 @@ export function renderUnifiedRuntimeConfigText(config: UnifiedRuntimeConfig): st ...(alias.mode !== null ? { mode: alias.mode } : {}), model_ids: [...alias.modelIds], ...(alias.endpointIds !== undefined ? { endpoint_ids: [...alias.endpointIds] } : {}), + ...(alias.preferredEndpointIds !== undefined + ? { preferred_endpoint_ids: [...alias.preferredEndpointIds] } + : {}), }, ]), ); diff --git a/role-model-router/apps/runtime-host-bridge/test/index.test.ts b/role-model-router/apps/runtime-host-bridge/test/index.test.ts index 630681d9..970c03fe 100644 --- a/role-model-router/apps/runtime-host-bridge/test/index.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/index.test.ts @@ -1689,6 +1689,56 @@ describe("runtime-host-bridge", () => { }); }); + test("maps custom alias endpoint preference into the normal routing-model signal", () => { + const result = ( + bridge as { + mapChatCompletionsRequest: ( + value: EndpointRegistryResult, + body: Record, + requestId: string, + aliases: readonly { + aliasId: string; + mode: "basic"; + modelIds: readonly string[]; + endpointIds: readonly string[]; + preferredEndpointIds: readonly string[]; + }[], + ) => { + routingRequest: { allowEndpoints: readonly string[] }; + routingModel?: { endpointId: string; preferredEndpointIds: readonly string[] }; + }; + } + ).mapChatCompletionsRequest( + registry, + { + model: "run94.fallback", + messages: [{ role: "user", content: "exercise ordered fallback" }], + }, + "req-run94-alias-preference", + [ + { + aliasId: "run94.fallback", + mode: "basic", + modelIds: ["moonshot/kimi-k2.5"], + endpointIds: [ + "moonshot.personal.primary.global.kimi-k2.5", + "moonshot.personal.kimi-code.global.kimi-k2.5", + ], + preferredEndpointIds: ["moonshot.personal.primary.global.kimi-k2.5"], + }, + ], + ); + + expect(result.routingRequest.allowEndpoints).toEqual([ + "moonshot.personal.kimi-code.global.kimi-k2.5", + "moonshot.personal.primary.global.kimi-k2.5", + ]); + expect(result.routingModel).toEqual({ + endpointId: "moonshot.personal.primary.global.kimi-k2.5", + preferredEndpointIds: ["moonshot.personal.primary.global.kimi-k2.5"], + }); + }); + test("maps request role_model intent metadata into the routing request", () => { const result = ( bridge as { diff --git a/role-model-router/apps/runtime-host-bridge/test/unified-runtime-config.test.ts b/role-model-router/apps/runtime-host-bridge/test/unified-runtime-config.test.ts index dafc514c..8f113a07 100644 --- a/role-model-router/apps/runtime-host-bridge/test/unified-runtime-config.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/unified-runtime-config.test.ts @@ -1086,6 +1086,53 @@ observed_data: ]); }); + test("round-trips an ordered preferred endpoint subset for a custom alias", () => { + const parsed = parseUnifiedRuntimeConfigText(` +version: "1.0" +execution_mode: remote_only +model_aliases: + run94.fallback: + mode: basic + model_ids: [deepseek/model] + endpoint_ids: [deepseek.control, deepseek.valid] + preferred_endpoint_ids: [deepseek.control] +`); + + expect(parsed.modelAliases).toEqual( + expect.arrayContaining([ + { + aliasId: "run94.fallback", + mode: "basic", + modelIds: ["deepseek/model"], + endpointIds: ["deepseek.control", "deepseek.valid"], + preferredEndpointIds: ["deepseek.control"], + }, + ]), + ); + expect(parse(renderUnifiedRuntimeConfigText(parsed))).toMatchObject({ + model_aliases: { + "run94.fallback": { + endpoint_ids: ["deepseek.control", "deepseek.valid"], + preferred_endpoint_ids: ["deepseek.control"], + }, + }, + }); + }); + + test("rejects alias preferred endpoints outside the endpoint allowlist", () => { + expect(() => + parseUnifiedRuntimeConfigText(` +version: "1.0" +execution_mode: remote_only +model_aliases: + run94.fallback: + model_ids: [deepseek/model] + endpoint_ids: [deepseek.valid] + preferred_endpoint_ids: [deepseek.control] +`), + ).toThrow(/preferred.*subset|preferred.*allowlist/i); + }); + test("accepts routing_strategy alias in partial runtime config patches", () => { const merged = mergeUnifiedRuntimeConfigDocuments( { From a7ec81e6a01840e9a8efe2c2669fba3ddde972e1 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:18:21 +0800 Subject: [PATCH 28/40] fix(runtime): rehydrate known degraded aliases safely --- .../apps/runtime-host-bridge/src/index.ts | 37 ++- .../backend-unified-runtime-config.test.ts | 219 ++++++++++++++++++ 2 files changed, 250 insertions(+), 6 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index a8c302ba..b9d61d5e 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -19701,6 +19701,7 @@ export async function createRuntimeBridgeBackend( }; const applyUnifiedRuntimeConfigState = async ( nextConfig: UnifiedRuntimeConfig | null, + applyMode: "startup" | "mutation" | "rollback" = "mutation", ): Promise => { const nextNormalizedCatalog = withBuiltinLocalOpenAIProvider( applyUnifiedLiteLLMAdapterFamilyOverrides( @@ -19826,11 +19827,35 @@ export async function createRuntimeBridgeBackend( nextModelAliases.length > 0 && currentRoutableInventory.endpointIds.length > 0 ) { + const aliasesToValidate = nextModelAliases.filter( + (alias) => !isPrimaryRoutingAliasId(alias.aliasId), + ); const aliasValidation = validateAliasInventoryResolution( - nextModelAliases.filter((alias) => !isPrimaryRoutingAliasId(alias.aliasId)), + aliasesToValidate, currentRoutableInventory, ); - if (!aliasValidation.valid) { + const knownEndpointIds = new Set(runtimeEndpoints.map((endpoint) => endpoint.endpointId)); + for (const account of currentAccounts) { + for (const region of account.regionPolicy.regions) { + for (const modelId of account.allowedModels) { + knownEndpointIds.add(createEndpointId(account.providerAccountId, region, modelId)); + } + } + } + const canRetainTemporarilyUnroutableAliases = + applyMode !== "mutation" && + aliasesToValidate + .filter( + (alias) => + resolveAliasAllowEndpoints(alias, currentRoutableInventory, currentRegistry) + .poolEmpty, + ) + .every( + (alias) => + (alias.endpointIds?.length ?? 0) > 0 && + alias.endpointIds?.every((endpointId) => knownEndpointIds.has(endpointId)), + ); + if (!aliasValidation.valid && !canRetainTemporarilyUnroutableAliases) { throw new Error(aliasValidation.errors[0] ?? "Alias inventory resolution failed."); } } @@ -19853,7 +19878,7 @@ export async function createRuntimeBridgeBackend( await syncLocalPeerState(await readStoredPeers()); rebuildCurrentState(); } else { - await applyUnifiedRuntimeConfigState(currentUnifiedRuntimeConfig); + await applyUnifiedRuntimeConfigState(currentUnifiedRuntimeConfig, "startup"); } let envelope: ReturnType; @@ -25091,7 +25116,7 @@ export async function createRuntimeBridgeBackend( await writeConfigTextAtomically(unifiedRuntimeConfigPath, previousText); } if (previousConfig) { - await applyUnifiedRuntimeConfigState(previousConfig); + await applyUnifiedRuntimeConfigState(previousConfig, "rollback"); } throw error; } @@ -26286,7 +26311,7 @@ export async function createRuntimeBridgeBackend( } catch (error) { try { await writeConfigTextAtomically(options.unifiedRuntimeConfigPath, previousText); - await applyUnifiedRuntimeConfigState(previousConfig); + await applyUnifiedRuntimeConfigState(previousConfig, "rollback"); writeResolvedControllerAssignment( previousPersistedController ? { @@ -26498,7 +26523,7 @@ export async function createRuntimeBridgeBackend( } : null, ); - await applyUnifiedRuntimeConfigState(previousConfig); + await applyUnifiedRuntimeConfigState(previousConfig, "rollback"); } else { rebuildCurrentState(); } diff --git a/role-model-router/apps/runtime-host-bridge/test/backend-unified-runtime-config.test.ts b/role-model-router/apps/runtime-host-bridge/test/backend-unified-runtime-config.test.ts index 1bfbd553..fc804236 100644 --- a/role-model-router/apps/runtime-host-bridge/test/backend-unified-runtime-config.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/backend-unified-runtime-config.test.ts @@ -80,6 +80,225 @@ afterEach(async () => { }); describe("runtime-host-bridge unified runtime backend", () => { + test("rehydrates a persisted alias with a temporarily unroutable known endpoint without weakening live config validation", async () => { + const tempRoot = await mkdtemp(path.join(os.tmpdir(), "role-model-run94-degraded-alias-")); + tempRoots.push(tempRoot); + const runtimeStateRoot = path.join(tempRoot, "state"); + const scopeId = "run94-degraded-alias"; + const unifiedRuntimeConfigPath = path.join(tempRoot, "runtime-config.yaml"); + const controlAccountId = "deepseek.personal.run94-control"; + const controlEndpointId = `${controlAccountId}.global.deepseek-v4-flash`; + const { databasePath } = initializeSqliteMemory({ + runtimeStateRoot, + scopeId, + channel: "development", + }); + upsertProviderAccount({ + databasePath, + account: { + providerAccountId: controlAccountId, + providerId: "deepseek", + providerKind: "provider-openai", + orgScope: "personal", + accountScope: "workspace-default", + credentialRef: { backend: "env", ref: "RUN94_CONTROL_API_KEY" }, + authMode: "api-key-static", + regionPolicy: { mode: "prefer", regions: ["global"] }, + baseUrlOverride: "http://127.0.0.1:3468/v1", + allowedModels: ["deepseek/deepseek-v4-flash"], + modelRoleBindings: [], + deniedModels: [], + entitlementTags: ["chat"], + budgetPolicyRef: "budget.test", + quotaPolicyRef: "quota.test", + status: "active", + healthStatus: "healthy", + rotationState: "stable", + }, + }); + upsertProviderAccount({ + databasePath, + account: { + providerAccountId: "deepseek.personal.run94-normal", + providerId: "deepseek", + providerKind: "provider-openai", + orgScope: "personal", + accountScope: "workspace-default", + credentialRef: { backend: "env", ref: "RUN94_NORMAL_API_KEY" }, + authMode: "api-key-static", + regionPolicy: { mode: "prefer", regions: ["global"] }, + baseUrlOverride: "http://127.0.0.1:3469/v1", + allowedModels: ["deepseek/deepseek-v4-pro"], + modelRoleBindings: [], + deniedModels: [], + entitlementTags: ["chat"], + budgetPolicyRef: "budget.test", + quotaPolicyRef: "quota.test", + status: "active", + healthStatus: "healthy", + rotationState: "stable", + }, + }); + const seedProviderCredentialEnvironment = { + RUN94_CONTROL_API_KEY: "test-only-control-key", + RUN94_NORMAL_API_KEY: "test-only-normal-key", + }; + const providerCredentialEnvironment = { + RUN94_NORMAL_API_KEY: "test-only-normal-key", + }; + const networkFetcher = async () => + new Response( + JSON.stringify({ + data: [{ id: "deepseek/deepseek-v4-flash" }, { id: "deepseek/deepseek-v4-pro" }], + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + const seedBackend = await createRuntimeBridgeBackend({ + repoRoot, + fixtureRoot: testFixtureRoot, + runtimeStateRoot, + scopeId, + runtimeVendorStartup: "disabled", + providerCredentialEnvironment: seedProviderCredentialEnvironment, + networkFetcher, + }); + await seedBackend.activateEndpoint({ + providerAccountId: controlAccountId, + modelId: "deepseek/deepseek-v4-flash", + region: "global", + }); + await seedBackend.activateEndpoint({ + providerAccountId: "deepseek.personal.run94-normal", + modelId: "deepseek/deepseek-v4-pro", + region: "global", + }); + await seedBackend.shutdown(); + upsertProviderAccount({ + databasePath, + account: { + providerAccountId: controlAccountId, + providerId: "deepseek", + providerKind: "provider-openai", + orgScope: "personal", + accountScope: "workspace-default", + credentialRef: { backend: "env", ref: "RUN94_CONTROL_API_KEY" }, + authMode: "api-key-static", + regionPolicy: { mode: "prefer", regions: ["global"] }, + baseUrlOverride: "http://127.0.0.1:3468/v1", + allowedModels: ["deepseek/deepseek-v4-flash"], + modelRoleBindings: [], + deniedModels: [], + entitlementTags: ["chat"], + budgetPolicyRef: "budget.test", + quotaPolicyRef: "quota.test", + status: "disabled", + healthStatus: "credentials-missing", + rotationState: "stable", + }, + }); + await writeFile( + unifiedRuntimeConfigPath, + [ + 'version: "1.0"', + "execution_mode: remote_only", + "model_aliases:", + " run94.error:", + ' mode: "basic"', + " model_ids:", + ' - "deepseek/deepseek-v4-flash"', + " endpoint_ids:", + ` - "${controlEndpointId}"`, + "", + ].join("\n"), + "utf8", + ); + + const backend = await createRuntimeBridgeBackend({ + repoRoot, + fixtureRoot: testFixtureRoot, + runtimeStateRoot, + scopeId, + unifiedRuntimeConfigPath, + runtimeVendorStartup: "disabled", + providerCredentialEnvironment, + networkFetcher, + }); + try { + await expect(backend.readRuntimeConfig()).resolves.toEqual( + expect.objectContaining({ + applied: true, + config: expect.objectContaining({ + modelAliases: expect.arrayContaining([ + expect.objectContaining({ + aliasId: "run94.error", + endpointIds: [controlEndpointId], + }), + ]), + }), + }), + ); + await expect(backend.readRouterSummary()).resolves.toEqual( + expect.objectContaining({ + aliasInventory: expect.arrayContaining([ + expect.objectContaining({ + aliasId: "run94.error", + allowEndpointIds: [], + readiness: "unavailable", + activeEndpointCount: 0, + healthyEndpointCount: 0, + }), + ]), + }), + ); + await expect( + backend.updateRuntimeConfig({ + modelAliases: [ + { + aliasId: "new.invalid.alias", + mode: "basic", + modelIds: ["deepseek/deepseek-v4-flash"], + endpointIds: [controlEndpointId], + }, + ], + }), + ).rejects.toThrow("ALIAS_POOL_EMPTY"); + } finally { + await backend.shutdown(); + } + + await writeFile( + unifiedRuntimeConfigPath, + [ + 'version: "1.0"', + "execution_mode: remote_only", + "model_aliases:", + " unknown.persisted.alias:", + ' mode: "basic"', + " model_ids:", + ' - "deepseek/deepseek-v4-flash"', + " endpoint_ids:", + ' - "deepseek.personal.unknown.global.deepseek-deepseek-v4-flash"', + "", + ].join("\n"), + "utf8", + ); + await expect( + createRuntimeBridgeBackend({ + repoRoot, + fixtureRoot: testFixtureRoot, + runtimeStateRoot, + scopeId, + unifiedRuntimeConfigPath, + runtimeVendorStartup: "disabled", + providerCredentialEnvironment, + networkFetcher, + }).then(async (unexpectedBackend) => { + await unexpectedBackend.shutdown(); + return unexpectedBackend; + }), + ).rejects.toThrow("ALIAS_POOL_EMPTY"); + }); + test("preserves the exact LiteLLM provider account on config-owned endpoint readback", async () => { const tempRoot = await mkdtemp(path.join(os.tmpdir(), "role-model-litellm-endpoint-account-")); tempRoots.push(tempRoot); From de6d80c714736da31d67a6e58b55df505853706a Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:53:43 +0800 Subject: [PATCH 29/40] fix(runtime): terminate sole-endpoint retryable failures after one attempt and fall back without same-endpoint retry --- .../apps/runtime-host-bridge/src/index.ts | 25 +++++++- .../runtime-host-bridge/test/index.test.ts | 62 ++++++++++++++----- 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/index.ts b/role-model-router/apps/runtime-host-bridge/src/index.ts index b9d61d5e..e4da4220 100644 --- a/role-model-router/apps/runtime-host-bridge/src/index.ts +++ b/role-model-router/apps/runtime-host-bridge/src/index.ts @@ -4215,11 +4215,15 @@ export function shouldRetryUpstreamExecutionOnSameEndpoint(input: { readonly errorClass: string; readonly statusCode: number; readonly alreadyRetried: boolean; + readonly fallbackEligible: boolean; + readonly hasOtherEligibleEndpoint: boolean; }): boolean { return ( input.retryable && !input.alreadyRetried && - classifyExecutionFailureCategory(input.errorClass, input.statusCode) !== "rate_limit" + classifyExecutionFailureCategory(input.errorClass, input.statusCode) !== "rate_limit" && + !input.fallbackEligible && + input.hasOtherEligibleEndpoint ); } @@ -23572,11 +23576,30 @@ export async function createRuntimeBridgeBackend( error.errorClass, error.statusCode, ); + const hasOtherEligibleEndpoint = (routed.projected.routeInput.candidates ?? []).some( + (candidate) => { + const candidateEndpointId = candidate.identity.endpoint_id; + if ( + candidateEndpointId === error.endpointId || + deniedEndpointIds.includes(candidateEndpointId) + ) + return false; + const candidateEndpoint = runtimeEndpoints.find( + (endpoint) => endpoint.endpointId === candidateEndpointId, + ); + return ( + candidateEndpoint?.lifecycleState === "active" && + candidateEndpoint.healthStatus !== "provider-unavailable" + ); + }, + ); const shouldRetry = shouldRetryUpstreamExecutionOnSameEndpoint({ retryable: error.retryable, errorClass: error.errorClass, statusCode: error.statusCode, alreadyRetried: retriedEndpointIds.has(error.endpointId), + fallbackEligible: error.fallbackEligible, + hasOtherEligibleEndpoint, }); let cooldownRecord: ExecutionCircuitRecord | undefined; if (shouldRetry) { diff --git a/role-model-router/apps/runtime-host-bridge/test/index.test.ts b/role-model-router/apps/runtime-host-bridge/test/index.test.ts index 970c03fe..7b10bda0 100644 --- a/role-model-router/apps/runtime-host-bridge/test/index.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/index.test.ts @@ -15795,7 +15795,7 @@ describe("runtime-host-bridge", () => { ).resolves.toEqual( expect.objectContaining({ executionSemantics: expect.objectContaining({ - retryCount: 1, + retryCount: 0, rerouteCount: 1, cooldownDecision: "recorded", idempotencyDecision: "not_needed", @@ -15806,17 +15806,10 @@ describe("runtime-host-bridge", () => { failureClass: "upstream_timeout", retryable: true, fallbackEligible: true, - cooldownRecorded: false, - }), - expect.objectContaining({ - failedEndpointId: "moonshot.personal.a-primary.global.kimi-k2.5", - failureClass: "quota_exhausted", - retryable: false, - fallbackEligible: true, cooldownRecorded: true, cooldownFailureCount: 1, errorPreview: expect.objectContaining({ - message: expect.stringContaining("usage limit"), + message: expect.stringContaining("timed out"), }), }), ]), @@ -15827,7 +15820,7 @@ describe("runtime-host-bridge", () => { expect.arrayContaining([ expect.objectContaining({ requestId: "req-runtime-bridge-live-fallback-001", - retryCount: 1, + retryCount: 0, rerouteCount: 1, cooldownDecision: "recorded", }), @@ -15844,7 +15837,6 @@ describe("runtime-host-bridge", () => { expect(followUpResult.endpointId).toBe("moonshot.personal.z-backup.global.kimi-k2.5"); expect(followUpResult.outputText).toBe("backup endpoint handled the request"); expect(seenAuthorizations).toEqual([ - "Bearer moonshot-primary-live-key", "Bearer moonshot-primary-live-key", "Bearer moonshot-backup-live-key", "Bearer moonshot-backup-live-key", @@ -16317,7 +16309,7 @@ describe("runtime-host-bridge", () => { "req-runtime-bridge-codex-timeout-001", ), ).rejects.toThrow(/could not reach the ai service|timed out/i); - expect(executeAttempts).toBe(2); + expect(executeAttempts).toBe(1); const timeoutTelemetryRows = ((await backend.listTelemetryRequests?.()) ?? []) as Array<{ requestId?: string; endpointId?: string; @@ -16355,7 +16347,7 @@ describe("runtime-host-bridge", () => { structuredInspectionAvailable: true, }), executionSemantics: expect.objectContaining({ - retryCount: 1, + retryCount: 0, failedAttempts: expect.arrayContaining([ expect.objectContaining({ failedEndpointId: endpoint.endpointId, @@ -16398,7 +16390,7 @@ describe("runtime-host-bridge", () => { } expect(secondFailure).toBeInstanceOf(Error); expect((secondFailure as Error).message).toMatch(/could not reach the ai service|timed out/i); - expect(executeAttempts).toBe(4); + expect(executeAttempts).toBe(2); await expect( backend.executeChatCompletions( { @@ -16419,7 +16411,7 @@ describe("runtime-host-bridge", () => { }, ), ).rejects.toThrow(/could not reach the ai service|timed out/i); - expect(executeAttempts).toBe(6); + expect(executeAttempts).toBe(3); await expect(backend.listEndpoints()).resolves.toEqual( expect.arrayContaining([ expect.objectContaining({ @@ -16501,7 +16493,7 @@ describe("runtime-host-bridge", () => { }), }), ); - expect(executeAttempts).toBe(6); + expect(executeAttempts).toBe(3); const telemetryResponse = await fetch( `http://127.0.0.1:${server.port}/api/role-model/telemetry/requests`, @@ -17883,6 +17875,8 @@ describe("runtime-host-bridge", () => { errorClass: string; statusCode: number; alreadyRetried: boolean; + fallbackEligible: boolean; + hasOtherEligibleEndpoint: boolean; }) => boolean; } ).shouldRetryUpstreamExecutionOnSameEndpoint; @@ -17892,6 +17886,8 @@ describe("runtime-host-bridge", () => { errorClass: "rate_limited", statusCode: 429, alreadyRetried: false, + fallbackEligible: false, + hasOtherEligibleEndpoint: false, }), ).toBe(false); expect( @@ -17900,14 +17896,48 @@ describe("runtime-host-bridge", () => { errorClass: "upstream_timeout", statusCode: 504, alreadyRetried: false, + fallbackEligible: false, + hasOtherEligibleEndpoint: false, + }), + ).toBe(false); + expect( + shouldRetry({ + retryable: true, + errorClass: "upstream_timeout", + statusCode: 504, + alreadyRetried: false, + fallbackEligible: false, + hasOtherEligibleEndpoint: true, }), ).toBe(true); + expect( + shouldRetry({ + retryable: true, + errorClass: "upstream_error", + statusCode: 503, + alreadyRetried: false, + fallbackEligible: true, + hasOtherEligibleEndpoint: true, + }), + ).toBe(false); + expect( + shouldRetry({ + retryable: true, + errorClass: "upstream_error", + statusCode: 503, + alreadyRetried: false, + fallbackEligible: true, + hasOtherEligibleEndpoint: false, + }), + ).toBe(false); expect( shouldRetry({ retryable: true, errorClass: "upstream_timeout", statusCode: 504, alreadyRetried: true, + fallbackEligible: false, + hasOtherEligibleEndpoint: true, }), ).toBe(false); }); From 19418cbd6515fc73b5ba3e18bb70e4b0b1038ed0 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:22:38 +0800 Subject: [PATCH 30/40] style(runtime): biome-format track-b-operations-api test --- .../runtime-host-bridge/test/track-b-operations-api.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 926d4f31..380337bc 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -2323,7 +2323,10 @@ describe("Track B operations APIs", () => { revocationEpoch: 5, }); await expect( - ops.updateContributionState({ action: "complete_disclosure", disclosureId: "disc-run94-old" }), + ops.updateContributionState({ + action: "complete_disclosure", + disclosureId: "disc-run94-old", + }), ).rejects.toThrow(/pending|stale|revoked/i); }); From 3d5a1a0cc9a3ba01685de091af4334eb5b07e680 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Wed, 26 Aug 2026 05:33:37 +0800 Subject: [PATCH 31/40] fix(runtime): emit schema-valid finish_reason stop for provider-failure Trace exports --- .../apps/runtime-host-bridge/src/track-b-operations.ts | 2 +- .../runtime-host-bridge/test/track-b-operations-api.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts index 7f3a8cf3..db99d4ab 100644 --- a/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts +++ b/role-model-router/apps/runtime-host-bridge/src/track-b-operations.ts @@ -925,7 +925,7 @@ export function buildVerifiersLiveExport(input: { mask: [], is_content: [], logprobs: [], - ...(index === messages.length ? { finish_reason: providerFailure ? "error" : "stop" } : {}), + ...(index === messages.length ? { finish_reason: "stop" } : {}), }; }); const routingDecisionId = boundedIdentity( diff --git a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts index 380337bc..65d06fca 100644 --- a/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/track-b-operations-api.test.ts @@ -270,7 +270,7 @@ describe("Track B operations APIs", () => { expect.objectContaining({ message: { role: "assistant", content: null }, sampled: false, - finish_reason: "error", + finish_reason: "stop", }), ]), is_completed: true, From 4cb4a743e527495a6625ba3ed5072e04ce1bbed4 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:57:38 +0800 Subject: [PATCH 32/40] test(runtime): make release-root packaging test platform-agnostic for Linux CI --- .../test/executable.test.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/test/executable.test.ts b/role-model-router/apps/runtime-host-bridge/test/executable.test.ts index 24d82b8b..677819fd 100644 --- a/role-model-router/apps/runtime-host-bridge/test/executable.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/executable.test.ts @@ -103,33 +103,37 @@ async function collectRuntimeDependencyGraph(): Promise< describe("runtime-host-bridge executable packaging", () => { test("resolves an explicit external release root so dev and stage package trees cannot overwrite each other", () => { + const distFixture = + process.platform === "win32" ? "D:/repo/role-model-router/dist" : "/repo/role-model-router/dist"; + const explicitRootFixture = + process.platform === "win32" ? "D:/TEMP/run94-packages/development" : "/tmp/run94-packages/development"; const defaultDir = packageSea.resolveReleaseOutputDirectory({ - distRoot: "D:/repo/role-model-router/dist", + distRoot: distFixture, releaseTarget: "win32-x64", env: {}, }); expect(path.normalize(defaultDir)).toBe( - path.normalize("D:/repo/role-model-router/dist/release/win32-x64"), + path.normalize(path.join(distFixture, "release", "win32-x64")), ); const isolatedDir = packageSea.resolveReleaseOutputDirectory({ - distRoot: "D:/repo/role-model-router/dist", + distRoot: distFixture, releaseTarget: "win32-x64", - env: { ROLE_MODEL_RELEASE_OUTPUT_ROOT: "D:/TEMP/run94-packages/development" }, + env: { ROLE_MODEL_RELEASE_OUTPUT_ROOT: explicitRootFixture }, }); expect(path.normalize(isolatedDir)).toBe( - path.normalize("D:/TEMP/run94-packages/development/win32-x64"), + path.normalize(path.join(explicitRootFixture, "win32-x64")), ); expect(() => packageSea.resolveReleaseOutputDirectory({ - distRoot: "D:/repo/role-model-router/dist", + distRoot: distFixture, releaseTarget: "../stage", env: {}, }), ).toThrow(/target/i); expect(() => packageSea.resolveReleaseOutputDirectory({ - distRoot: "D:/repo/role-model-router/dist", + distRoot: distFixture, releaseTarget: "win32-x64", env: { ROLE_MODEL_RELEASE_OUTPUT_ROOT: "relative/output" }, }), From d0ea49ff09a2bd02fdbb1aa40c0ccbd02c892424 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:59:22 +0800 Subject: [PATCH 33/40] chore(ci): re-run paired lanes with updated private pairing From 4e874a4e8a16e38d5b588b8cc77701ef89110e5b Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:03:39 +0800 Subject: [PATCH 34/40] style(runtime): biome-format platform-agnostic release-root test --- .../apps/runtime-host-bridge/test/executable.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/test/executable.test.ts b/role-model-router/apps/runtime-host-bridge/test/executable.test.ts index 677819fd..c3156a4c 100644 --- a/role-model-router/apps/runtime-host-bridge/test/executable.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/executable.test.ts @@ -104,9 +104,13 @@ async function collectRuntimeDependencyGraph(): Promise< describe("runtime-host-bridge executable packaging", () => { test("resolves an explicit external release root so dev and stage package trees cannot overwrite each other", () => { const distFixture = - process.platform === "win32" ? "D:/repo/role-model-router/dist" : "/repo/role-model-router/dist"; + process.platform === "win32" + ? "D:/repo/role-model-router/dist" + : "/repo/role-model-router/dist"; const explicitRootFixture = - process.platform === "win32" ? "D:/TEMP/run94-packages/development" : "/tmp/run94-packages/development"; + process.platform === "win32" + ? "D:/TEMP/run94-packages/development" + : "/tmp/run94-packages/development"; const defaultDir = packageSea.resolveReleaseOutputDirectory({ distRoot: distFixture, releaseTarget: "win32-x64", From fb1bfc92261806dbf70bdb5b816820fbf5bb7531 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:11:55 +0800 Subject: [PATCH 35/40] chore(ci): re-run with diagnostic private pairing From 36696c0bb351204d270393879a45833b6ac7cc2d Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:30:24 +0800 Subject: [PATCH 36/40] chore(ci): ignore the nested private paired checkout in the public tree --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index cc31d1bc..bf8334f5 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ tmp-role-model-launch.log role-model-router/apps/runtime-ui/build/ role-model-router/apps/runtime-ui/test-results/ /.recursive/config/recursive-router-discovered.json +# CI paired-checkout directory (checkout of try-works/role-model-internal nested by the track-b lane) +_private/ From 53c45b903cc1af8d389693b602857c5b247c8043 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Thu, 27 Aug 2026 04:59:57 +0800 Subject: [PATCH 37/40] ci: use the explicit ci-contract packaged launch mode for the Track B browser gate --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fdca7f9..6135f87d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -210,6 +210,7 @@ jobs: done test -s "$recommendation_material" node _private/scripts/track-b/launch-packaged-runtime.mjs \ + --ci-contract \ --public-root "$GITHUB_WORKSPACE" \ --manifest "$GITHUB_WORKSPACE/_private/dist/run00-dev/track-b-runtime-manifest.json" \ --qa-extension-manifest "$GITHUB_WORKSPACE/role-model-router/apps/runtime-host-bridge/test/fixtures/recursive-87-synthetic-extension.manifest.json" \ From 0bd76758154879a141b852fa7efdbf00cfe5fdfb Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Thu, 27 Aug 2026 05:11:20 +0800 Subject: [PATCH 38/40] ci: point the Track B browser gate at the run-94 sp8 specs --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6135f87d..0539eae0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -229,4 +229,4 @@ jobs: --url http://127.0.0.1:3462/healthz \ --timeout-ms 120000 \ --poll-interval-ms 1000 - pnpm --filter @role-model-router/runtime-ui exec playwright test --grep @recursive:87-direct-track-b-semantic-completion + pnpm --filter @role-model-router/runtime-ui exec playwright test --grep @recursive:94-direct-track-b-storage-graph-cloud-roundtrip From 00136b88adf4f7fb7972dfe899cb977da720c2fe Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Thu, 27 Aug 2026 05:19:38 +0800 Subject: [PATCH 39/40] ci(run94): update workflow contract tests to the run-94 browser gate --- .../test/recursive-87-ci-contract.test.ts | 7 ++++--- scripts/ci-workflow.test.mjs | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/role-model-router/apps/runtime-host-bridge/test/recursive-87-ci-contract.test.ts b/role-model-router/apps/runtime-host-bridge/test/recursive-87-ci-contract.test.ts index 7de67f4f..45fb33c2 100644 --- a/role-model-router/apps/runtime-host-bridge/test/recursive-87-ci-contract.test.ts +++ b/role-model-router/apps/runtime-host-bridge/test/recursive-87-ci-contract.test.ts @@ -9,15 +9,16 @@ const workflow = readFileSync(path.join(root, ".github/workflows/ci.yml"), "utf8 const browserSpec = readFileSync( path.join( root, - "role-model-router/apps/runtime-ui/e2e/recursive-87-direct-track-b-semantic-completion.sp5.retention.spec.ts", + "role-model-router/apps/runtime-ui/e2e/recursive-94-direct-track-b-storage-graph-cloud-roundtrip.sp8.storage-ui.spec.ts", ), "utf8", ); test("SP6 public CI binds host integration to the same tagged live browser gate", () => { expect(workflow).toContain("recursive-87-ci-contract.test.ts"); - expect(workflow).toContain("@recursive:87-direct-track-b-semantic-completion"); + expect(workflow).toContain("@recursive:94-direct-track-b-storage-graph-cloud-roundtrip"); + expect(workflow).toContain("--ci-contract"); expect(workflow).toContain("RUNTIME_LIVE_BASE_URL"); - expect(browserSpec).toContain("@sp5"); + expect(browserSpec).toContain("@sp8"); expect(browserSpec).toContain("Physical storage inventory"); }); diff --git a/scripts/ci-workflow.test.mjs b/scripts/ci-workflow.test.mjs index 81c12f6a..5289a440 100644 --- a/scripts/ci-workflow.test.mjs +++ b/scripts/ci-workflow.test.mjs @@ -34,7 +34,8 @@ test("Track B CI is always-on, explicit, and runs the tagged browser contract", assert.match(workflow, /^ {2}track-b-runtime:/m); assert.doesNotMatch(workflow, /track-b-runtime:[\s\S]*?if:\s*\$\{\{\s*false\s*\}\}/); assert.match(workflow, /recursive-87-ci-contract\.test\.ts/); - assert.match(workflow, /@recursive:87-direct-track-b-semantic-completion/); + assert.match(workflow, /@recursive:94-direct-track-b-storage-graph-cloud-roundtrip/); + assert.match(workflow, /--ci-contract/); assert.match(workflow, /playwright install --with-deps chromium/); assert.match(workflow, /repository:\s*try-works\/role-model-internal/); assert.match(workflow, /PRIVATE_PAIRED_SHA/); @@ -66,7 +67,7 @@ test("Track B browser gate waits for semantic readiness without weakening exact const privateCheckout = trackBJob.indexOf("Checkout exact private paired repository"); const readinessGate = trackBJob.indexOf("wait-for-runtime-readiness.mjs"); const browserGate = trackBJob.indexOf( - "playwright test --grep @recursive:87-direct-track-b-semantic-completion", + "playwright test --grep @recursive:94-direct-track-b-storage-graph-cloud-roundtrip", ); assert.match(trackBJob, /if \[\[ ! "\$PRIVATE_PAIRED_SHA" =~ \^\[0-9a-f\]\{40\}\$ \]\]/); From 8064a68baf632fdc3b3d1ef2bfa077483b6b1d68 Mon Sep 17 00:00:00 2001 From: try-works <262919414+try-works@users.noreply.github.com> Date: Thu, 27 Aug 2026 05:41:41 +0800 Subject: [PATCH 40/40] test(run94): assert the exact Logical classes summary label in the storage-ui spec --- ...track-b-storage-graph-cloud-roundtrip.sp8.storage-ui.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/role-model-router/apps/runtime-ui/e2e/recursive-94-direct-track-b-storage-graph-cloud-roundtrip.sp8.storage-ui.spec.ts b/role-model-router/apps/runtime-ui/e2e/recursive-94-direct-track-b-storage-graph-cloud-roundtrip.sp8.storage-ui.spec.ts index be8364f7..88b3cd52 100644 --- a/role-model-router/apps/runtime-ui/e2e/recursive-94-direct-track-b-storage-graph-cloud-roundtrip.sp8.storage-ui.spec.ts +++ b/role-model-router/apps/runtime-ui/e2e/recursive-94-direct-track-b-storage-graph-cloud-roundtrip.sp8.storage-ui.spec.ts @@ -47,7 +47,7 @@ test.describe("@recursive:94-direct-track-b-storage-graph-cloud-roundtrip @sp8 @ await expect(page.getByRole("heading", { name: "Physical storage inventory" })).toBeVisible(); const summary = page.getByLabel("Storage retention summary"); await expect(summary.getByText("Physical", { exact: true })).toBeVisible(); - await expect(summary.getByText("Logical", { exact: true })).toBeVisible(); + await expect(summary.getByText("Logical classes", { exact: true })).toBeVisible(); await expect(summary.getByText("Reclaimable", { exact: true })).toBeVisible(); await expect(summary.getByText("Unavailable", { exact: true })).toBeVisible(); await expect(summary.getByText("Legal holds")).toBeVisible();