From 4bfe1c577a7de5f9674b1ac085b960c643695721 Mon Sep 17 00:00:00 2001 From: Benchmax Builder Date: Wed, 12 Aug 2026 16:24:57 +0500 Subject: [PATCH 1/8] Enforce daily enrichment spend cap Co-authored-by: Codex --- .env.example | 3 ++ lib/data/showcase-enrichment.ts | 83 +++++++++++++++++++++++++++++++ lib/pipeline/enrichment-budget.ts | 48 ++++++++++++++++++ scripts/phase2-preflight.mjs | 7 +++ scripts/prepare-main-deploy.mjs | 11 ++++ tests/showcase-enrichment.test.ts | 55 +++++++++++++++++++- worker/index.ts | 6 +++ wrangler.jsonc | 3 ++ 8 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 lib/pipeline/enrichment-budget.ts diff --git a/.env.example b/.env.example index 3df1646..5611f42 100644 --- a/.env.example +++ b/.env.example @@ -47,6 +47,9 @@ BENCHMAX_JUDGE_DAILY_SAMPLE_BUDGET= # pricing changes. Benchmax never hard-codes a provider price. BENCHMAX_JUDGE_INPUT_MICROUSD_PER_MILLION_TOKENS= BENCHMAX_JUDGE_OUTPUT_MICROUSD_PER_MILLION_TOKENS= +# Maximum measured E2B preview-enrichment spend per UTC day. This must be set +# before submissions are enabled; budget deferrals leave public previews pending. +BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET= BENCHMAX_SANDBOX_MICROUSD_PER_HOUR= # User-content Worker only: exact HTTPS origin allowed to frame legacy diff --git a/lib/data/showcase-enrichment.ts b/lib/data/showcase-enrichment.ts index ae4e0a2..8000119 100644 --- a/lib/data/showcase-enrichment.ts +++ b/lib/data/showcase-enrichment.ts @@ -3,6 +3,7 @@ import { and, asc, eq, inArray, isNotNull, lte, or, sql } from "drizzle-orm"; import { getDb } from "@/db"; import { artifacts, + auditEvents, showcaseEnrichmentArtifacts, showcaseEnrichmentSpendRecords, showcaseEnrichments, @@ -16,6 +17,12 @@ import { type ShowcaseEnrichmentMessage, } from "@/lib/pipeline/enrichment-messages"; import { sanitizeEnrichmentFailureCode } from "@/lib/pipeline/enrichment-policy"; +import { + configuredDailyEnrichmentBudget, + enrichmentBudgetDeferralAuditId, + enrichmentBudgetWindow, + isEnrichmentBudgetExhausted, +} from "@/lib/pipeline/enrichment-budget"; const ZIP_CONTENT_TYPES = [ "application/zip", @@ -32,6 +39,7 @@ export type ShowcaseEnrichmentArtifactKind = export type ShowcaseEnrichmentClaim = | { action: "execute"; attemptCount: number; leaseExpiresAt: Date } + | { action: "defer"; retryAt: Date } | { action: "retry"; leaseExpiresAt: Date } | { action: "skip" }; @@ -182,6 +190,8 @@ export async function claimShowcaseEnrichment( enrichmentId: string, now = new Date(), ): Promise { + const dailyBudgetMicrousd = configuredDailyEnrichmentBudget(); + const { dayStartedAt, nextDayStartedAt } = enrichmentBudgetWindow(now); const leaseExpiresAt = new Date(now.getTime() + ENRICHMENT_LEASE_MS); const [claimed] = await getDb() .update(showcaseEnrichments) @@ -203,6 +213,12 @@ export async function claimShowcaseEnrichment( lte(showcaseEnrichments.leaseExpiresAt, now), ), ), + sql`( + SELECT coalesce(sum(${showcaseEnrichmentSpendRecords.costMicrousd}), 0) + FROM ${showcaseEnrichmentSpendRecords} + WHERE ${showcaseEnrichmentSpendRecords.createdAt} >= ${dayStartedAt} + AND ${showcaseEnrichmentSpendRecords.createdAt} < ${nextDayStartedAt} + ) < ${dailyBudgetMicrousd}`, ), ) .returning({ attemptCount: showcaseEnrichments.attemptCount }); @@ -216,17 +232,84 @@ export async function claimShowcaseEnrichment( const [existing] = await getDb() .select({ leaseExpiresAt: showcaseEnrichments.leaseExpiresAt, + spentMicrousd: sql`( + SELECT coalesce(sum(${showcaseEnrichmentSpendRecords.costMicrousd}), 0) + FROM ${showcaseEnrichmentSpendRecords} + WHERE ${showcaseEnrichmentSpendRecords.createdAt} >= ${dayStartedAt} + AND ${showcaseEnrichmentSpendRecords.createdAt} < ${nextDayStartedAt} + )`, status: showcaseEnrichments.status, }) .from(showcaseEnrichments) .where(eq(showcaseEnrichments.id, enrichmentId)) .limit(1); + if ( + existing && + (existing.status === "queued" || + (existing.status === "running" && + existing.leaseExpiresAt && + existing.leaseExpiresAt <= now)) && + isEnrichmentBudgetExhausted( + Number(existing.spentMicrousd), + dailyBudgetMicrousd, + ) + ) { + if (existing.status === "running") { + await getDb() + .update(showcaseEnrichments) + .set({ status: "queued", leaseExpiresAt: null, updatedAt: now }) + .where( + and( + eq(showcaseEnrichments.id, enrichmentId), + eq(showcaseEnrichments.status, "running"), + lte(showcaseEnrichments.leaseExpiresAt, now), + ), + ); + } + await recordEnrichmentBudgetDeferral({ + budgetMicrousd: dailyBudgetMicrousd, + dayStartedAt, + enrichmentId, + nextDayStartedAt, + spentMicrousd: Number(existing.spentMicrousd), + }); + return { action: "defer", retryAt: nextDayStartedAt }; + } if (existing?.status === "running" && existing.leaseExpiresAt) { return { action: "retry", leaseExpiresAt: existing.leaseExpiresAt }; } return { action: "skip" }; } +async function recordEnrichmentBudgetDeferral(input: { + budgetMicrousd: number; + dayStartedAt: Date; + enrichmentId: string; + nextDayStartedAt: Date; + spentMicrousd: number; +}) { + await getDb() + .insert(auditEvents) + .values({ + id: enrichmentBudgetDeferralAuditId( + input.enrichmentId, + input.dayStartedAt, + ), + actorUserId: null, + entityId: input.enrichmentId, + entityType: "showcase-enrichment", + action: "showcase.preview_enrichment_budget_deferred", + metadataJson: canonicalJson({ + budgetMicrousd: input.budgetMicrousd, + dayStartedAt: input.dayStartedAt.toISOString(), + retryAt: input.nextDayStartedAt.toISOString(), + spentMicrousd: input.spentMicrousd, + }), + createdAt: new Date(), + }) + .onConflictDoNothing({ target: auditEvents.id }); +} + export async function readShowcaseEnrichmentContract(enrichmentId: string) { const [row] = await getDb() .select({ diff --git a/lib/pipeline/enrichment-budget.ts b/lib/pipeline/enrichment-budget.ts new file mode 100644 index 0000000..8a9633c --- /dev/null +++ b/lib/pipeline/enrichment-budget.ts @@ -0,0 +1,48 @@ +const MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD = 1_000_000_000; + +export function configuredDailyEnrichmentBudget( + value = process.env.BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET, +) { + const parsed = Number(value); + if ( + !Number.isSafeInteger(parsed) || + parsed < 1 || + parsed > MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD + ) { + throw new EnrichmentBudgetConfigurationError(); + } + return parsed; +} + +export function enrichmentBudgetWindow(now = new Date()) { + const dayStartedAt = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()), + ); + return { + dayStartedAt, + nextDayStartedAt: new Date(dayStartedAt.getTime() + 24 * 60 * 60 * 1_000), + }; +} + +export function enrichmentBudgetDeferralAuditId( + enrichmentId: string, + dayStartedAt: Date, +) { + return `showcase-enrichment-budget:${dayStartedAt.toISOString().slice(0, 10)}:${enrichmentId}`; +} + +export function isEnrichmentBudgetExhausted( + spentMicrousd: number, + budgetMicrousd: number, +) { + return spentMicrousd >= budgetMicrousd; +} + +export class EnrichmentBudgetConfigurationError extends Error { + constructor() { + super( + "The daily preview enrichment budget is not configured. The Test remains public with its preview pending.", + ); + this.name = "EnrichmentBudgetConfigurationError"; + } +} diff --git a/scripts/phase2-preflight.mjs b/scripts/phase2-preflight.mjs index f6d3a9b..764a6ae 100644 --- a/scripts/phase2-preflight.mjs +++ b/scripts/phase2-preflight.mjs @@ -116,11 +116,18 @@ const requiredEnvNames = [ "BENCHMAX_JUDGE_DAILY_SAMPLE_BUDGET", "BENCHMAX_JUDGE_INPUT_MICROUSD_PER_MILLION_TOKENS", "BENCHMAX_JUDGE_OUTPUT_MICROUSD_PER_MILLION_TOKENS", + "BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET", "BENCHMAX_SANDBOX_MICROUSD_PER_HOUR", "BENCHMAX_APP_ORIGIN", ]; for (const name of requiredEnvNames) assert(envNames.has(name), `missing .env.example key: ${name}`); +assert.match( + environments.staging.main.vars?.BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET ?? "", + /^[1-9][0-9]*$/, + "main Worker staging must configure BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET", +); + const forbiddenBindingPattern = /\bIMAGES\b/; for (const filePath of sourceFiles(rootDirectory)) { if (filePath === path.join(rootDirectory, "scripts", "phase2-preflight.mjs")) continue; diff --git a/scripts/prepare-main-deploy.mjs b/scripts/prepare-main-deploy.mjs index a5f5166..ac3a4cc 100644 --- a/scripts/prepare-main-deploy.mjs +++ b/scripts/prepare-main-deploy.mjs @@ -15,6 +15,17 @@ const sourceConfig = readJsonc(rootDirectory, "wrangler.jsonc"); const environment = sourceConfig.env?.[environmentName]; if (!environment) throw new Error(`Missing main Worker environment: ${environmentName}`); +const requiredEnvironmentVars = [ + "BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET", +]; +for (const key of requiredEnvironmentVars) { + if (!/^[1-9][0-9]*$/.test(environment.vars?.[key] ?? "")) { + throw new Error( + `main Worker ${environmentName} environment must set a positive integer ${key} before deployment`, + ); + } +} + // Every key the environment block must override. An absent key would previously // serialize as undefined and silently DELETE the section from the deploy config // (e.g. no triggers -> every cron sweep dead but preflight green). diff --git a/tests/showcase-enrichment.test.ts b/tests/showcase-enrichment.test.ts index c6a7860..11c06d1 100644 --- a/tests/showcase-enrichment.test.ts +++ b/tests/showcase-enrichment.test.ts @@ -12,6 +12,13 @@ import { enrichmentRetryDelaySeconds, sanitizeEnrichmentFailureCode, } from "../lib/pipeline/enrichment-policy"; +import { + configuredDailyEnrichmentBudget, + EnrichmentBudgetConfigurationError, + enrichmentBudgetDeferralAuditId, + enrichmentBudgetWindow, + isEnrichmentBudgetExhausted, +} from "../lib/pipeline/enrichment-budget"; import { usercontentWorker, type UsercontentEnv, @@ -89,7 +96,37 @@ test("generic preview checks are complete but never become a judge rubric", () = ); }); -test("the independent enrichment core has no run, judge, or budget dependency", async () => { +test("preview enrichment has an explicit UTC daily spend cap and stable audit dedupe", () => { + assert.equal(configuredDailyEnrichmentBudget("31200"), 31_200); + assert.equal(isEnrichmentBudgetExhausted(31_199, 31_200), false); + assert.equal(isEnrichmentBudgetExhausted(31_200, 31_200), true); + assert.equal(isEnrichmentBudgetExhausted(31_201, 31_200), true); + for (const value of [undefined, "", "0", "-1", "1.5", "1000000001", "nope"]) { + assert.throws( + () => configuredDailyEnrichmentBudget(value), + EnrichmentBudgetConfigurationError, + ); + } + + const beforeReset = enrichmentBudgetWindow( + new Date("2026-08-12T23:59:59.999Z"), + ); + assert.equal(beforeReset.dayStartedAt.toISOString(), "2026-08-12T00:00:00.000Z"); + assert.equal(beforeReset.nextDayStartedAt.toISOString(), "2026-08-13T00:00:00.000Z"); + assert.equal( + enrichmentBudgetDeferralAuditId(enrichmentId, beforeReset.dayStartedAt), + enrichmentBudgetDeferralAuditId( + enrichmentId, + new Date("2026-08-12T00:00:00.000Z"), + ), + ); + assert.notEqual( + enrichmentBudgetDeferralAuditId(enrichmentId, beforeReset.dayStartedAt), + enrichmentBudgetDeferralAuditId(enrichmentId, beforeReset.nextDayStartedAt), + ); +}); + +test("the enrichment core stays independent from runs and the judge budget", async () => { const [dataModule, evaluatorModule] = await Promise.all([ readFile( path.join(projectRoot, "lib", "data", "showcase-enrichment.ts"), @@ -112,6 +149,22 @@ test("the independent enrichment core has no run, judge, or budget dependency", ]) { assert.doesNotMatch(implementation, new RegExp(forbidden)); } + assert.match(dataModule, /showcase\.preview_enrichment_budget_deferred/); + assert.match(dataModule, /onConflictDoNothing\(\{ target: auditEvents\.id \}\)/); + assert.match(dataModule, /status: "queued", leaseExpiresAt: null/); +}); + +test("staging and deploy preparation require the enrichment spend cap", async () => { + const [config, envExample, preflight, prepare] = await Promise.all([ + readFile(path.join(projectRoot, "wrangler.jsonc"), "utf8"), + readFile(path.join(projectRoot, ".env.example"), "utf8"), + readFile(path.join(projectRoot, "scripts", "phase2-preflight.mjs"), "utf8"), + readFile(path.join(projectRoot, "scripts", "prepare-main-deploy.mjs"), "utf8"), + ]); + for (const source of [config, envExample, preflight, prepare]) { + assert.match(source, /BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET/); + } + assert.match(config, /"BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET": "31200"/); }); test("completed derived evidence is served only through every public safety gate", async () => { diff --git a/worker/index.ts b/worker/index.ts index b7e6534..d2077c7 100644 --- a/worker/index.ts +++ b/worker/index.ts @@ -428,6 +428,12 @@ async function consumeShowcaseEnrichmentMessage( message.ack(); return; } + if (claim.action === "defer") { + // The row intentionally stays queued. The reconciliation sweep will send + // a fresh message after the UTC spend window resets. + message.ack(); + return; + } if (claim.action === "retry") { message.retry({ delaySeconds: enrichmentRetryDelaySeconds(claim.leaseExpiresAt), diff --git a/wrangler.jsonc b/wrangler.jsonc index 392c9fe..829b4cd 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -86,6 +86,9 @@ "BENCHMAX_JUDGE_DAILY_SAMPLE_BUDGET": "30", "BENCHMAX_JUDGE_INPUT_MICROUSD_PER_MILLION_TOKENS": "950000", "BENCHMAX_JUDGE_OUTPUT_MICROUSD_PER_MILLION_TOKENS": "4000000", + // Eight launch-volume enrichments at the pinned 120-second ceiling, + // priced at the measured 117000 microUSD/hour staging rate. + "BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET": "31200", "BENCHMAX_SANDBOX_MICROUSD_PER_HOUR": "117000" }, "d1_databases": [ From 36a283ad41b4883e8af5081514d0797620c0f67e Mon Sep 17 00:00:00 2001 From: Benchmax Builder Date: Wed, 12 Aug 2026 16:39:56 +0500 Subject: [PATCH 2/8] Keep production config preparation testable Co-authored-by: Codex --- scripts/prepare-main-deploy.mjs | 9 ++++++--- tests/showcase-enrichment.test.ts | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/prepare-main-deploy.mjs b/scripts/prepare-main-deploy.mjs index ac3a4cc..f335d25 100644 --- a/scripts/prepare-main-deploy.mjs +++ b/scripts/prepare-main-deploy.mjs @@ -15,9 +15,12 @@ const sourceConfig = readJsonc(rootDirectory, "wrangler.jsonc"); const environment = sourceConfig.env?.[environmentName]; if (!environment) throw new Error(`Missing main Worker environment: ${environmentName}`); -const requiredEnvironmentVars = [ - "BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET", -]; +// Staging is the only submission-enabled environment today. Production config +// preparation remains testable with its deliberate empty vars block, while the +// runtime still fails closed if enrichment is invoked without this value. +const requiredEnvironmentVars = environmentName === "staging" + ? ["BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET"] + : []; for (const key of requiredEnvironmentVars) { if (!/^[1-9][0-9]*$/.test(environment.vars?.[key] ?? "")) { throw new Error( diff --git a/tests/showcase-enrichment.test.ts b/tests/showcase-enrichment.test.ts index 11c06d1..2402b27 100644 --- a/tests/showcase-enrichment.test.ts +++ b/tests/showcase-enrichment.test.ts @@ -165,6 +165,7 @@ test("staging and deploy preparation require the enrichment spend cap", async () assert.match(source, /BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET/); } assert.match(config, /"BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET": "31200"/); + assert.match(prepare, /environmentName === "staging"/); }); test("completed derived evidence is served only through every public safety gate", async () => { From acd235c463b2b73d401463c12e5fa10aefb8af55 Mon Sep 17 00:00:00 2001 From: Benchmax Builder Date: Wed, 12 Aug 2026 16:57:52 +0500 Subject: [PATCH 3/8] Keep deferred previews recoverable Co-authored-by: Codex --- lib/data/showcase-enrichment.ts | 21 ++++++++++++++++----- tests/showcase-enrichment.test.ts | 12 ++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/data/showcase-enrichment.ts b/lib/data/showcase-enrichment.ts index 8000119..4b04826 100644 --- a/lib/data/showcase-enrichment.ts +++ b/lib/data/showcase-enrichment.ts @@ -19,6 +19,7 @@ import { import { sanitizeEnrichmentFailureCode } from "@/lib/pipeline/enrichment-policy"; import { configuredDailyEnrichmentBudget, + EnrichmentBudgetConfigurationError, enrichmentBudgetDeferralAuditId, enrichmentBudgetWindow, isEnrichmentBudgetExhausted, @@ -190,8 +191,18 @@ export async function claimShowcaseEnrichment( enrichmentId: string, now = new Date(), ): Promise { - const dailyBudgetMicrousd = configuredDailyEnrichmentBudget(); const { dayStartedAt, nextDayStartedAt } = enrichmentBudgetWindow(now); + let dailyBudgetMicrousd: number; + try { + dailyBudgetMicrousd = configuredDailyEnrichmentBudget(); + } catch (error) { + if (error instanceof EnrichmentBudgetConfigurationError) { + // Configuration must fail closed without consuming the durable queue + // retry budget or turning an optional public preview terminally failed. + return { action: "defer", retryAt: nextDayStartedAt }; + } + throw error; + } const leaseExpiresAt = new Date(now.getTime() + ENRICHMENT_LEASE_MS); const [claimed] = await getDb() .update(showcaseEnrichments) @@ -216,8 +227,8 @@ export async function claimShowcaseEnrichment( sql`( SELECT coalesce(sum(${showcaseEnrichmentSpendRecords.costMicrousd}), 0) FROM ${showcaseEnrichmentSpendRecords} - WHERE ${showcaseEnrichmentSpendRecords.createdAt} >= ${dayStartedAt} - AND ${showcaseEnrichmentSpendRecords.createdAt} < ${nextDayStartedAt} + WHERE ${showcaseEnrichmentSpendRecords.createdAt} >= ${dayStartedAt.getTime()} + AND ${showcaseEnrichmentSpendRecords.createdAt} < ${nextDayStartedAt.getTime()} ) < ${dailyBudgetMicrousd}`, ), ) @@ -235,8 +246,8 @@ export async function claimShowcaseEnrichment( spentMicrousd: sql`( SELECT coalesce(sum(${showcaseEnrichmentSpendRecords.costMicrousd}), 0) FROM ${showcaseEnrichmentSpendRecords} - WHERE ${showcaseEnrichmentSpendRecords.createdAt} >= ${dayStartedAt} - AND ${showcaseEnrichmentSpendRecords.createdAt} < ${nextDayStartedAt} + WHERE ${showcaseEnrichmentSpendRecords.createdAt} >= ${dayStartedAt.getTime()} + AND ${showcaseEnrichmentSpendRecords.createdAt} < ${nextDayStartedAt.getTime()} )`, status: showcaseEnrichments.status, }) diff --git a/tests/showcase-enrichment.test.ts b/tests/showcase-enrichment.test.ts index 2402b27..778c1a0 100644 --- a/tests/showcase-enrichment.test.ts +++ b/tests/showcase-enrichment.test.ts @@ -152,6 +152,18 @@ test("the enrichment core stays independent from runs and the judge budget", asy assert.match(dataModule, /showcase\.preview_enrichment_budget_deferred/); assert.match(dataModule, /onConflictDoNothing\(\{ target: auditEvents\.id \}\)/); assert.match(dataModule, /status: "queued", leaseExpiresAt: null/); + assert.match( + dataModule, + /error instanceof EnrichmentBudgetConfigurationError[\s\S]*?action: "defer"/, + ); + assert.equal( + (dataModule.match(/dayStartedAt\.getTime\(\)/g) ?? []).length, + 2, + ); + assert.equal( + (dataModule.match(/nextDayStartedAt\.getTime\(\)/g) ?? []).length, + 2, + ); }); test("staging and deploy preparation require the enrichment spend cap", async () => { From fa287ecd5a34d2aa1b26ea9a9a83ca5158db3252 Mon Sep 17 00:00:00 2001 From: Benchmax Builder Date: Wed, 12 Aug 2026 17:10:28 +0500 Subject: [PATCH 4/8] Align enrichment budget deployment guards Co-authored-by: Codex --- lib/data/showcase-enrichment.ts | 32 +++++++++++++++++++++++++++++++ lib/pipeline/enrichment-budget.ts | 9 ++++++++- scripts/phase2-preflight.mjs | 11 ++++++++--- scripts/prepare-main-deploy.mjs | 17 +++++++++++----- tests/showcase-enrichment.test.ts | 21 ++++++++++++++++++++ 5 files changed, 81 insertions(+), 9 deletions(-) diff --git a/lib/data/showcase-enrichment.ts b/lib/data/showcase-enrichment.ts index 4b04826..1b8cfb0 100644 --- a/lib/data/showcase-enrichment.ts +++ b/lib/data/showcase-enrichment.ts @@ -20,6 +20,7 @@ import { sanitizeEnrichmentFailureCode } from "@/lib/pipeline/enrichment-policy" import { configuredDailyEnrichmentBudget, EnrichmentBudgetConfigurationError, + enrichmentBudgetConfigurationDeferralAuditId, enrichmentBudgetDeferralAuditId, enrichmentBudgetWindow, isEnrichmentBudgetExhausted, @@ -199,6 +200,11 @@ export async function claimShowcaseEnrichment( if (error instanceof EnrichmentBudgetConfigurationError) { // Configuration must fail closed without consuming the durable queue // retry budget or turning an optional public preview terminally failed. + await recordEnrichmentBudgetConfigurationDeferral({ + dayStartedAt, + enrichmentId, + nextDayStartedAt, + }); return { action: "defer", retryAt: nextDayStartedAt }; } throw error; @@ -292,6 +298,32 @@ export async function claimShowcaseEnrichment( return { action: "skip" }; } +async function recordEnrichmentBudgetConfigurationDeferral(input: { + dayStartedAt: Date; + enrichmentId: string; + nextDayStartedAt: Date; +}) { + await getDb() + .insert(auditEvents) + .values({ + id: enrichmentBudgetConfigurationDeferralAuditId( + input.enrichmentId, + input.dayStartedAt, + ), + actorUserId: null, + entityId: input.enrichmentId, + entityType: "showcase-enrichment", + action: "showcase.preview_enrichment_budget_configuration_deferred", + metadataJson: canonicalJson({ + dayStartedAt: input.dayStartedAt.toISOString(), + reason: "invalid-runtime-configuration", + retryAt: input.nextDayStartedAt.toISOString(), + }), + createdAt: new Date(), + }) + .onConflictDoNothing({ target: auditEvents.id }); +} + async function recordEnrichmentBudgetDeferral(input: { budgetMicrousd: number; dayStartedAt: Date; diff --git a/lib/pipeline/enrichment-budget.ts b/lib/pipeline/enrichment-budget.ts index 8a9633c..2d0e907 100644 --- a/lib/pipeline/enrichment-budget.ts +++ b/lib/pipeline/enrichment-budget.ts @@ -1,4 +1,4 @@ -const MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD = 1_000_000_000; +export const MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD = 1_000_000_000; export function configuredDailyEnrichmentBudget( value = process.env.BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET, @@ -31,6 +31,13 @@ export function enrichmentBudgetDeferralAuditId( return `showcase-enrichment-budget:${dayStartedAt.toISOString().slice(0, 10)}:${enrichmentId}`; } +export function enrichmentBudgetConfigurationDeferralAuditId( + enrichmentId: string, + dayStartedAt: Date, +) { + return `showcase-enrichment-budget-configuration:${dayStartedAt.toISOString().slice(0, 10)}:${enrichmentId}`; +} + export function isEnrichmentBudgetExhausted( spentMicrousd: number, budgetMicrousd: number, diff --git a/scripts/phase2-preflight.mjs b/scripts/phase2-preflight.mjs index 764a6ae..57f02e5 100644 --- a/scripts/phase2-preflight.mjs +++ b/scripts/phase2-preflight.mjs @@ -34,6 +34,7 @@ const environments = { }, }; const requiredCrons = ["*/2 * * * *", "0 3 * * 1"]; +const MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD = 1_000_000_000; const databaseIds = {}; const queueNamesByEnvironment = {}; @@ -122,9 +123,13 @@ const requiredEnvNames = [ ]; for (const name of requiredEnvNames) assert(envNames.has(name), `missing .env.example key: ${name}`); -assert.match( - environments.staging.main.vars?.BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET ?? "", - /^[1-9][0-9]*$/, +const stagingEnrichmentBudget = Number( + environments.staging.main.vars?.BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET, +); +assert( + Number.isSafeInteger(stagingEnrichmentBudget) && + stagingEnrichmentBudget >= 1 && + stagingEnrichmentBudget <= MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD, "main Worker staging must configure BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET", ); diff --git a/scripts/prepare-main-deploy.mjs b/scripts/prepare-main-deploy.mjs index f335d25..bbac9ba 100644 --- a/scripts/prepare-main-deploy.mjs +++ b/scripts/prepare-main-deploy.mjs @@ -14,15 +14,22 @@ const builtConfig = JSON.parse(fs.readFileSync(builtConfigPath, "utf8")); const sourceConfig = readJsonc(rootDirectory, "wrangler.jsonc"); const environment = sourceConfig.env?.[environmentName]; if (!environment) throw new Error(`Missing main Worker environment: ${environmentName}`); +const MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD = 1_000_000_000; -// Staging is the only submission-enabled environment today. Production config -// preparation remains testable with its deliberate empty vars block, while the -// runtime still fails closed if enrichment is invoked without this value. -const requiredEnvironmentVars = environmentName === "staging" +// Staging is submission-enabled today. Any future environment with public +// routes must also configure the cap before its deploy config can be prepared. +const requiresEnrichmentBudget = + environmentName === "staging" || (environment.routes?.length ?? 0) > 0; +const requiredEnvironmentVars = requiresEnrichmentBudget ? ["BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET"] : []; for (const key of requiredEnvironmentVars) { - if (!/^[1-9][0-9]*$/.test(environment.vars?.[key] ?? "")) { + const value = Number(environment.vars?.[key]); + if ( + !Number.isSafeInteger(value) || + value < 1 || + value > MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD + ) { throw new Error( `main Worker ${environmentName} environment must set a positive integer ${key} before deployment`, ); diff --git a/tests/showcase-enrichment.test.ts b/tests/showcase-enrichment.test.ts index 778c1a0..800aa3c 100644 --- a/tests/showcase-enrichment.test.ts +++ b/tests/showcase-enrichment.test.ts @@ -15,6 +15,7 @@ import { import { configuredDailyEnrichmentBudget, EnrichmentBudgetConfigurationError, + enrichmentBudgetConfigurationDeferralAuditId, enrichmentBudgetDeferralAuditId, enrichmentBudgetWindow, isEnrichmentBudgetExhausted, @@ -124,6 +125,13 @@ test("preview enrichment has an explicit UTC daily spend cap and stable audit de enrichmentBudgetDeferralAuditId(enrichmentId, beforeReset.dayStartedAt), enrichmentBudgetDeferralAuditId(enrichmentId, beforeReset.nextDayStartedAt), ); + assert.notEqual( + enrichmentBudgetConfigurationDeferralAuditId( + enrichmentId, + beforeReset.dayStartedAt, + ), + enrichmentBudgetDeferralAuditId(enrichmentId, beforeReset.dayStartedAt), + ); }); test("the enrichment core stays independent from runs and the judge budget", async () => { @@ -150,6 +158,10 @@ test("the enrichment core stays independent from runs and the judge budget", asy assert.doesNotMatch(implementation, new RegExp(forbidden)); } assert.match(dataModule, /showcase\.preview_enrichment_budget_deferred/); + assert.match( + dataModule, + /showcase\.preview_enrichment_budget_configuration_deferred/, + ); assert.match(dataModule, /onConflictDoNothing\(\{ target: auditEvents\.id \}\)/); assert.match(dataModule, /status: "queued", leaseExpiresAt: null/); assert.match( @@ -178,6 +190,15 @@ test("staging and deploy preparation require the enrichment spend cap", async () } assert.match(config, /"BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET": "31200"/); assert.match(prepare, /environmentName === "staging"/); + assert.match(prepare, /environment\.routes\?\.length/); + assert.equal( + (preflight.match(/MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD/g) ?? []).length, + 2, + ); + assert.equal( + (prepare.match(/MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD/g) ?? []).length, + 2, + ); }); test("completed derived evidence is served only through every public safety gate", async () => { From 2b83e29b34437654f4ed2b47c0a47072ee25dd21 Mon Sep 17 00:00:00 2001 From: Benchmax Builder Date: Wed, 12 Aug 2026 18:00:37 +0500 Subject: [PATCH 5/8] Reserve in-flight enrichment spend Co-authored-by: Codex --- lib/data/showcase-enrichment.ts | 39 +++++++++++++++++++++++++++---- lib/pipeline/enrichment-budget.ts | 23 ++++++++++++++++-- tests/showcase-enrichment.test.ts | 11 ++++++--- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/lib/data/showcase-enrichment.ts b/lib/data/showcase-enrichment.ts index 1b8cfb0..ee4732f 100644 --- a/lib/data/showcase-enrichment.ts +++ b/lib/data/showcase-enrichment.ts @@ -11,7 +11,10 @@ import { } from "@/db/schema"; import { canonicalJson, canonicalSha256 } from "@/lib/security/canonical"; import { sha256Hex } from "@/lib/security/policy"; -import { sandboxRateFromEnv } from "@/lib/data/result-spend"; +import { + sandboxRateFromEnv, + SpendPricingConfigurationError, +} from "@/lib/data/result-spend"; import { showcaseEnrichmentMessage, type ShowcaseEnrichmentMessage, @@ -24,6 +27,7 @@ import { enrichmentBudgetDeferralAuditId, enrichmentBudgetWindow, isEnrichmentBudgetExhausted, + projectedEnrichmentAttemptMicrousd, } from "@/lib/pipeline/enrichment-budget"; const ZIP_CONTENT_TYPES = [ @@ -194,10 +198,17 @@ export async function claimShowcaseEnrichment( ): Promise { const { dayStartedAt, nextDayStartedAt } = enrichmentBudgetWindow(now); let dailyBudgetMicrousd: number; + let projectedAttemptMicrousd: number; try { dailyBudgetMicrousd = configuredDailyEnrichmentBudget(); + projectedAttemptMicrousd = projectedEnrichmentAttemptMicrousd( + sandboxRateFromEnv(), + ); } catch (error) { - if (error instanceof EnrichmentBudgetConfigurationError) { + if ( + error instanceof EnrichmentBudgetConfigurationError || + error instanceof SpendPricingConfigurationError + ) { // Configuration must fail closed without consuming the durable queue // retry budget or turning an optional public preview terminally failed. await recordEnrichmentBudgetConfigurationDeferral({ @@ -235,7 +246,12 @@ export async function claimShowcaseEnrichment( FROM ${showcaseEnrichmentSpendRecords} WHERE ${showcaseEnrichmentSpendRecords.createdAt} >= ${dayStartedAt.getTime()} AND ${showcaseEnrichmentSpendRecords.createdAt} < ${nextDayStartedAt.getTime()} - ) < ${dailyBudgetMicrousd}`, + ) + ( + SELECT count(*) * ${projectedAttemptMicrousd} + FROM showcase_enrichments inflight_enrichment + WHERE inflight_enrichment.status = 'running' + AND inflight_enrichment.lease_expires_at > ${now.getTime()} + ) + ${projectedAttemptMicrousd} <= ${dailyBudgetMicrousd}`, ), ) .returning({ attemptCount: showcaseEnrichments.attemptCount }); @@ -249,6 +265,12 @@ export async function claimShowcaseEnrichment( const [existing] = await getDb() .select({ leaseExpiresAt: showcaseEnrichments.leaseExpiresAt, + inFlightCount: sql`( + SELECT count(*) + FROM showcase_enrichments inflight_enrichment + WHERE inflight_enrichment.status = 'running' + AND inflight_enrichment.lease_expires_at > ${now.getTime()} + )`, spentMicrousd: sql`( SELECT coalesce(sum(${showcaseEnrichmentSpendRecords.costMicrousd}), 0) FROM ${showcaseEnrichmentSpendRecords} @@ -267,7 +289,9 @@ export async function claimShowcaseEnrichment( existing.leaseExpiresAt && existing.leaseExpiresAt <= now)) && isEnrichmentBudgetExhausted( - Number(existing.spentMicrousd), + Number(existing.spentMicrousd) + + Number(existing.inFlightCount) * projectedAttemptMicrousd, + projectedAttemptMicrousd, dailyBudgetMicrousd, ) ) { @@ -288,6 +312,9 @@ export async function claimShowcaseEnrichment( dayStartedAt, enrichmentId, nextDayStartedAt, + projectedAttemptMicrousd, + reservedMicrousd: + Number(existing.inFlightCount) * projectedAttemptMicrousd, spentMicrousd: Number(existing.spentMicrousd), }); return { action: "defer", retryAt: nextDayStartedAt }; @@ -329,6 +356,8 @@ async function recordEnrichmentBudgetDeferral(input: { dayStartedAt: Date; enrichmentId: string; nextDayStartedAt: Date; + projectedAttemptMicrousd: number; + reservedMicrousd: number; spentMicrousd: number; }) { await getDb() @@ -346,6 +375,8 @@ async function recordEnrichmentBudgetDeferral(input: { budgetMicrousd: input.budgetMicrousd, dayStartedAt: input.dayStartedAt.toISOString(), retryAt: input.nextDayStartedAt.toISOString(), + projectedAttemptMicrousd: input.projectedAttemptMicrousd, + reservedMicrousd: input.reservedMicrousd, spentMicrousd: input.spentMicrousd, }), createdAt: new Date(), diff --git a/lib/pipeline/enrichment-budget.ts b/lib/pipeline/enrichment-budget.ts index 2d0e907..a6d9858 100644 --- a/lib/pipeline/enrichment-budget.ts +++ b/lib/pipeline/enrichment-budget.ts @@ -39,10 +39,29 @@ export function enrichmentBudgetConfigurationDeferralAuditId( } export function isEnrichmentBudgetExhausted( - spentMicrousd: number, + committedMicrousd: number, + projectedAttemptMicrousd: number, budgetMicrousd: number, ) { - return spentMicrousd >= budgetMicrousd; + return committedMicrousd + projectedAttemptMicrousd > budgetMicrousd; +} + +export function projectedEnrichmentAttemptMicrousd( + rateMicrousdPerHour: number, + maximumDurationMs = 120_000, +) { + if ( + !Number.isSafeInteger(rateMicrousdPerHour) || + rateMicrousdPerHour < 0 || + !Number.isSafeInteger(maximumDurationMs) || + maximumDurationMs < 1 + ) { + throw new RangeError("Projected enrichment spend inputs are invalid."); + } + const numerator = BigInt(maximumDurationMs) * BigInt(rateMicrousdPerHour); + return Number( + (numerator + BigInt(3_599_999)) / BigInt(3_600_000), + ); } export class EnrichmentBudgetConfigurationError extends Error { diff --git a/tests/showcase-enrichment.test.ts b/tests/showcase-enrichment.test.ts index 800aa3c..d948245 100644 --- a/tests/showcase-enrichment.test.ts +++ b/tests/showcase-enrichment.test.ts @@ -19,6 +19,7 @@ import { enrichmentBudgetDeferralAuditId, enrichmentBudgetWindow, isEnrichmentBudgetExhausted, + projectedEnrichmentAttemptMicrousd, } from "../lib/pipeline/enrichment-budget"; import { usercontentWorker, @@ -99,9 +100,9 @@ test("generic preview checks are complete but never become a judge rubric", () = test("preview enrichment has an explicit UTC daily spend cap and stable audit dedupe", () => { assert.equal(configuredDailyEnrichmentBudget("31200"), 31_200); - assert.equal(isEnrichmentBudgetExhausted(31_199, 31_200), false); - assert.equal(isEnrichmentBudgetExhausted(31_200, 31_200), true); - assert.equal(isEnrichmentBudgetExhausted(31_201, 31_200), true); + assert.equal(projectedEnrichmentAttemptMicrousd(117_000), 3_900); + assert.equal(isEnrichmentBudgetExhausted(27_300, 3_900, 31_200), false); + assert.equal(isEnrichmentBudgetExhausted(27_301, 3_900, 31_200), true); for (const value of [undefined, "", "0", "-1", "1.5", "1000000001", "nope"]) { assert.throws( () => configuredDailyEnrichmentBudget(value), @@ -176,6 +177,10 @@ test("the enrichment core stays independent from runs and the judge budget", asy (dataModule.match(/nextDayStartedAt\.getTime\(\)/g) ?? []).length, 2, ); + assert.match( + dataModule, + /count\(\*\) \* \$\{projectedAttemptMicrousd\}[\s\S]*?lease_expires_at > \$\{now\.getTime\(\)\}[\s\S]*?\+ \$\{projectedAttemptMicrousd\} <= \$\{dailyBudgetMicrousd\}/, + ); }); test("staging and deploy preparation require the enrichment spend cap", async () => { From e3d86cb604493035443d0c2acbec8becae5825f5 Mon Sep 17 00:00:00 2001 From: Benchmax Builder Date: Wed, 12 Aug 2026 18:35:18 +0500 Subject: [PATCH 6/8] Back off budget-deferred enrichments Co-authored-by: Codex --- lib/data/showcase-enrichment.ts | 52 ++++++++++++++++++++++-------- lib/evaluation/showcase-preview.ts | 17 ++++++++-- lib/pipeline/enrichment-budget.ts | 6 +++- tests/showcase-enrichment.test.ts | 18 +++++++++++ 4 files changed, 76 insertions(+), 17 deletions(-) diff --git a/lib/data/showcase-enrichment.ts b/lib/data/showcase-enrichment.ts index ee4732f..abae547 100644 --- a/lib/data/showcase-enrichment.ts +++ b/lib/data/showcase-enrichment.ts @@ -176,7 +176,12 @@ export async function reconcileShowcaseEnrichments(limit = 50) { const queued = await getDb() .select({ id: showcaseEnrichments.id }) .from(showcaseEnrichments) - .where(eq(showcaseEnrichments.status, "queued")) + .where( + and( + eq(showcaseEnrichments.status, "queued"), + lte(showcaseEnrichments.updatedAt, now), + ), + ) .orderBy(asc(showcaseEnrichments.updatedAt), asc(showcaseEnrichments.id)) .limit(boundedLimit); const dispatched: string[] = []; @@ -216,6 +221,11 @@ export async function claimShowcaseEnrichment( enrichmentId, nextDayStartedAt, }); + await deferShowcaseEnrichmentUntil( + enrichmentId, + now, + nextDayStartedAt, + ); return { action: "defer", retryAt: nextDayStartedAt }; } throw error; @@ -295,18 +305,11 @@ export async function claimShowcaseEnrichment( dailyBudgetMicrousd, ) ) { - if (existing.status === "running") { - await getDb() - .update(showcaseEnrichments) - .set({ status: "queued", leaseExpiresAt: null, updatedAt: now }) - .where( - and( - eq(showcaseEnrichments.id, enrichmentId), - eq(showcaseEnrichments.status, "running"), - lte(showcaseEnrichments.leaseExpiresAt, now), - ), - ); - } + await deferShowcaseEnrichmentUntil( + enrichmentId, + now, + nextDayStartedAt, + ); await recordEnrichmentBudgetDeferral({ budgetMicrousd: dailyBudgetMicrousd, dayStartedAt, @@ -325,6 +328,29 @@ export async function claimShowcaseEnrichment( return { action: "skip" }; } +async function deferShowcaseEnrichmentUntil( + enrichmentId: string, + now: Date, + retryAt: Date, +) { + await getDb() + .update(showcaseEnrichments) + .set({ status: "queued", leaseExpiresAt: null, updatedAt: retryAt }) + .where( + and( + eq(showcaseEnrichments.id, enrichmentId), + lte(showcaseEnrichments.updatedAt, now), + or( + eq(showcaseEnrichments.status, "queued"), + and( + eq(showcaseEnrichments.status, "running"), + lte(showcaseEnrichments.leaseExpiresAt, now), + ), + ), + ), + ); +} + async function recordEnrichmentBudgetConfigurationDeferral(input: { dayStartedAt: Date; enrichmentId: string; diff --git a/lib/evaluation/showcase-preview.ts b/lib/evaluation/showcase-preview.ts index fb62040..657b316 100644 --- a/lib/evaluation/showcase-preview.ts +++ b/lib/evaluation/showcase-preview.ts @@ -15,6 +15,7 @@ import { } from "@/lib/evaluation/preview-spec"; import { canonicalJson } from "@/lib/security/canonical"; import { constantTimeEqualHex, sha256Hex } from "@/lib/security/policy"; +import { SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS } from "@/lib/pipeline/enrichment-budget"; const previewReportSchema = z .object({ @@ -72,7 +73,7 @@ export async function executeShowcasePreviewEnrichment( const templateId = requiredSecret("E2B_TEMPLATE_ID"); const templateBuildHash = requiredSha256("E2B_TEMPLATE_BUILD_HASH"); const attemptKey = `sandbox:${enrichmentId}:preview:${crypto.randomUUID()}`; - const startedAt = Date.now(); + let sandboxStartedAt: number | null = null; let sandbox: Sandbox | null = null; let spendStatus: "completed" | "failed" = "failed"; try { @@ -80,13 +81,14 @@ export async function executeShowcasePreviewEnrichment( apiKey: requiredSecret("E2B_API_KEY"), allowInternetAccess: false, secure: true, - timeoutMs: EVALUATION_ENVIRONMENT_V1.wallClockSeconds * 1_000, + timeoutMs: SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, lifecycle: { onTimeout: "kill" }, metadata: { benchmaxEnrichmentId: enrichmentId, sourceSha256, }, }); + sandboxStartedAt = Date.now(); const specJson = canonicalJson(buildShowcasePreviewSpec(sourceSha256)); await sandbox.files.write( "/workspace/input/source.zip", @@ -178,7 +180,16 @@ export async function executeShowcasePreviewEnrichment( await sandbox?.kill().catch(() => false); await recordShowcaseEnrichmentSpend({ attemptKey, - durationMs: Math.max(0, Date.now() - startedAt), + // The measured rate applies to live sandbox time. E2B kills that sandbox + // at the same lifecycle ceiling reserved during the claim; provisioning + // before Sandbox.create resolves is not counted as sandbox runtime. + durationMs: + sandboxStartedAt === null + ? 0 + : Math.min( + Math.max(0, Date.now() - sandboxStartedAt), + SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, + ), enrichmentId, status: spendStatus, }).catch((error) => { diff --git a/lib/pipeline/enrichment-budget.ts b/lib/pipeline/enrichment-budget.ts index a6d9858..59aa221 100644 --- a/lib/pipeline/enrichment-budget.ts +++ b/lib/pipeline/enrichment-budget.ts @@ -1,4 +1,8 @@ +import { EVALUATION_ENVIRONMENT_V1 } from "@/lib/domain/ranked-catalog"; + export const MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD = 1_000_000_000; +export const SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS = + EVALUATION_ENVIRONMENT_V1.wallClockSeconds * 1_000; export function configuredDailyEnrichmentBudget( value = process.env.BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET, @@ -48,7 +52,7 @@ export function isEnrichmentBudgetExhausted( export function projectedEnrichmentAttemptMicrousd( rateMicrousdPerHour: number, - maximumDurationMs = 120_000, + maximumDurationMs = SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, ) { if ( !Number.isSafeInteger(rateMicrousdPerHour) || diff --git a/tests/showcase-enrichment.test.ts b/tests/showcase-enrichment.test.ts index d948245..f4a08f5 100644 --- a/tests/showcase-enrichment.test.ts +++ b/tests/showcase-enrichment.test.ts @@ -20,6 +20,7 @@ import { enrichmentBudgetWindow, isEnrichmentBudgetExhausted, projectedEnrichmentAttemptMicrousd, + SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, } from "../lib/pipeline/enrichment-budget"; import { usercontentWorker, @@ -100,6 +101,7 @@ test("generic preview checks are complete but never become a judge rubric", () = test("preview enrichment has an explicit UTC daily spend cap and stable audit dedupe", () => { assert.equal(configuredDailyEnrichmentBudget("31200"), 31_200); + assert.equal(SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, 120_000); assert.equal(projectedEnrichmentAttemptMicrousd(117_000), 3_900); assert.equal(isEnrichmentBudgetExhausted(27_300, 3_900, 31_200), false); assert.equal(isEnrichmentBudgetExhausted(27_301, 3_900, 31_200), true); @@ -165,6 +167,14 @@ test("the enrichment core stays independent from runs and the judge budget", asy ); assert.match(dataModule, /onConflictDoNothing\(\{ target: auditEvents\.id \}\)/); assert.match(dataModule, /status: "queued", leaseExpiresAt: null/); + assert.match( + dataModule, + /eq\(showcaseEnrichments\.status, "queued"\),[\s\S]*?lte\(showcaseEnrichments\.updatedAt, now\)/, + ); + assert.match( + dataModule, + /set\(\{ status: "queued", leaseExpiresAt: null, updatedAt: retryAt \}\)/, + ); assert.match( dataModule, /error instanceof EnrichmentBudgetConfigurationError[\s\S]*?action: "defer"/, @@ -181,6 +191,14 @@ test("the enrichment core stays independent from runs and the judge budget", asy dataModule, /count\(\*\) \* \$\{projectedAttemptMicrousd\}[\s\S]*?lease_expires_at > \$\{now\.getTime\(\)\}[\s\S]*?\+ \$\{projectedAttemptMicrousd\} <= \$\{dailyBudgetMicrousd\}/, ); + assert.match( + dataModule, + /eq\(showcaseEnrichments\.id, enrichmentId\),[\s\S]*?lte\(showcaseEnrichments\.updatedAt, now\),[\s\S]*?eq\(showcaseEnrichments\.status, "queued"\)/, + ); + assert.match( + evaluatorModule, + /sandboxStartedAt = Date\.now\(\)[\s\S]*?durationMs:[\s\S]*?SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS/, + ); }); test("staging and deploy preparation require the enrichment spend cap", async () => { From aaa94b7815bdafbf958e3661f8cc6e3ba4893594 Mon Sep 17 00:00:00 2001 From: Benchmax Builder Date: Wed, 12 Aug 2026 18:51:47 +0500 Subject: [PATCH 7/8] Harden enrichment deferral timing Co-authored-by: Codex --- lib/data/showcase-enrichment.ts | 14 +++++++++----- lib/evaluation/showcase-preview.ts | 20 ++++++++------------ lib/pipeline/enrichment-budget.ts | 4 +++- tests/showcase-enrichment.test.ts | 20 +++++++++++++------- wrangler.jsonc | 4 ++-- 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/lib/data/showcase-enrichment.ts b/lib/data/showcase-enrichment.ts index abae547..0eee778 100644 --- a/lib/data/showcase-enrichment.ts +++ b/lib/data/showcase-enrichment.ts @@ -35,6 +35,7 @@ const ZIP_CONTENT_TYPES = [ "application/x-zip-compressed", ] as const; const ENRICHMENT_LEASE_MS = 5 * 60 * 1000; +const ENRICHMENT_CONFIGURATION_RETRY_MS = 5 * 60 * 1000; const MICROS_PER_HOUR_DIVISOR = 3_600_000; export type ShowcaseEnrichmentArtifactKind = @@ -216,17 +217,20 @@ export async function claimShowcaseEnrichment( ) { // Configuration must fail closed without consuming the durable queue // retry budget or turning an optional public preview terminally failed. + const configurationRetryAt = new Date( + now.getTime() + ENRICHMENT_CONFIGURATION_RETRY_MS, + ); await recordEnrichmentBudgetConfigurationDeferral({ dayStartedAt, enrichmentId, - nextDayStartedAt, + retryAt: configurationRetryAt, }); await deferShowcaseEnrichmentUntil( enrichmentId, now, - nextDayStartedAt, + configurationRetryAt, ); - return { action: "defer", retryAt: nextDayStartedAt }; + return { action: "defer", retryAt: configurationRetryAt }; } throw error; } @@ -354,7 +358,7 @@ async function deferShowcaseEnrichmentUntil( async function recordEnrichmentBudgetConfigurationDeferral(input: { dayStartedAt: Date; enrichmentId: string; - nextDayStartedAt: Date; + retryAt: Date; }) { await getDb() .insert(auditEvents) @@ -370,7 +374,7 @@ async function recordEnrichmentBudgetConfigurationDeferral(input: { metadataJson: canonicalJson({ dayStartedAt: input.dayStartedAt.toISOString(), reason: "invalid-runtime-configuration", - retryAt: input.nextDayStartedAt.toISOString(), + retryAt: input.retryAt.toISOString(), }), createdAt: new Date(), }) diff --git a/lib/evaluation/showcase-preview.ts b/lib/evaluation/showcase-preview.ts index 657b316..33cf6fb 100644 --- a/lib/evaluation/showcase-preview.ts +++ b/lib/evaluation/showcase-preview.ts @@ -73,7 +73,7 @@ export async function executeShowcasePreviewEnrichment( const templateId = requiredSecret("E2B_TEMPLATE_ID"); const templateBuildHash = requiredSha256("E2B_TEMPLATE_BUILD_HASH"); const attemptKey = `sandbox:${enrichmentId}:preview:${crypto.randomUUID()}`; - let sandboxStartedAt: number | null = null; + const attemptStartedAt = Date.now(); let sandbox: Sandbox | null = null; let spendStatus: "completed" | "failed" = "failed"; try { @@ -88,7 +88,6 @@ export async function executeShowcasePreviewEnrichment( sourceSha256, }, }); - sandboxStartedAt = Date.now(); const specJson = canonicalJson(buildShowcasePreviewSpec(sourceSha256)); await sandbox.files.write( "/workspace/input/source.zip", @@ -180,16 +179,13 @@ export async function executeShowcasePreviewEnrichment( await sandbox?.kill().catch(() => false); await recordShowcaseEnrichmentSpend({ attemptKey, - // The measured rate applies to live sandbox time. E2B kills that sandbox - // at the same lifecycle ceiling reserved during the claim; provisioning - // before Sandbox.create resolves is not counted as sandbox runtime. - durationMs: - sandboxStartedAt === null - ? 0 - : Math.min( - Math.max(0, Date.now() - sandboxStartedAt), - SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, - ), + // Conservatively include provisioning and cleanup. The stored duration + // and claim reservation share the sandbox lifecycle ceiling, which has + // explicit headroom above the inner evaluator command timeout. + durationMs: Math.min( + Math.max(0, Date.now() - attemptStartedAt), + SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, + ), enrichmentId, status: spendStatus, }).catch((error) => { diff --git a/lib/pipeline/enrichment-budget.ts b/lib/pipeline/enrichment-budget.ts index 59aa221..8828a77 100644 --- a/lib/pipeline/enrichment-budget.ts +++ b/lib/pipeline/enrichment-budget.ts @@ -1,8 +1,10 @@ import { EVALUATION_ENVIRONMENT_V1 } from "@/lib/domain/ranked-catalog"; export const MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD = 1_000_000_000; +export const SHOWCASE_ENRICHMENT_SANDBOX_OVERHEAD_MS = 60_000; export const SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS = - EVALUATION_ENVIRONMENT_V1.wallClockSeconds * 1_000; + EVALUATION_ENVIRONMENT_V1.wallClockSeconds * 1_000 + + SHOWCASE_ENRICHMENT_SANDBOX_OVERHEAD_MS; export function configuredDailyEnrichmentBudget( value = process.env.BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET, diff --git a/tests/showcase-enrichment.test.ts b/tests/showcase-enrichment.test.ts index f4a08f5..0fbb663 100644 --- a/tests/showcase-enrichment.test.ts +++ b/tests/showcase-enrichment.test.ts @@ -21,6 +21,7 @@ import { isEnrichmentBudgetExhausted, projectedEnrichmentAttemptMicrousd, SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, + SHOWCASE_ENRICHMENT_SANDBOX_OVERHEAD_MS, } from "../lib/pipeline/enrichment-budget"; import { usercontentWorker, @@ -100,11 +101,12 @@ test("generic preview checks are complete but never become a judge rubric", () = }); test("preview enrichment has an explicit UTC daily spend cap and stable audit dedupe", () => { - assert.equal(configuredDailyEnrichmentBudget("31200"), 31_200); - assert.equal(SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, 120_000); - assert.equal(projectedEnrichmentAttemptMicrousd(117_000), 3_900); - assert.equal(isEnrichmentBudgetExhausted(27_300, 3_900, 31_200), false); - assert.equal(isEnrichmentBudgetExhausted(27_301, 3_900, 31_200), true); + assert.equal(configuredDailyEnrichmentBudget("46800"), 46_800); + assert.equal(SHOWCASE_ENRICHMENT_SANDBOX_OVERHEAD_MS, 60_000); + assert.equal(SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, 180_000); + assert.equal(projectedEnrichmentAttemptMicrousd(117_000), 5_850); + assert.equal(isEnrichmentBudgetExhausted(40_950, 5_850, 46_800), false); + assert.equal(isEnrichmentBudgetExhausted(40_951, 5_850, 46_800), true); for (const value of [undefined, "", "0", "-1", "1.5", "1000000001", "nope"]) { assert.throws( () => configuredDailyEnrichmentBudget(value), @@ -197,7 +199,11 @@ test("the enrichment core stays independent from runs and the judge budget", asy ); assert.match( evaluatorModule, - /sandboxStartedAt = Date\.now\(\)[\s\S]*?durationMs:[\s\S]*?SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS/, + /attemptStartedAt = Date\.now\(\)[\s\S]*?durationMs: Math\.min[\s\S]*?SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS/, + ); + assert.match( + dataModule, + /ENRICHMENT_CONFIGURATION_RETRY_MS = 5 \* 60 \* 1000[\s\S]*?retryAt: configurationRetryAt/, ); }); @@ -211,7 +217,7 @@ test("staging and deploy preparation require the enrichment spend cap", async () for (const source of [config, envExample, preflight, prepare]) { assert.match(source, /BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET/); } - assert.match(config, /"BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET": "31200"/); + assert.match(config, /"BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET": "46800"/); assert.match(prepare, /environmentName === "staging"/); assert.match(prepare, /environment\.routes\?\.length/); assert.equal( diff --git a/wrangler.jsonc b/wrangler.jsonc index 829b4cd..30e9d64 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -86,9 +86,9 @@ "BENCHMAX_JUDGE_DAILY_SAMPLE_BUDGET": "30", "BENCHMAX_JUDGE_INPUT_MICROUSD_PER_MILLION_TOKENS": "950000", "BENCHMAX_JUDGE_OUTPUT_MICROUSD_PER_MILLION_TOKENS": "4000000", - // Eight launch-volume enrichments at the pinned 120-second ceiling, + // Eight launch-volume enrichments at the pinned 180-second ceiling, // priced at the measured 117000 microUSD/hour staging rate. - "BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET": "31200", + "BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET": "46800", "BENCHMAX_SANDBOX_MICROUSD_PER_HOUR": "117000" }, "d1_databases": [ From 17831b20161ffad06054eb9cea640119595f737f Mon Sep 17 00:00:00 2001 From: Benchmax Builder Date: Wed, 12 Aug 2026 19:15:37 +0500 Subject: [PATCH 8/8] Retry transient enrichment reservations Co-authored-by: Codex --- lib/data/showcase-enrichment.ts | 38 ++++++++++++++++++++++++------- tests/showcase-enrichment.test.ts | 6 ++++- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/lib/data/showcase-enrichment.ts b/lib/data/showcase-enrichment.ts index 0eee778..49f69fc 100644 --- a/lib/data/showcase-enrichment.ts +++ b/lib/data/showcase-enrichment.ts @@ -285,6 +285,12 @@ export async function claimShowcaseEnrichment( WHERE inflight_enrichment.status = 'running' AND inflight_enrichment.lease_expires_at > ${now.getTime()} )`, + nextLeaseExpiresAt: sql`( + SELECT min(inflight_enrichment.lease_expires_at) + FROM showcase_enrichments inflight_enrichment + WHERE inflight_enrichment.status = 'running' + AND inflight_enrichment.lease_expires_at > ${now.getTime()} + )`, spentMicrousd: sql`( SELECT coalesce(sum(${showcaseEnrichmentSpendRecords.costMicrousd}), 0) FROM ${showcaseEnrichmentSpendRecords} @@ -309,22 +315,38 @@ export async function claimShowcaseEnrichment( dailyBudgetMicrousd, ) ) { + const spentMicrousd = Number(existing.spentMicrousd); + const reservedMicrousd = + Number(existing.inFlightCount) * projectedAttemptMicrousd; + const recordedSpendExhausted = isEnrichmentBudgetExhausted( + spentMicrousd, + projectedAttemptMicrousd, + dailyBudgetMicrousd, + ); + const retryAt = recordedSpendExhausted + ? nextDayStartedAt + : new Date( + Math.min( + nextDayStartedAt.getTime(), + Number(existing.nextLeaseExpiresAt) || + now.getTime() + ENRICHMENT_LEASE_MS, + ), + ); await deferShowcaseEnrichmentUntil( enrichmentId, now, - nextDayStartedAt, + retryAt, ); await recordEnrichmentBudgetDeferral({ budgetMicrousd: dailyBudgetMicrousd, dayStartedAt, enrichmentId, - nextDayStartedAt, + retryAt, projectedAttemptMicrousd, - reservedMicrousd: - Number(existing.inFlightCount) * projectedAttemptMicrousd, - spentMicrousd: Number(existing.spentMicrousd), + reservedMicrousd, + spentMicrousd, }); - return { action: "defer", retryAt: nextDayStartedAt }; + return { action: "defer", retryAt }; } if (existing?.status === "running" && existing.leaseExpiresAt) { return { action: "retry", leaseExpiresAt: existing.leaseExpiresAt }; @@ -385,7 +407,7 @@ async function recordEnrichmentBudgetDeferral(input: { budgetMicrousd: number; dayStartedAt: Date; enrichmentId: string; - nextDayStartedAt: Date; + retryAt: Date; projectedAttemptMicrousd: number; reservedMicrousd: number; spentMicrousd: number; @@ -404,7 +426,7 @@ async function recordEnrichmentBudgetDeferral(input: { metadataJson: canonicalJson({ budgetMicrousd: input.budgetMicrousd, dayStartedAt: input.dayStartedAt.toISOString(), - retryAt: input.nextDayStartedAt.toISOString(), + retryAt: input.retryAt.toISOString(), projectedAttemptMicrousd: input.projectedAttemptMicrousd, reservedMicrousd: input.reservedMicrousd, spentMicrousd: input.spentMicrousd, diff --git a/tests/showcase-enrichment.test.ts b/tests/showcase-enrichment.test.ts index 0fbb663..5b51b94 100644 --- a/tests/showcase-enrichment.test.ts +++ b/tests/showcase-enrichment.test.ts @@ -187,7 +187,7 @@ test("the enrichment core stays independent from runs and the judge budget", asy ); assert.equal( (dataModule.match(/nextDayStartedAt\.getTime\(\)/g) ?? []).length, - 2, + 3, ); assert.match( dataModule, @@ -205,6 +205,10 @@ test("the enrichment core stays independent from runs and the judge budget", asy dataModule, /ENRICHMENT_CONFIGURATION_RETRY_MS = 5 \* 60 \* 1000[\s\S]*?retryAt: configurationRetryAt/, ); + assert.match( + dataModule, + /SELECT min\(inflight_enrichment\.lease_expires_at\)[\s\S]*?recordedSpendExhausted[\s\S]*?nextLeaseExpiresAt/, + ); }); test("staging and deploy preparation require the enrichment spend cap", async () => {