diff --git a/.env.example b/.env.example index 3df1646..5611f42 100644 --- a/.env.example +++ b/.env.example @@ -47,6 +47,9 @@ BENCHMAX_JUDGE_DAILY_SAMPLE_BUDGET= # pricing changes. Benchmax never hard-codes a provider price. BENCHMAX_JUDGE_INPUT_MICROUSD_PER_MILLION_TOKENS= BENCHMAX_JUDGE_OUTPUT_MICROUSD_PER_MILLION_TOKENS= +# Maximum measured E2B preview-enrichment spend per UTC day. This must be set +# before submissions are enabled; budget deferrals leave public previews pending. +BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET= BENCHMAX_SANDBOX_MICROUSD_PER_HOUR= # User-content Worker only: exact HTTPS origin allowed to frame legacy diff --git a/lib/data/showcase-enrichment.ts b/lib/data/showcase-enrichment.ts index ae4e0a2..49f69fc 100644 --- a/lib/data/showcase-enrichment.ts +++ b/lib/data/showcase-enrichment.ts @@ -3,6 +3,7 @@ import { and, asc, eq, inArray, isNotNull, lte, or, sql } from "drizzle-orm"; import { getDb } from "@/db"; import { artifacts, + auditEvents, showcaseEnrichmentArtifacts, showcaseEnrichmentSpendRecords, showcaseEnrichments, @@ -10,18 +11,31 @@ import { } from "@/db/schema"; import { canonicalJson, canonicalSha256 } from "@/lib/security/canonical"; import { sha256Hex } from "@/lib/security/policy"; -import { sandboxRateFromEnv } from "@/lib/data/result-spend"; +import { + sandboxRateFromEnv, + SpendPricingConfigurationError, +} from "@/lib/data/result-spend"; import { showcaseEnrichmentMessage, type ShowcaseEnrichmentMessage, } from "@/lib/pipeline/enrichment-messages"; import { sanitizeEnrichmentFailureCode } from "@/lib/pipeline/enrichment-policy"; +import { + configuredDailyEnrichmentBudget, + EnrichmentBudgetConfigurationError, + enrichmentBudgetConfigurationDeferralAuditId, + enrichmentBudgetDeferralAuditId, + enrichmentBudgetWindow, + isEnrichmentBudgetExhausted, + projectedEnrichmentAttemptMicrousd, +} from "@/lib/pipeline/enrichment-budget"; const ZIP_CONTENT_TYPES = [ "application/zip", "application/x-zip-compressed", ] as const; const ENRICHMENT_LEASE_MS = 5 * 60 * 1000; +const ENRICHMENT_CONFIGURATION_RETRY_MS = 5 * 60 * 1000; const MICROS_PER_HOUR_DIVISOR = 3_600_000; export type ShowcaseEnrichmentArtifactKind = @@ -32,6 +46,7 @@ export type ShowcaseEnrichmentArtifactKind = export type ShowcaseEnrichmentClaim = | { action: "execute"; attemptCount: number; leaseExpiresAt: Date } + | { action: "defer"; retryAt: Date } | { action: "retry"; leaseExpiresAt: Date } | { action: "skip" }; @@ -162,7 +177,12 @@ export async function reconcileShowcaseEnrichments(limit = 50) { const queued = await getDb() .select({ id: showcaseEnrichments.id }) .from(showcaseEnrichments) - .where(eq(showcaseEnrichments.status, "queued")) + .where( + and( + eq(showcaseEnrichments.status, "queued"), + lte(showcaseEnrichments.updatedAt, now), + ), + ) .orderBy(asc(showcaseEnrichments.updatedAt), asc(showcaseEnrichments.id)) .limit(boundedLimit); const dispatched: string[] = []; @@ -182,6 +202,38 @@ export async function claimShowcaseEnrichment( enrichmentId: string, now = new Date(), ): Promise { + const { dayStartedAt, nextDayStartedAt } = enrichmentBudgetWindow(now); + let dailyBudgetMicrousd: number; + let projectedAttemptMicrousd: number; + try { + dailyBudgetMicrousd = configuredDailyEnrichmentBudget(); + projectedAttemptMicrousd = projectedEnrichmentAttemptMicrousd( + sandboxRateFromEnv(), + ); + } catch (error) { + if ( + error instanceof EnrichmentBudgetConfigurationError || + error instanceof SpendPricingConfigurationError + ) { + // Configuration must fail closed without consuming the durable queue + // retry budget or turning an optional public preview terminally failed. + const configurationRetryAt = new Date( + now.getTime() + ENRICHMENT_CONFIGURATION_RETRY_MS, + ); + await recordEnrichmentBudgetConfigurationDeferral({ + dayStartedAt, + enrichmentId, + retryAt: configurationRetryAt, + }); + await deferShowcaseEnrichmentUntil( + enrichmentId, + now, + configurationRetryAt, + ); + return { action: "defer", retryAt: configurationRetryAt }; + } + throw error; + } const leaseExpiresAt = new Date(now.getTime() + ENRICHMENT_LEASE_MS); const [claimed] = await getDb() .update(showcaseEnrichments) @@ -203,6 +255,17 @@ export async function claimShowcaseEnrichment( lte(showcaseEnrichments.leaseExpiresAt, now), ), ), + sql`( + SELECT coalesce(sum(${showcaseEnrichmentSpendRecords.costMicrousd}), 0) + FROM ${showcaseEnrichmentSpendRecords} + WHERE ${showcaseEnrichmentSpendRecords.createdAt} >= ${dayStartedAt.getTime()} + AND ${showcaseEnrichmentSpendRecords.createdAt} < ${nextDayStartedAt.getTime()} + ) + ( + SELECT count(*) * ${projectedAttemptMicrousd} + FROM showcase_enrichments inflight_enrichment + WHERE inflight_enrichment.status = 'running' + AND inflight_enrichment.lease_expires_at > ${now.getTime()} + ) + ${projectedAttemptMicrousd} <= ${dailyBudgetMicrousd}`, ), ) .returning({ attemptCount: showcaseEnrichments.attemptCount }); @@ -216,17 +279,163 @@ export async function claimShowcaseEnrichment( const [existing] = await getDb() .select({ leaseExpiresAt: showcaseEnrichments.leaseExpiresAt, + inFlightCount: sql`( + SELECT count(*) + FROM showcase_enrichments inflight_enrichment + WHERE inflight_enrichment.status = 'running' + AND inflight_enrichment.lease_expires_at > ${now.getTime()} + )`, + nextLeaseExpiresAt: sql`( + SELECT min(inflight_enrichment.lease_expires_at) + FROM showcase_enrichments inflight_enrichment + WHERE inflight_enrichment.status = 'running' + AND inflight_enrichment.lease_expires_at > ${now.getTime()} + )`, + spentMicrousd: sql`( + SELECT coalesce(sum(${showcaseEnrichmentSpendRecords.costMicrousd}), 0) + FROM ${showcaseEnrichmentSpendRecords} + WHERE ${showcaseEnrichmentSpendRecords.createdAt} >= ${dayStartedAt.getTime()} + AND ${showcaseEnrichmentSpendRecords.createdAt} < ${nextDayStartedAt.getTime()} + )`, status: showcaseEnrichments.status, }) .from(showcaseEnrichments) .where(eq(showcaseEnrichments.id, enrichmentId)) .limit(1); + if ( + existing && + (existing.status === "queued" || + (existing.status === "running" && + existing.leaseExpiresAt && + existing.leaseExpiresAt <= now)) && + isEnrichmentBudgetExhausted( + Number(existing.spentMicrousd) + + Number(existing.inFlightCount) * projectedAttemptMicrousd, + projectedAttemptMicrousd, + dailyBudgetMicrousd, + ) + ) { + const spentMicrousd = Number(existing.spentMicrousd); + const reservedMicrousd = + Number(existing.inFlightCount) * projectedAttemptMicrousd; + const recordedSpendExhausted = isEnrichmentBudgetExhausted( + spentMicrousd, + projectedAttemptMicrousd, + dailyBudgetMicrousd, + ); + const retryAt = recordedSpendExhausted + ? nextDayStartedAt + : new Date( + Math.min( + nextDayStartedAt.getTime(), + Number(existing.nextLeaseExpiresAt) || + now.getTime() + ENRICHMENT_LEASE_MS, + ), + ); + await deferShowcaseEnrichmentUntil( + enrichmentId, + now, + retryAt, + ); + await recordEnrichmentBudgetDeferral({ + budgetMicrousd: dailyBudgetMicrousd, + dayStartedAt, + enrichmentId, + retryAt, + projectedAttemptMicrousd, + reservedMicrousd, + spentMicrousd, + }); + return { action: "defer", retryAt }; + } if (existing?.status === "running" && existing.leaseExpiresAt) { return { action: "retry", leaseExpiresAt: existing.leaseExpiresAt }; } return { action: "skip" }; } +async function deferShowcaseEnrichmentUntil( + enrichmentId: string, + now: Date, + retryAt: Date, +) { + await getDb() + .update(showcaseEnrichments) + .set({ status: "queued", leaseExpiresAt: null, updatedAt: retryAt }) + .where( + and( + eq(showcaseEnrichments.id, enrichmentId), + lte(showcaseEnrichments.updatedAt, now), + or( + eq(showcaseEnrichments.status, "queued"), + and( + eq(showcaseEnrichments.status, "running"), + lte(showcaseEnrichments.leaseExpiresAt, now), + ), + ), + ), + ); +} + +async function recordEnrichmentBudgetConfigurationDeferral(input: { + dayStartedAt: Date; + enrichmentId: string; + retryAt: Date; +}) { + await getDb() + .insert(auditEvents) + .values({ + id: enrichmentBudgetConfigurationDeferralAuditId( + input.enrichmentId, + input.dayStartedAt, + ), + actorUserId: null, + entityId: input.enrichmentId, + entityType: "showcase-enrichment", + action: "showcase.preview_enrichment_budget_configuration_deferred", + metadataJson: canonicalJson({ + dayStartedAt: input.dayStartedAt.toISOString(), + reason: "invalid-runtime-configuration", + retryAt: input.retryAt.toISOString(), + }), + createdAt: new Date(), + }) + .onConflictDoNothing({ target: auditEvents.id }); +} + +async function recordEnrichmentBudgetDeferral(input: { + budgetMicrousd: number; + dayStartedAt: Date; + enrichmentId: string; + retryAt: Date; + projectedAttemptMicrousd: number; + reservedMicrousd: number; + spentMicrousd: number; +}) { + await getDb() + .insert(auditEvents) + .values({ + id: enrichmentBudgetDeferralAuditId( + input.enrichmentId, + input.dayStartedAt, + ), + actorUserId: null, + entityId: input.enrichmentId, + entityType: "showcase-enrichment", + action: "showcase.preview_enrichment_budget_deferred", + metadataJson: canonicalJson({ + budgetMicrousd: input.budgetMicrousd, + dayStartedAt: input.dayStartedAt.toISOString(), + retryAt: input.retryAt.toISOString(), + projectedAttemptMicrousd: input.projectedAttemptMicrousd, + reservedMicrousd: input.reservedMicrousd, + spentMicrousd: input.spentMicrousd, + }), + createdAt: new Date(), + }) + .onConflictDoNothing({ target: auditEvents.id }); +} + export async function readShowcaseEnrichmentContract(enrichmentId: string) { const [row] = await getDb() .select({ diff --git a/lib/evaluation/showcase-preview.ts b/lib/evaluation/showcase-preview.ts index fb62040..33cf6fb 100644 --- a/lib/evaluation/showcase-preview.ts +++ b/lib/evaluation/showcase-preview.ts @@ -15,6 +15,7 @@ import { } from "@/lib/evaluation/preview-spec"; import { canonicalJson } from "@/lib/security/canonical"; import { constantTimeEqualHex, sha256Hex } from "@/lib/security/policy"; +import { SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS } from "@/lib/pipeline/enrichment-budget"; const previewReportSchema = z .object({ @@ -72,7 +73,7 @@ export async function executeShowcasePreviewEnrichment( const templateId = requiredSecret("E2B_TEMPLATE_ID"); const templateBuildHash = requiredSha256("E2B_TEMPLATE_BUILD_HASH"); const attemptKey = `sandbox:${enrichmentId}:preview:${crypto.randomUUID()}`; - const startedAt = Date.now(); + const attemptStartedAt = Date.now(); let sandbox: Sandbox | null = null; let spendStatus: "completed" | "failed" = "failed"; try { @@ -80,7 +81,7 @@ export async function executeShowcasePreviewEnrichment( apiKey: requiredSecret("E2B_API_KEY"), allowInternetAccess: false, secure: true, - timeoutMs: EVALUATION_ENVIRONMENT_V1.wallClockSeconds * 1_000, + timeoutMs: SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, lifecycle: { onTimeout: "kill" }, metadata: { benchmaxEnrichmentId: enrichmentId, @@ -178,7 +179,13 @@ export async function executeShowcasePreviewEnrichment( await sandbox?.kill().catch(() => false); await recordShowcaseEnrichmentSpend({ attemptKey, - durationMs: Math.max(0, Date.now() - startedAt), + // Conservatively include provisioning and cleanup. The stored duration + // and claim reservation share the sandbox lifecycle ceiling, which has + // explicit headroom above the inner evaluator command timeout. + durationMs: Math.min( + Math.max(0, Date.now() - attemptStartedAt), + SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, + ), enrichmentId, status: spendStatus, }).catch((error) => { diff --git a/lib/pipeline/enrichment-budget.ts b/lib/pipeline/enrichment-budget.ts new file mode 100644 index 0000000..8828a77 --- /dev/null +++ b/lib/pipeline/enrichment-budget.ts @@ -0,0 +1,80 @@ +import { EVALUATION_ENVIRONMENT_V1 } from "@/lib/domain/ranked-catalog"; + +export const MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD = 1_000_000_000; +export const SHOWCASE_ENRICHMENT_SANDBOX_OVERHEAD_MS = 60_000; +export const SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS = + EVALUATION_ENVIRONMENT_V1.wallClockSeconds * 1_000 + + SHOWCASE_ENRICHMENT_SANDBOX_OVERHEAD_MS; + +export function configuredDailyEnrichmentBudget( + value = process.env.BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET, +) { + const parsed = Number(value); + if ( + !Number.isSafeInteger(parsed) || + parsed < 1 || + parsed > MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD + ) { + throw new EnrichmentBudgetConfigurationError(); + } + return parsed; +} + +export function enrichmentBudgetWindow(now = new Date()) { + const dayStartedAt = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()), + ); + return { + dayStartedAt, + nextDayStartedAt: new Date(dayStartedAt.getTime() + 24 * 60 * 60 * 1_000), + }; +} + +export function enrichmentBudgetDeferralAuditId( + enrichmentId: string, + dayStartedAt: Date, +) { + return `showcase-enrichment-budget:${dayStartedAt.toISOString().slice(0, 10)}:${enrichmentId}`; +} + +export function enrichmentBudgetConfigurationDeferralAuditId( + enrichmentId: string, + dayStartedAt: Date, +) { + return `showcase-enrichment-budget-configuration:${dayStartedAt.toISOString().slice(0, 10)}:${enrichmentId}`; +} + +export function isEnrichmentBudgetExhausted( + committedMicrousd: number, + projectedAttemptMicrousd: number, + budgetMicrousd: number, +) { + return committedMicrousd + projectedAttemptMicrousd > budgetMicrousd; +} + +export function projectedEnrichmentAttemptMicrousd( + rateMicrousdPerHour: number, + maximumDurationMs = SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, +) { + if ( + !Number.isSafeInteger(rateMicrousdPerHour) || + rateMicrousdPerHour < 0 || + !Number.isSafeInteger(maximumDurationMs) || + maximumDurationMs < 1 + ) { + throw new RangeError("Projected enrichment spend inputs are invalid."); + } + const numerator = BigInt(maximumDurationMs) * BigInt(rateMicrousdPerHour); + return Number( + (numerator + BigInt(3_599_999)) / BigInt(3_600_000), + ); +} + +export class EnrichmentBudgetConfigurationError extends Error { + constructor() { + super( + "The daily preview enrichment budget is not configured. The Test remains public with its preview pending.", + ); + this.name = "EnrichmentBudgetConfigurationError"; + } +} diff --git a/scripts/phase2-preflight.mjs b/scripts/phase2-preflight.mjs index f6d3a9b..57f02e5 100644 --- a/scripts/phase2-preflight.mjs +++ b/scripts/phase2-preflight.mjs @@ -34,6 +34,7 @@ const environments = { }, }; const requiredCrons = ["*/2 * * * *", "0 3 * * 1"]; +const MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD = 1_000_000_000; const databaseIds = {}; const queueNamesByEnvironment = {}; @@ -116,11 +117,22 @@ const requiredEnvNames = [ "BENCHMAX_JUDGE_DAILY_SAMPLE_BUDGET", "BENCHMAX_JUDGE_INPUT_MICROUSD_PER_MILLION_TOKENS", "BENCHMAX_JUDGE_OUTPUT_MICROUSD_PER_MILLION_TOKENS", + "BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET", "BENCHMAX_SANDBOX_MICROUSD_PER_HOUR", "BENCHMAX_APP_ORIGIN", ]; for (const name of requiredEnvNames) assert(envNames.has(name), `missing .env.example key: ${name}`); +const stagingEnrichmentBudget = Number( + environments.staging.main.vars?.BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET, +); +assert( + Number.isSafeInteger(stagingEnrichmentBudget) && + stagingEnrichmentBudget >= 1 && + stagingEnrichmentBudget <= MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD, + "main Worker staging must configure BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET", +); + const forbiddenBindingPattern = /\bIMAGES\b/; for (const filePath of sourceFiles(rootDirectory)) { if (filePath === path.join(rootDirectory, "scripts", "phase2-preflight.mjs")) continue; diff --git a/scripts/prepare-main-deploy.mjs b/scripts/prepare-main-deploy.mjs index a5f5166..bbac9ba 100644 --- a/scripts/prepare-main-deploy.mjs +++ b/scripts/prepare-main-deploy.mjs @@ -14,6 +14,27 @@ const builtConfig = JSON.parse(fs.readFileSync(builtConfigPath, "utf8")); const sourceConfig = readJsonc(rootDirectory, "wrangler.jsonc"); const environment = sourceConfig.env?.[environmentName]; if (!environment) throw new Error(`Missing main Worker environment: ${environmentName}`); +const MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD = 1_000_000_000; + +// Staging is submission-enabled today. Any future environment with public +// routes must also configure the cap before its deploy config can be prepared. +const requiresEnrichmentBudget = + environmentName === "staging" || (environment.routes?.length ?? 0) > 0; +const requiredEnvironmentVars = requiresEnrichmentBudget + ? ["BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET"] + : []; +for (const key of requiredEnvironmentVars) { + const value = Number(environment.vars?.[key]); + if ( + !Number.isSafeInteger(value) || + value < 1 || + value > MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD + ) { + throw new Error( + `main Worker ${environmentName} environment must set a positive integer ${key} before deployment`, + ); + } +} // Every key the environment block must override. An absent key would previously // serialize as undefined and silently DELETE the section from the deploy config diff --git a/tests/showcase-enrichment.test.ts b/tests/showcase-enrichment.test.ts index c6a7860..5b51b94 100644 --- a/tests/showcase-enrichment.test.ts +++ b/tests/showcase-enrichment.test.ts @@ -12,6 +12,17 @@ import { enrichmentRetryDelaySeconds, sanitizeEnrichmentFailureCode, } from "../lib/pipeline/enrichment-policy"; +import { + configuredDailyEnrichmentBudget, + EnrichmentBudgetConfigurationError, + enrichmentBudgetConfigurationDeferralAuditId, + enrichmentBudgetDeferralAuditId, + enrichmentBudgetWindow, + isEnrichmentBudgetExhausted, + projectedEnrichmentAttemptMicrousd, + SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, + SHOWCASE_ENRICHMENT_SANDBOX_OVERHEAD_MS, +} from "../lib/pipeline/enrichment-budget"; import { usercontentWorker, type UsercontentEnv, @@ -89,7 +100,46 @@ test("generic preview checks are complete but never become a judge rubric", () = ); }); -test("the independent enrichment core has no run, judge, or budget dependency", async () => { +test("preview enrichment has an explicit UTC daily spend cap and stable audit dedupe", () => { + assert.equal(configuredDailyEnrichmentBudget("46800"), 46_800); + assert.equal(SHOWCASE_ENRICHMENT_SANDBOX_OVERHEAD_MS, 60_000); + assert.equal(SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS, 180_000); + assert.equal(projectedEnrichmentAttemptMicrousd(117_000), 5_850); + assert.equal(isEnrichmentBudgetExhausted(40_950, 5_850, 46_800), false); + assert.equal(isEnrichmentBudgetExhausted(40_951, 5_850, 46_800), true); + for (const value of [undefined, "", "0", "-1", "1.5", "1000000001", "nope"]) { + assert.throws( + () => configuredDailyEnrichmentBudget(value), + EnrichmentBudgetConfigurationError, + ); + } + + const beforeReset = enrichmentBudgetWindow( + new Date("2026-08-12T23:59:59.999Z"), + ); + assert.equal(beforeReset.dayStartedAt.toISOString(), "2026-08-12T00:00:00.000Z"); + assert.equal(beforeReset.nextDayStartedAt.toISOString(), "2026-08-13T00:00:00.000Z"); + assert.equal( + enrichmentBudgetDeferralAuditId(enrichmentId, beforeReset.dayStartedAt), + enrichmentBudgetDeferralAuditId( + enrichmentId, + new Date("2026-08-12T00:00:00.000Z"), + ), + ); + assert.notEqual( + enrichmentBudgetDeferralAuditId(enrichmentId, beforeReset.dayStartedAt), + enrichmentBudgetDeferralAuditId(enrichmentId, beforeReset.nextDayStartedAt), + ); + assert.notEqual( + enrichmentBudgetConfigurationDeferralAuditId( + enrichmentId, + beforeReset.dayStartedAt, + ), + enrichmentBudgetDeferralAuditId(enrichmentId, beforeReset.dayStartedAt), + ); +}); + +test("the enrichment core stays independent from runs and the judge budget", async () => { const [dataModule, evaluatorModule] = await Promise.all([ readFile( path.join(projectRoot, "lib", "data", "showcase-enrichment.ts"), @@ -112,6 +162,76 @@ test("the independent enrichment core has no run, judge, or budget dependency", ]) { assert.doesNotMatch(implementation, new RegExp(forbidden)); } + assert.match(dataModule, /showcase\.preview_enrichment_budget_deferred/); + assert.match( + dataModule, + /showcase\.preview_enrichment_budget_configuration_deferred/, + ); + assert.match(dataModule, /onConflictDoNothing\(\{ target: auditEvents\.id \}\)/); + assert.match(dataModule, /status: "queued", leaseExpiresAt: null/); + assert.match( + dataModule, + /eq\(showcaseEnrichments\.status, "queued"\),[\s\S]*?lte\(showcaseEnrichments\.updatedAt, now\)/, + ); + assert.match( + dataModule, + /set\(\{ status: "queued", leaseExpiresAt: null, updatedAt: retryAt \}\)/, + ); + assert.match( + dataModule, + /error instanceof EnrichmentBudgetConfigurationError[\s\S]*?action: "defer"/, + ); + assert.equal( + (dataModule.match(/dayStartedAt\.getTime\(\)/g) ?? []).length, + 2, + ); + assert.equal( + (dataModule.match(/nextDayStartedAt\.getTime\(\)/g) ?? []).length, + 3, + ); + assert.match( + dataModule, + /count\(\*\) \* \$\{projectedAttemptMicrousd\}[\s\S]*?lease_expires_at > \$\{now\.getTime\(\)\}[\s\S]*?\+ \$\{projectedAttemptMicrousd\} <= \$\{dailyBudgetMicrousd\}/, + ); + assert.match( + dataModule, + /eq\(showcaseEnrichments\.id, enrichmentId\),[\s\S]*?lte\(showcaseEnrichments\.updatedAt, now\),[\s\S]*?eq\(showcaseEnrichments\.status, "queued"\)/, + ); + assert.match( + evaluatorModule, + /attemptStartedAt = Date\.now\(\)[\s\S]*?durationMs: Math\.min[\s\S]*?SHOWCASE_ENRICHMENT_SANDBOX_MAX_DURATION_MS/, + ); + assert.match( + dataModule, + /ENRICHMENT_CONFIGURATION_RETRY_MS = 5 \* 60 \* 1000[\s\S]*?retryAt: configurationRetryAt/, + ); + assert.match( + dataModule, + /SELECT min\(inflight_enrichment\.lease_expires_at\)[\s\S]*?recordedSpendExhausted[\s\S]*?nextLeaseExpiresAt/, + ); +}); + +test("staging and deploy preparation require the enrichment spend cap", async () => { + const [config, envExample, preflight, prepare] = await Promise.all([ + readFile(path.join(projectRoot, "wrangler.jsonc"), "utf8"), + readFile(path.join(projectRoot, ".env.example"), "utf8"), + readFile(path.join(projectRoot, "scripts", "phase2-preflight.mjs"), "utf8"), + readFile(path.join(projectRoot, "scripts", "prepare-main-deploy.mjs"), "utf8"), + ]); + for (const source of [config, envExample, preflight, prepare]) { + assert.match(source, /BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET/); + } + assert.match(config, /"BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET": "46800"/); + assert.match(prepare, /environmentName === "staging"/); + assert.match(prepare, /environment\.routes\?\.length/); + assert.equal( + (preflight.match(/MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD/g) ?? []).length, + 2, + ); + assert.equal( + (prepare.match(/MAX_DAILY_ENRICHMENT_BUDGET_MICROUSD/g) ?? []).length, + 2, + ); }); test("completed derived evidence is served only through every public safety gate", async () => { diff --git a/worker/index.ts b/worker/index.ts index b7e6534..d2077c7 100644 --- a/worker/index.ts +++ b/worker/index.ts @@ -428,6 +428,12 @@ async function consumeShowcaseEnrichmentMessage( message.ack(); return; } + if (claim.action === "defer") { + // The row intentionally stays queued. The reconciliation sweep will send + // a fresh message after the UTC spend window resets. + message.ack(); + return; + } if (claim.action === "retry") { message.retry({ delaySeconds: enrichmentRetryDelaySeconds(claim.leaseExpiresAt), diff --git a/wrangler.jsonc b/wrangler.jsonc index 392c9fe..30e9d64 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -86,6 +86,9 @@ "BENCHMAX_JUDGE_DAILY_SAMPLE_BUDGET": "30", "BENCHMAX_JUDGE_INPUT_MICROUSD_PER_MILLION_TOKENS": "950000", "BENCHMAX_JUDGE_OUTPUT_MICROUSD_PER_MILLION_TOKENS": "4000000", + // Eight launch-volume enrichments at the pinned 180-second ceiling, + // priced at the measured 117000 microUSD/hour staging rate. + "BENCHMAX_ENRICHMENT_DAILY_MICROUSD_BUDGET": "46800", "BENCHMAX_SANDBOX_MICROUSD_PER_HOUR": "117000" }, "d1_databases": [