Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions app/api/showcases/[id]/enrichment/retry/route.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import { requireAuthorizedUser } from "@/lib/auth/authorization";
import { appendAuditEvent } from "@/lib/data/audit";
import { retryShowcaseEnrichment } from "@/lib/data/showcase-enrichment";
import { apiErrorResponse } from "@/lib/http/api";
import { secureJson } from "@/lib/security/http";
import { enforceRateLimit } from "@/lib/security/rate-limit";

export async function POST(
request: Request,
context: { params: Promise<{ id: string }> },
) {
try {
const { identity, user } = await requireAuthorizedUser(request);
await enforceRateLimit(identity.subject, {
action: "showcase-enrichment-retry",
limit: 10,
windowMs: 24 * 60 * 60 * 1000,
});
const { id } = await context.params;
const retry = await retryShowcaseEnrichment(id, user.id);
if (!retry) {
return secureJson(
{ error: "No failed automated preview is available to retry." },
{ status: 409 },
);
}
await appendAuditEvent({
actorUserId: user.id,
entityType: "showcase",
entityId: id,
action: "showcase.enrichment_retried",
metadata: { dispatchDeferred: retry.dispatchDeferred },
});
return secureJson({ retry });
} catch (error) {
return apiErrorResponse(error);
}
}
40 changes: 40 additions & 0 deletions app/api/showcases/[id]/processing/retry/route.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import { requireAuthorizedUser } from "@/lib/auth/authorization";
import { appendAuditEvent } from "@/lib/data/audit";
import { retryShowcaseProcessing } from "@/lib/data/showcase-processing";
import { apiErrorResponse } from "@/lib/http/api";
import { secureJson } from "@/lib/security/http";
import { enforceRateLimit } from "@/lib/security/rate-limit";

export async function POST(
request: Request,
context: { params: Promise<{ id: string }> },
) {
try {
const { identity, user } = await requireAuthorizedUser(request);
await enforceRateLimit(identity.subject, {
action: "showcase-processing-retry",
limit: 20,
windowMs: 24 * 60 * 60 * 1000,
});
const { id } = await context.params;
const processing = await retryShowcaseProcessing(id, user.id);
await appendAuditEvent({
actorUserId: user.id,
entityType: "showcase",
entityId: id,
action:
processing.outcome === "ready"
? "showcase.processing_retry_completed"
: processing.outcome === "blocked"
? "showcase.processing_retry_blocked"
: "showcase.processing_retry_pending",
metadata: {
outcome: processing.outcome,
scannedArtifactCount: processing.scannedArtifactIds.length,
},
});
return secureJson({ processing });
} catch (error) {
return apiErrorResponse(error);
}
}
35 changes: 22 additions & 13 deletions app/api/showcases/[id]/publish/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { apiErrorResponse } from "@/lib/http/api";
import { secureJson } from "@/lib/security/http";
import { enforceRateLimit } from "@/lib/security/rate-limit";
import { verifyApprovedShowcaseArtifacts } from "@/lib/security/artifact-scanner";
import { queuePublishedResult } from "@/lib/data/results";
import { scheduleShowcaseEnrichment } from "@/lib/data/showcase-enrichment";

export async function POST(
request: Request,
Expand All @@ -31,27 +31,36 @@ export async function POST(
}
await verifyApprovedShowcaseArtifacts(id);
const showcase = await publishShowcase(id, user.id);
let run: Awaited<ReturnType<typeof queuePublishedResult>>["run"] | null =
null;
let judgeQueueDeferred = false;
let enrichment = {
dispatchDeferred: false,
eligible: false,
enrichmentId: null as string | null,
};
try {
const queued = await queuePublishedResult(showcase.id);
run = queued.run;
judgeQueueDeferred = queued.judgeQueueDeferred;
enrichment = await scheduleShowcaseEnrichment(showcase.id);
} catch {
judgeQueueDeferred = true;
enrichment = {
dispatchDeferred: true,
eligible: true,
enrichmentId: null,
};
}
Comment on lines 41 to 47

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Publication record can claim an automated preview was scheduled when none exists

When scheduling the automated preview fails outright, the publication record is written as if the Test were preview-eligible and merely delayed (enrichment = { dispatchDeferred: true, eligible: true, ... } at app/api/showcases/[id]/publish/route.ts:42-46), so the audit trail and the response claim a preview is coming for Tests that will never have one.
Impact: Operators reading publication records, and the contributor reading the publish response, are told an automated preview is pending for submissions that are not eligible for one.

Why the catch branch cannot know eligibility

scheduleShowcaseEnrichment (lib/data/showcase-enrichment.ts:38-69) already swallows queue-dispatch failures internally and returns { dispatchDeferred: true, eligible: true } in that case. The only way it throws is when ensureShowcaseEnrichment itself fails (for example a database error) — at which point no showcase_enrichments row exists and eligibility is unknown: the Test may have no compatible source ZIP at all.

Hard-coding eligible: true in the route's catch therefore reports an eligibility the code never determined. Reporting eligible: false (or a distinct unknown marker) would keep the record honest; the scheduled reconciliation sweep still creates and dispatches the row later if the submission really is eligible.

Suggested change
} catch {
judgeQueueDeferred = true;
enrichment = {
dispatchDeferred: true,
eligible: true,
enrichmentId: null,
};
}
} catch {
enrichment = {
dispatchDeferred: true,
eligible: false,
enrichmentId: null,
};
}
Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

await appendAuditEvent({
actorUserId: user.id,
entityType: "showcase",
entityId: showcase.id,
action: "showcase.published",
metadata: { judgeQueueDeferred },
metadata: {
enrichmentDeferred: enrichment.dispatchDeferred,
enrichmentEligible: enrichment.eligible,
reviewStatus: "awaiting_review",
},
});
return secureJson({
showcase,
enrichment,
reviewStatus: "awaiting_review",
});
return secureJson(
{ showcase, run, judgeQueueDeferred },
{ status: judgeQueueDeferred ? 202 : 200 },
);
} catch (error) {
return apiErrorResponse(error);
}
Expand Down
28 changes: 27 additions & 1 deletion app/api/uploads/sessions/[sessionId]/complete/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
promoteUploadSessionObjectKey,
} from "@/lib/data/uploads";
import { apiErrorResponse } from "@/lib/http/api";
import { recordShowcaseProcessingFailure } from "@/lib/data/showcase-processing";
import { secureJson } from "@/lib/security/http";
import { enforceRateLimit } from "@/lib/security/rate-limit";
import { scanQuarantinedArtifact } from "@/lib/security/artifact-scanner";
Expand Down Expand Up @@ -99,7 +100,32 @@ export async function POST(
}

const artifact = await finalizeUploadedArtifact({ sessionId: session.id });
const scan = await scanQuarantinedArtifact(artifact);
let scan;
try {
scan = await scanQuarantinedArtifact(artifact);
} catch (error) {
const failure = await recordShowcaseProcessingFailure(
artifact.showcaseId,
error,
);
if (failure) {
await appendAuditEvent({
actorUserId: user.id,
entityType: "showcase",
entityId: artifact.showcaseId,
action: "showcase.processing_failed",
metadata: { code: failure.code, stage: "artifact_scan" },
}).catch(() => undefined);
}
return secureJson(
{
code: "processing_failed",
error:
"Evidence processing could not finish. Retry processing from your dashboard.",
},
{ status: 503 },
);
}
await appendAuditEvent({
actorUserId: user.id,
entityType: "artifact",
Expand Down
23 changes: 20 additions & 3 deletions app/components/ShowcaseCard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,21 @@ export function ShowcaseCard({ showcase }: { showcase: Showcase }) {
<div className="card-body">
<div className="card-meta">
<span>{showcase.model}</span>
<span>{showcase.reasoning} reasoning</span>
<span>{showcase.harness}</span>
</div>
<h3>
<Link href={`/results/${showcase.slug}`}>{showcase.title}</Link>
<Link href={`/tests/${showcase.slug}`}>{showcase.title}</Link>
</h3>
<p>{showcase.description}</p>
<small>Declared by contributor — not independently verified</small>
<div className="evidence-list">
{showcase.evidence.map((item) => (
<span key={item}>{item}</span>
))}
</div>
<div className="card-meta">
<span>{showcase.status}</span>
<span>{simpleStatus(showcase.status, showcase.scoreBps)}</span>
<span>{showcase.reasoning} reasoning</span>
{showcase.scoreBps !== null && (
<strong>{(showcase.scoreBps / 100).toFixed(2)}</strong>
)}
Expand All @@ -58,3 +60,18 @@ export function ShowcaseCard({ showcase }: { showcase: Showcase }) {
</article>
);
}

function simpleStatus(status: string, scoreBps: number | null) {
const normalized = status.toLowerCase();
if (normalized === "ranked" || normalized.includes("ranked #")) {
return "Ranked";
}
if (
normalized === "reviewed" ||
scoreBps !== null ||
normalized.includes("scored")
) {
return "Reviewed";
}
return "Awaiting review";
}
10 changes: 5 additions & 5 deletions app/components/SiteFooter.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,15 @@ export function SiteFooter() {
<span className="brand-mark">B/</span>
<span>BENCHMAX</span>
</Link>
<p>Real tests. Inspectable evidence. Rankings that earn trust.</p>
<p>Public AI Tests with inspectable prompts, setup, and evidence.</p>
</div>
<div className="footer-links">
<div>
<span>PRODUCT</span>
<Link href="/explore">Explore</Link>
<Link href="/tests">Tests</Link>
<Link href="/tests">All Tests</Link>
<Link href="/models">Models</Link>
<Link href="/leaderboards">Leaderboards</Link>
<Link href="/submit">Submit result</Link>
<Link href="/submit">Submit Test</Link>
</div>
<div>
<span>TRUST</span>
Expand All @@ -31,7 +31,7 @@ export function SiteFooter() {
</div>
<div className="footer-base section-wrap">
<span>© 2026 Benchmax</span>
<span>Public methodology · community results v1</span>
<span>Community Tests · declared setup · inspectable evidence</span>
</div>
</footer>
);
Expand Down
18 changes: 6 additions & 12 deletions app/components/SiteHeader.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,33 +14,27 @@ export function SiteHeader() {
<span>BENCHMAX</span>
</Link>
<nav className="desktop-nav" aria-label="Main navigation">
<Link href="/explore">Explore</Link>
<Link href="/tests">All Tests</Link>
<Link href="/models">Models</Link>
<Link href="/leaderboards">Leaderboards</Link>
<Link href="/models">Model summaries</Link>
<Link href="/tests">Tests</Link>
<Link href="/methodology">Methodology</Link>
</nav>
<div className="header-actions">
<AuthControls configured={authConfigured} />
<Link className="header-run" href="/tests">
Add a test
</Link>
<Link className="header-upload" href="/submit">
Submit result
Submit Test
</Link>
</div>
<details className="mobile-menu">
<summary aria-label="Open navigation">Menu</summary>
<nav aria-label="Mobile navigation">
<Link href="/explore">Explore</Link>
<Link href="/tests">All Tests</Link>
<Link href="/models">Models</Link>
<Link href="/leaderboards">Leaderboards</Link>
<Link href="/models">Model summaries</Link>
<Link href="/tests">Tests</Link>
<Link href="/methodology">Methodology</Link>
<span className="mobile-menu-divider" aria-hidden="true" />
<Link href="/tests">Add a test</Link>
<Link className="mobile-upload" href="/submit">
Submit result
Submit Test
</Link>
<AuthControls configured={authConfigured} />
</nav>
Expand Down
16 changes: 7 additions & 9 deletions app/contributors/[handle]/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,21 @@ export default async function ContributorPage({
<span className="section-index">CONTRIBUTOR</span>
<h1>@{contributor.handle}</h1>
<p>
{contributor.displayName} shares inspectable model test results
and their evidence.
{contributor.displayName} shares public AI Tests with inspectable
prompts, declared setup, and evidence.
</p>
</div>
<dl>
<div>
<dt>Public results</dt>
<dt>Public Tests</dt>
<dd>{results.length}</dd>
</div>
</dl>
</header>
<div className="section-heading compact">
<div>
<span className="section-index">PUBLIC RECORD</span>
<h2>Submitted results</h2>
<h2>Submitted Tests</h2>
</div>
</div>
{results.length > 0 ? (
Expand All @@ -68,18 +68,16 @@ export default async function ContributorPage({
</div>
) : publicResultsPage === null ? (
<div className="empty-state">
<strong>Public results are temporarily unavailable.</strong>
<strong>Public Tests are temporarily unavailable.</strong>
<p>
Benchmax does not show substitute data when this contributor’s
public records cannot be read.
</p>
</div>
) : (
<div className="empty-state">
<strong>No public results yet.</strong>
<p>
This active contributor has not published a model test result.
</p>
<strong>No public Tests yet.</strong>
<p>This contributor has not published a Test.</p>
</div>
)}
</main>
Expand Down
Loading
Loading