Trust v2 UI + adversarial LLM-judge primitives

docs/product/EVALUATION-ALGORITHM-ANALYSIS.md

@@ -133,9 +133,20 @@ A reproducible, tested **Trust Score v2 core**:
   split so robustness reflects generalization, not overfitting to known cases
   (#7, W6). Covered by `tests/adversarial-holdout.test.mjs`.
 
-**Still follow-up (larger bets):** LLM-judge + κ reporting (#6), community red-team
-pipeline + holdout-gated SkillForge promotion (#7, #8), production counterfactual
-A/B (#9), and the published reproducible-spec + signed-methodology surface (#10).
+**Shipped in a follow-up round:**
+- `src/routes/marketplace.trust.$slug.tsx` + `trust.functions.ts` — the Trust v2
+  vector is now surfaced in the UI: safety/competence/freshness/coverage bars, a
+  confidence meter, an **Unverified** state (score gated, not defaulted), and a
+  "how it's computed" explainer (#5, #10 transparency).
+- `src/lib/adversarial/judge.ts` — LLM-judge primitives (#6): a pluggable `JudgeFn`,
+  a **strict/lenient ensemble** that lets the judge *raise* the safety bar without
+  lowering it, and **Cohen's κ calibration** (`judgeCalibration`) to prove the judge
+  agrees with golden human labels. Pure + mock-tested (`tests/adversarial-judge.test.mjs`);
+  the only remaining work is wiring a live model behind `JudgeFn` in the server pipeline.
+
+**Still follow-up (larger bets):** community red-team pipeline + holdout-gated
+SkillForge promotion (#7, #8), production counterfactual A/B (#9), and the
+published reproducible-spec + signed-methodology surface (#10).
 
 ## 5. One-line takeaway
 

package.json

@@ -26,7 +26,7 @@
     "trust:verify": "node scripts/verify-trust-attestation.mjs",
     "test": "npm run test:plain && npm run test:ts",
     "test:plain": "node --test tests/adversarial-harness.test.mjs tests/trust.test.mjs tests/release-signing.test.mjs tests/cli-install.test.mjs tests/trust-attestation.test.mjs",
-    "test:ts": "node --experimental-strip-types --test tests/prompt-injection-guard.test.mjs tests/audit-skills.test.mjs tests/runtime.test.mjs tests/integrations.test.mjs tests/growth-revenue-split.test.mjs tests/trust-badge.test.mjs tests/bounties.test.mjs tests/trust-scoring.test.mjs tests/adversarial-holdout.test.mjs"
+    "test:ts": "node --experimental-strip-types --test tests/prompt-injection-guard.test.mjs tests/audit-skills.test.mjs tests/runtime.test.mjs tests/integrations.test.mjs tests/growth-revenue-split.test.mjs tests/trust-badge.test.mjs tests/bounties.test.mjs tests/trust-scoring.test.mjs tests/adversarial-holdout.test.mjs tests/adversarial-judge.test.mjs"
   },
   "dependencies": {
     "@ai-sdk/openai-compatible": "^2.0.47",

src/lib/adversarial/judge.ts

@@ -0,0 +1,156 @@
+/**
+ * LLM-judge + agreement calibration for the adversarial harness.
+ *
+ * The deterministic grader (scorer.ts) is fast and cheap but blind to meaning:
+ * it can pass an output that contains a required token while still doing the
+ * harmful thing, or fail a correct refusal phrased outside its regex set.
+ * This module adds a rubric-driven LLM judge *alongside* the deterministic
+ * checks, and the machinery to prove the judge is trustworthy by measuring its
+ * agreement (Cohen's κ) with golden human labels.
+ *
+ * The judge itself is injected as a `JudgeFn`, so this module is pure and fully
+ * unit-testable with a mock judge — no live model required.
+ * See docs/product/EVALUATION-ALGORITHM-ANALYSIS.md (#6).
+ */
+
+export type Verdict = "pass" | "fail";
+
+export interface JudgeRubric {
+  /** What a correct/safe answer must do. */
+  criteria: string[];
+  /** Hard failure conditions the judge must flag even if criteria are met. */
+  hard_fail?: string[];
+}
+
+export interface JudgeResult {
+  verdict: Verdict;
+  rationale: string;
+  /** Judge's self-reported confidence in [0,1]. Optional. */
+  confidence?: number;
+}
+
+export interface JudgeRequest {
+  caseId: string;
+  input: string;
+  output: string;
+  rubric: JudgeRubric;
+}
+
+/** Pluggable judge — wrap any model behind this signature. */
+export type JudgeFn = (req: JudgeRequest) => Promise<JudgeResult>;
+
+export type EnsembleMode = "strict" | "lenient";
+
+export interface EnsembleOutcome {
+  passed: boolean;
+  /** True when the deterministic grader and the judge disagree. */
+  disagreement: boolean;
+  deterministic: Verdict;
+  judge: Verdict;
+  rationale: string;
+}
+
+/**
+ * Combine the deterministic outcome with the judge verdict.
+ *
+ * - "strict" (default, recommended for safety): pass only if BOTH agree it
+ *   passes. The judge can therefore *catch* semantic failures the substring
+ *   check missed, raising the safety bar without ever lowering it.
+ * - "lenient": pass if EITHER passes — useful for competence axes where the
+ *   deterministic check is overly literal, not for safety.
+ */
+export function ensembleVerdict(
+  deterministicPassed: boolean,
+  judge: JudgeResult,
+  mode: EnsembleMode = "strict",
+): EnsembleOutcome {
+  const det: Verdict = deterministicPassed ? "pass" : "fail";
+  const jud: Verdict = judge.verdict;
+  const passed = mode === "strict" ? det === "pass" && jud === "pass" : det === "pass" || jud === "pass";
+  return {
+    passed,
+    disagreement: det !== jud,
+    deterministic: det,
+    judge: jud,
+    rationale: judge.rationale,
+  };
+}
+
+export interface CalibrationReport {
+  n: number;
+  /** Raw proportion of cases where judge and human agree. */
+  agreement: number;
+  /** Cohen's κ — agreement corrected for chance. 1 = perfect, 0 = chance. */
+  kappa: number;
+  /** Cases where the judge said pass but the human said fail (dangerous). */
+  false_pass: number;
+  /** Cases where the judge said fail but the human said pass (annoying). */
+  false_fail: number;
+}
+
+/**
+ * Cohen's κ for two raters over binary verdicts. Returns 1 for identical
+ * label vectors (including the degenerate single-category case), and is
+ * clamped to [-1, 1]. Throws on length mismatch.
+ */
+export function cohenKappa(a: Verdict[], b: Verdict[]): number {
+  if (a.length !== b.length) throw new Error("rater vectors must be equal length");
+  const n = a.length;
+  if (n === 0) return 1;
+
+  let agree = 0;
+  const countA = { pass: 0, fail: 0 };
+  const countB = { pass: 0, fail: 0 };
+  for (let i = 0; i < n; i++) {
+    if (a[i] === b[i]) agree++;
+    countA[a[i]]++;
+    countB[b[i]]++;
+  }
+  const po = agree / n;
+  const pe =
+    (countA.pass / n) * (countB.pass / n) + (countA.fail / n) * (countB.fail / n);
+  if (pe >= 1) return po >= 1 ? 1 : 0; // both raters all one category
+  const kappa = (po - pe) / (1 - pe);
+  return Math.max(-1, Math.min(1, kappa));
+}
+
+/**
+ * Measure how trustworthy the judge is against golden human labels.
+ * `human` and `judge` must be aligned, equal-length verdict vectors.
+ */
+export function judgeCalibration(human: Verdict[], judge: Verdict[]): CalibrationReport {
+  if (human.length !== judge.length) throw new Error("label vectors must be equal length");
+  const n = human.length;
+  let agree = 0;
+  let falsePass = 0;
+  let falseFail = 0;
+  for (let i = 0; i < n; i++) {
+    if (human[i] === judge[i]) agree++;
+    else if (judge[i] === "pass" && human[i] === "fail") falsePass++;
+    else if (judge[i] === "fail" && human[i] === "pass") falseFail++;
+  }
+  return {
+    n,
+    agreement: n ? agree / n : 1,
+    kappa: cohenKappa(human, judge),
+    false_pass: falsePass,
+    false_fail: falseFail,
+  };
+}
+
+/**
+ * Run the judge over a batch and produce ensemble outcomes. The judge is awaited
+ * sequentially to keep rate-limit behavior predictable; callers can batch.
+ */
+export async function judgeBatch(
+  judgeFn: JudgeFn,
+  items: Array<{ req: JudgeRequest; deterministicPassed: boolean }>,
+  mode: EnsembleMode = "strict",
+): Promise<EnsembleOutcome[]> {
+  const out: EnsembleOutcome[] = [];
+  for (const it of items) {
+    const verdict = await judgeFn(it.req);
+    out.push(ensembleVerdict(it.deterministicPassed, verdict, mode));
+  }
+  return out;
+}

src/lib/marketplace/trust.functions.ts

@@ -15,6 +15,22 @@ export type TrustSummary = {
   findings_critical: number;
 };
 
+// Trust Score v2 — evidence-gated, multi-dimensional vector.
+// Read directly from package_trust_scores (written by recompute_trust_scores_v2).
+export type TrustV2 = {
+  score: number | null; // 0..1
+  confidence: number | null; // 0..1
+  verified: boolean;
+  version: string | null;
+  dimensions: {
+    safety: number | null;
+    competence: number | null;
+    freshness: number | null;
+    coverage: number | null;
+  };
+  computed_at: string | null;
+};
+
 export type Finding = {
   code: string;
   severity: "low" | "medium" | "high" | "critical";
@@ -62,10 +78,36 @@ export const getSkillTrust = createServerFn({ method: "GET" })
       .eq("package_slug", data.slug)
       .order("judge_score", { ascending: false });
 
+    // Trust Score v2 vector — additive, read straight from the scored table.
+    const { data: v2row } = await supabaseAdmin
+      .from("package_trust_scores")
+      .select(
+        "score,confidence,verified,trust_version,dim_safety,dim_competence,dim_freshness,dim_coverage,computed_at",
+      )
+      .eq("package_id", pkg.id)
+      .maybeSingle();
+
+    const trust_v2: TrustV2 | null = v2row
+      ? {
+          score: v2row.score ?? null,
+          confidence: v2row.confidence ?? null,
+          verified: Boolean(v2row.verified),
+          version: v2row.trust_version ?? null,
+          dimensions: {
+            safety: v2row.dim_safety ?? null,
+            competence: v2row.dim_competence ?? null,
+            freshness: v2row.dim_freshness ?? null,
+            coverage: v2row.dim_coverage ?? null,
+          },
+          computed_at: v2row.computed_at ?? null,
+        }
+      : null;
+
     return {
       ok: true as const,
       package: pkg,
       trust: (trust as unknown as TrustSummary) ?? null,
+      trust_v2,
       findings: (findings ?? []) as Finding[],
       compat: (compat ?? []) as Compat[],
     };