diff --git a/packages/shared/src/research/claim-verifier.test.ts b/packages/shared/src/research/claim-verifier.test.ts index 4598a13..ba0ee7b 100644 --- a/packages/shared/src/research/claim-verifier.test.ts +++ b/packages/shared/src/research/claim-verifier.test.ts @@ -3,9 +3,9 @@ import type { JudgeClient } from '../evaluation/judge-client.ts'; import { CLAIM_VERIFIER_VERSION, createClaimVerifier, + verifyReportClaims, type ClaimVerificationInput, } from './claim-verifier.ts'; - function input( claimText: string, evidence: ClaimVerificationInput['evidence'], @@ -143,4 +143,80 @@ describe('claim verifier', () => { expect(result.evidenceIds).toEqual(['source-evidence']); expect(result.evidenceIds).not.toContain('Explanation only.'); }); + + it('verifies all claims across research report sections (mock judge)', async () => { + const judge = judgeReturning('{"status":"supported","reason":"Confirmed by filing excerpt."}'); + const verifier = createClaimVerifier(judge); + const report = { + id: 'report-123', + sections: [ + { + evidence: [ + { claim: 'Revenue rose 10%', runId: 'sec-edgar-run-1', summary: 'SEC 10-Q filing notes 10% revenue increase' }, + { claim: 'Gross margin expanded', runId: 'run-2', summary: 'Margin up 150bps' }, + ], + }, + { + evidence: [ + { claim: 'Revenue rose 10%', runId: 'sec-edgar-run-1', summary: 'Same claim in conclusion' }, + ], + }, + ], + }; + const summary = await verifyReportClaims(report, verifier); + + expect(summary.reportId).toBe('report-123'); + expect(summary.totalClaims).toBe(2); // Deduplicated identical claim statement + expect(summary.supported).toBe(2); + expect(summary.contradicted).toBe(0); + expect(summary.insufficientEvidence).toBe(0); + expect(summary.results).toHaveLength(2); + for (const r of summary.results) { + expect(r.claimId.startsWith('claim_')).toBe(true); + for (const evId of r.evidenceIds) { + expect(evId.startsWith('ev_')).toBe(true); + } + } + }); + + it('keeps claimId and evidenceId stable when unrelated sections or claims are inserted/reordered', async () => { + const judge = judgeReturning('{"status":"supported","reason":"Confirmed."}'); + const verifier = createClaimVerifier(judge); + + const baseReport = { + id: 'rep-1', + sections: [ + { + evidence: [ + { claim: 'Free cash flow increased', runId: 'fcf-run', summary: 'FCF up 20% YoY' }, + ], + }, + ], + }; + + const prependedReport = { + id: 'rep-1', + sections: [ + { + evidence: [ + { claim: 'Unrelated intro fact', runId: 'intro-run', summary: 'Company founded in 2010' }, + ], + }, + { + evidence: [ + { claim: 'Free cash flow increased', runId: 'fcf-run', summary: 'FCF up 20% YoY' }, + ], + }, + ], + }; + + const baseSummary = await verifyReportClaims(baseReport, verifier); + const prependedSummary = await verifyReportClaims(prependedReport, verifier); + + const baseClaim = baseSummary.results.find((r) => r.evidenceIds.length > 0)!; + const matchedPrependedClaim = prependedSummary.results.find((r) => r.claimId === baseClaim.claimId); + + expect(matchedPrependedClaim).toBeDefined(); + expect(matchedPrependedClaim?.evidenceIds).toEqual(baseClaim.evidenceIds); + }); }); diff --git a/packages/shared/src/research/claim-verifier.ts b/packages/shared/src/research/claim-verifier.ts index eb2afc7..53413a4 100644 --- a/packages/shared/src/research/claim-verifier.ts +++ b/packages/shared/src/research/claim-verifier.ts @@ -1,12 +1,14 @@ +import { createHash } from 'node:crypto'; import type { JudgeClient } from '../evaluation/judge-client.ts'; +import { createCodeError } from '../agent/errors.ts'; +import { redact } from '../diagnostics/redact.ts'; -export const CLAIM_VERIFIER_VERSION = 'claim-verifier-v1'; +export const CLAIM_VERIFIER_VERSION = '1.0.0'; export type ClaimVerificationStatus = | 'supported' | 'contradicted' | 'insufficient_evidence'; - export interface ClaimVerifierEvidence { id: string; content: string; @@ -32,6 +34,87 @@ export interface ClaimVerifier { verify(input: ClaimVerificationInput, signal?: AbortSignal): Promise; } +export interface ReportClaimVerificationSummary { + reportId: string; + verifiedAt: number; + totalClaims: number; + supported: number; + contradicted: number; + insufficientEvidence: number; + results: ClaimVerificationResult[]; +} + +/** + * Stable claim id derived deterministically from the statement and report scope (Issue #102). + * Insertion of unrelated claims before or after will not shift existing IDs. + */ +function deriveStableClaimId(statement: string, reportId?: string): string { + const hash = createHash('sha256').update([statement.trim(), reportId ?? ''].join('|')).digest('hex'); + return `claim_${hash.slice(0, 24)}`; +} + +/** + * Stable evidence id derived deterministically from evidence content and summary (Issue #102). + */ +function deriveStableEvidenceId(content: string, runId?: string): string { + const hash = createHash('sha256').update([content.trim(), runId ?? ''].join('|')).digest('hex'); + return `ev_${hash.slice(0, 24)}`; +} + +/** + * Verify all atomic claims within a ResearchReport against their associated evidence. + * Extracts EvidenceRef claims across all sections and checks each using the ClaimVerifier. + */ +export async function verifyReportClaims( + report: { id: string; sections?: Array<{ evidence?: Array<{ claim: string; runId?: string; summary?: string }> }> }, + verifier: ClaimVerifier, + signal?: AbortSignal +): Promise { + const seenClaims = new Map }>(); + for (const section of report.sections ?? []) { + for (const ref of section.evidence ?? []) { + if (!ref.claim) continue; + const claimText = ref.claim; + if (!seenClaims.has(claimText)) { + seenClaims.set(claimText, { + claimId: deriveStableClaimId(claimText, report.id), + evidenceMap: new Map(), + }); + } + const entry = seenClaims.get(claimText)!; + if (ref.summary) { + const evId = deriveStableEvidenceId(ref.summary, ref.runId); + if (!entry.evidenceMap.has(evId)) { + entry.evidenceMap.set(evId, { id: evId, content: ref.summary }); + } + } + } + } + + const results: ClaimVerificationResult[] = []; + let supported = 0; + let contradicted = 0; + let insufficientEvidence = 0; + + for (const [claimText, { claimId, evidenceMap }] of seenClaims.entries()) { + const evidenceList = [...evidenceMap.values()]; + const res = await verifier.verify({ claimId, claimText, evidence: evidenceList }, signal); + results.push(res); + if (res.status === 'supported') supported += 1; + else if (res.status === 'contradicted') contradicted += 1; + else insufficientEvidence += 1; + } + return { + reportId: report.id, + verifiedAt: Date.now(), + totalClaims: results.length, + supported, + contradicted, + insufficientEvidence, + results, + }; +} + interface JudgeResult { status: ClaimVerificationStatus; reason: string; diff --git a/packages/shared/src/research/index.ts b/packages/shared/src/research/index.ts index 2b07154..15359e4 100644 --- a/packages/shared/src/research/index.ts +++ b/packages/shared/src/research/index.ts @@ -21,3 +21,14 @@ export { type SanitizePolicy, type SanitizeResult, } from './sanitize.ts'; +export { + CLAIM_VERIFIER_VERSION, + createClaimVerifier, + verifyReportClaims, + type ClaimVerificationInput, + type ClaimVerificationResult, + type ClaimVerificationStatus, + type ClaimVerifier, + type ClaimVerifierEvidence, + type ReportClaimVerificationSummary, +} from './claim-verifier.ts';