diff --git a/packages/shared/src/research/claim-verifier.test.ts b/packages/shared/src/research/claim-verifier.test.ts index 4598a13..896c3d9 100644 --- a/packages/shared/src/research/claim-verifier.test.ts +++ b/packages/shared/src/research/claim-verifier.test.ts @@ -41,6 +41,42 @@ describe('claim verifier', () => { expect(judge.calls).toHaveLength(0); }); + it('fails closed without calling the judge when evidence content is blank', async () => { + const judge = judgeReturning('{"status":"supported","reason":"invented"}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue increased.', [{ id: 'e-1', content: ' \n\t' }]), + ); + + expect(result.status).toBe('insufficient_evidence'); + expect(result.reason).toBe('invalid_evidence: evidence content must not be blank (index 0)'); + expect(judge.calls).toHaveLength(0); + }); + + it('fails closed without calling the judge when an evidence ID is blank', async () => { + const judge = judgeReturning('{"status":"supported","reason":"invented"}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue increased.', [{ id: ' ', content: 'Revenue increased.' }]), + ); + + expect(result.status).toBe('insufficient_evidence'); + expect(result.reason).toBe('invalid_evidence: evidence id must not be blank (index 0)'); + expect(judge.calls).toHaveLength(0); + }); + + it('fails closed without calling the judge when valid and invalid evidence are mixed', async () => { + const judge = judgeReturning('{"status":"supported","reason":"invented"}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue increased.', [ + { id: 'e-1', content: 'Revenue increased.' }, + { id: 'e-2', content: '' }, + ]), + ); + + expect(result.status).toBe('insufficient_evidence'); + expect(result.reason).toBe('invalid_evidence: evidence content must not be blank (index 1)'); + expect(judge.calls).toHaveLength(0); + }); + it('returns supported when the supplied evidence directly supports the claim', async () => { const judge = judgeReturning('{"status":"supported","reason":"The filing reports the same revenue increase."}'); const result = await createClaimVerifier(judge).verify( diff --git a/packages/shared/src/research/claim-verifier.ts b/packages/shared/src/research/claim-verifier.ts index eb2afc7..331bff6 100644 --- a/packages/shared/src/research/claim-verifier.ts +++ b/packages/shared/src/research/claim-verifier.ts @@ -98,6 +98,18 @@ function result( }; } +function invalidEvidenceReason(input: ClaimVerificationInput): string | undefined { + for (const [index, item] of input.evidence.entries()) { + if (typeof item.id !== 'string' || item.id.trim() === '') { + return `invalid_evidence: evidence id must not be blank (index ${index})`; + } + if (typeof item.content !== 'string' || item.content.trim() === '') { + return `invalid_evidence: evidence content must not be blank (index ${index})`; + } + } + return undefined; +} + export function createClaimVerifier(client: JudgeClient): ClaimVerifier { return { async verify(input, signal) { @@ -109,6 +121,11 @@ export function createClaimVerifier(client: JudgeClient): ClaimVerifier { ); } + const invalidReason = invalidEvidenceReason(input); + if (invalidReason !== undefined) { + return result(input, 'insufficient_evidence', invalidReason); + } + try { const reply = await client.complete(SYSTEM_PROMPT, userPrompt(input), signal); const parsed = parseJudgeResult(reply);