From 3cfe07a6fa5e02e1ce952999d7c123ac05054010 Mon Sep 17 00:00:00 2001 From: KKK-keria <294860373+KKK-keria@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:39:44 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E6=8B=92=E7=BB=9D=E6=97=A0=E6=95=88?= =?UTF-8?q?=E7=9A=84=20claim=20verification=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/research/claim-verifier.test.ts | 36 +++++++++++++++++++ .../shared/src/research/claim-verifier.ts | 17 +++++++++ 2 files changed, 53 insertions(+) diff --git a/packages/shared/src/research/claim-verifier.test.ts b/packages/shared/src/research/claim-verifier.test.ts index 4598a138..896c3d97 100644 --- a/packages/shared/src/research/claim-verifier.test.ts +++ b/packages/shared/src/research/claim-verifier.test.ts @@ -41,6 +41,42 @@ describe('claim verifier', () => { expect(judge.calls).toHaveLength(0); }); + it('fails closed without calling the judge when evidence content is blank', async () => { + const judge = judgeReturning('{"status":"supported","reason":"invented"}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue increased.', [{ id: 'e-1', content: ' \n\t' }]), + ); + + expect(result.status).toBe('insufficient_evidence'); + expect(result.reason).toBe('invalid_evidence: evidence content must not be blank (index 0)'); + expect(judge.calls).toHaveLength(0); + }); + + it('fails closed without calling the judge when an evidence ID is blank', async () => { + const judge = judgeReturning('{"status":"supported","reason":"invented"}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue increased.', [{ id: ' ', content: 'Revenue increased.' }]), + ); + + expect(result.status).toBe('insufficient_evidence'); + expect(result.reason).toBe('invalid_evidence: evidence id must not be blank (index 0)'); + expect(judge.calls).toHaveLength(0); + }); + + it('fails closed without calling the judge when valid and invalid evidence are mixed', async () => { + const judge = judgeReturning('{"status":"supported","reason":"invented"}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue increased.', [ + { id: 'e-1', content: 'Revenue increased.' }, + { id: 'e-2', content: '' }, + ]), + ); + + expect(result.status).toBe('insufficient_evidence'); + expect(result.reason).toBe('invalid_evidence: evidence content must not be blank (index 1)'); + expect(judge.calls).toHaveLength(0); + }); + it('returns supported when the supplied evidence directly supports the claim', async () => { const judge = judgeReturning('{"status":"supported","reason":"The filing reports the same revenue increase."}'); const result = await createClaimVerifier(judge).verify( diff --git a/packages/shared/src/research/claim-verifier.ts b/packages/shared/src/research/claim-verifier.ts index eb2afc74..331bff68 100644 --- a/packages/shared/src/research/claim-verifier.ts +++ b/packages/shared/src/research/claim-verifier.ts @@ -98,6 +98,18 @@ function result( }; } +function invalidEvidenceReason(input: ClaimVerificationInput): string | undefined { + for (const [index, item] of input.evidence.entries()) { + if (typeof item.id !== 'string' || item.id.trim() === '') { + return `invalid_evidence: evidence id must not be blank (index ${index})`; + } + if (typeof item.content !== 'string' || item.content.trim() === '') { + return `invalid_evidence: evidence content must not be blank (index ${index})`; + } + } + return undefined; +} + export function createClaimVerifier(client: JudgeClient): ClaimVerifier { return { async verify(input, signal) { @@ -109,6 +121,11 @@ export function createClaimVerifier(client: JudgeClient): ClaimVerifier { ); } + const invalidReason = invalidEvidenceReason(input); + if (invalidReason !== undefined) { + return result(input, 'insufficient_evidence', invalidReason); + } + try { const reply = await client.complete(SYSTEM_PROMPT, userPrompt(input), signal); const parsed = parseJudgeResult(reply);