Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions packages/shared/src/research/claim-verifier.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,42 @@ describe('claim verifier', () => {
expect(judge.calls).toHaveLength(0);
});

it('fails closed without calling the judge when evidence content is blank', async () => {
const judge = judgeReturning('{"status":"supported","reason":"invented"}');
const result = await createClaimVerifier(judge).verify(
input('Revenue increased.', [{ id: 'e-1', content: ' \n\t' }]),
);

expect(result.status).toBe('insufficient_evidence');
expect(result.reason).toBe('invalid_evidence: evidence content must not be blank (index 0)');
expect(judge.calls).toHaveLength(0);
});

it('fails closed without calling the judge when an evidence ID is blank', async () => {
const judge = judgeReturning('{"status":"supported","reason":"invented"}');
const result = await createClaimVerifier(judge).verify(
input('Revenue increased.', [{ id: ' ', content: 'Revenue increased.' }]),
);

expect(result.status).toBe('insufficient_evidence');
expect(result.reason).toBe('invalid_evidence: evidence id must not be blank (index 0)');
expect(judge.calls).toHaveLength(0);
});

it('fails closed without calling the judge when valid and invalid evidence are mixed', async () => {
const judge = judgeReturning('{"status":"supported","reason":"invented"}');
const result = await createClaimVerifier(judge).verify(
input('Revenue increased.', [
{ id: 'e-1', content: 'Revenue increased.' },
{ id: 'e-2', content: '' },
]),
);

expect(result.status).toBe('insufficient_evidence');
expect(result.reason).toBe('invalid_evidence: evidence content must not be blank (index 1)');
expect(judge.calls).toHaveLength(0);
});

it('returns supported when the supplied evidence directly supports the claim', async () => {
const judge = judgeReturning('{"status":"supported","reason":"The filing reports the same revenue increase."}');
const result = await createClaimVerifier(judge).verify(
Expand Down
17 changes: 17 additions & 0 deletions packages/shared/src/research/claim-verifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,18 @@ function result(
};
}

function invalidEvidenceReason(input: ClaimVerificationInput): string | undefined {
for (const [index, item] of input.evidence.entries()) {
if (typeof item.id !== 'string' || item.id.trim() === '') {
return `invalid_evidence: evidence id must not be blank (index ${index})`;
}
if (typeof item.content !== 'string' || item.content.trim() === '') {
return `invalid_evidence: evidence content must not be blank (index ${index})`;
}
}
return undefined;
}

export function createClaimVerifier(client: JudgeClient): ClaimVerifier {
return {
async verify(input, signal) {
Expand All @@ -109,6 +121,11 @@ export function createClaimVerifier(client: JudgeClient): ClaimVerifier {
);
}

const invalidReason = invalidEvidenceReason(input);
if (invalidReason !== undefined) {
return result(input, 'insufficient_evidence', invalidReason);
}

try {
const reply = await client.complete(SYSTEM_PROMPT, userPrompt(input), signal);
const parsed = parseJudgeResult(reply);
Expand Down
Loading