diff --git a/packages/core/src/research.ts b/packages/core/src/research.ts index b9e54f2..db10d21 100644 --- a/packages/core/src/research.ts +++ b/packages/core/src/research.ts @@ -37,8 +37,44 @@ export interface EvidenceRef { summary?: string; /** Canonical instrument id linking this evidence to one listing. */ instrumentId?: string; + /** + * The minimal, immutable source snapshots that were available when this + * evidence was added to the report. They are deliberately separate from a + * later live retrieval: a live response must never overwrite this record. + */ + sourceSnapshots?: SourceProvenanceSnapshot[]; } +/** A privacy-bounded snapshot of the source material used as evidence. */ +export interface SourceProvenanceSnapshot { + /** Versioned so later snapshot formats remain distinguishable on reload. */ + schemaVersion: 'folio-source-provenance/v1'; + /** Stable identity for the document/source, independent of retrieval time. */ + sourceId: string; + /** Provider document identity when one is available. */ + documentId?: string; + /** Canonical HTTP(S) URL, with fragments and tracking parameters removed. */ + canonicalUrl?: string; + title?: string; + author?: string; + /** Epoch milliseconds, when the source published a timestamp. */ + publishedAt?: number; + /** Epoch milliseconds when Folio retrieved this source. */ + retrievedAt: number; + /** Retrieval boundary, e.g. the provider and capability that supplied it. */ + retrieval: { provider: string; method: string }; + /** The smallest source span that was actually handed to the report path. */ + excerpt: string; + excerptHash: string; + /** Hash of the minimal source content used to derive the excerpt. */ + sourceContentHash: string; + /** Version/ETag/revision supplied by the source, when available. */ + sourceVersion?: string; +} + +/** Result of comparing a newly retrieved source with an original snapshot. */ +export type SourceDriftStatus = 'current' | 'source_drifted' | 'source_unavailable'; + /** Condensed outcome of one capability run, embedded in the report. */ export interface CapabilityRunSummary { runId: string; diff --git a/packages/shared/src/research/index.ts b/packages/shared/src/research/index.ts index 2b07154..19264f8 100644 --- a/packages/shared/src/research/index.ts +++ b/packages/shared/src/research/index.ts @@ -10,6 +10,17 @@ export { ResearchRunner, type ResearchRunnerOptions, type ResearchRunRequest, ty export { LocalResearchSynthesizer } from './synthesizer-local.ts'; export { createAgentSynthesizer, parseSynthesisJson, type ResearchAgentRunner } from './agent-synth.ts'; export { ResearchReportRepository, type ReportSummary } from './repository.ts'; +export { + createSourceProvenanceSnapshot, + detectSourceDrift, + hashSourceText, + refreshSourceSnapshot, + snapshotCapabilityEvidence, + type LiveSourceObservation, + type SourceDriftCheck, + type SourceSnapshotInput, + type SourceSnapshotRefresher, +} from './source-provenance.ts'; export { ResearchService, type ResearchServiceOptions } from './service.ts'; export { INJECTION_DEFENSE_RULES, diff --git a/packages/shared/src/research/repository.test.ts b/packages/shared/src/research/repository.test.ts index ba972c6..bd2dfdf 100644 --- a/packages/shared/src/research/repository.test.ts +++ b/packages/shared/src/research/repository.test.ts @@ -59,6 +59,25 @@ describe('ResearchReportRepository', () => { expect(await second.getReport('r1')).toMatchObject({ id: 'r1', symbol: 'NVDA.US' }); }); + it('keeps source provenance snapshots intact after restart', async () => { + const original = report('r1', 'NVDA.US'); + original.sections = [{ + key: 'research.news', title: 'News', verdict: 'neutral', summary: 'Summary', + evidence: [{ + capabilityId: 'research.news', runId: 'cap-1', claim: 'Summary', fetchedAt: 1, + sourceSnapshots: [{ + schemaVersion: 'folio-source-provenance/v1', sourceId: 'src_test', documentId: 'news-1', + retrievedAt: 1, retrieval: { provider: 'test', method: 'capability:research.news' }, + excerpt: 'Original v1 evidence', excerptHash: 'sha256:excerpt', sourceContentHash: 'sha256:content', + }], + }], + }]; + await new ResearchReportRepository(store).saveReport(original); + + const reloaded = await new ResearchReportRepository(new JsonFileStore(dir)).getReport('r1'); + expect(reloaded?.sections[0].evidence[0].sourceSnapshots).toEqual(original.sections[0].evidence[0].sourceSnapshots); + }); + it('lists reports per symbol', async () => { const repo = new ResearchReportRepository(store); await repo.saveReport(report('r1', 'NVDA.US')); diff --git a/packages/shared/src/research/runner.test.ts b/packages/shared/src/research/runner.test.ts index 97ad64f..1d3c3bc 100644 --- a/packages/shared/src/research/runner.test.ts +++ b/packages/shared/src/research/runner.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'bun:test'; -import type { ResearchRunSummary } from '@finagent/core'; +import { Type } from '@sinclair/typebox'; +import type { FinanceCapability, ResearchRunSummary } from '@finagent/core'; import { createCapabilityRegistry } from '../capabilities/index.ts'; import { LocalResearchSynthesizer } from './synthesizer-local.ts'; import { ResearchRunner } from './runner.ts'; @@ -60,6 +61,13 @@ describe('ResearchRunner', () => { for (const ref of section.evidence) { expect(runIds.has(ref.runId)).toBe(true); expect(ref.capabilityId).toBe(section.key); + // Every persisted report evidence ref carries an immutable source + // snapshot; a later live retrieval cannot silently replace it. + expect(ref.sourceSnapshots).toHaveLength(1); + expect(ref.sourceSnapshots![0]).toMatchObject({ + schemaVersion: 'folio-source-provenance/v1', + retrievedAt: 1_700_000_000_000, + }); } } }); @@ -95,6 +103,41 @@ describe('ResearchRunner', () => { expect(result.report!.capabilityRuns).toHaveLength(RESEARCH_CAPABILITY_PLAN.length); }); + it('persists per-article provenance from the real research.news report assembly path', async () => { + const news: FinanceCapability = { + id: 'research.news', name: 'News', description: 'Controlled news source', category: 'research', + riskLevel: 'read', auth: 'public', toolName: 'get_news', inputSchema: Type.Object({ symbol: Type.String() }), + async execute() { + return { + data: [{ + id: 'article-1', title: 'Issuer result', summary: 'Revenue grew 10%.', + url: 'https://news.example.test/result?utm_source=research', timestamp: 1_700_000_000, + symbols: ['NVDA.US'], + }], + provenance: { provider: 'controlled-news', fetchedAt: 1_700_000_001_000, stale: false }, + summary: 'One issuer result.', + }; + }, + }; + const registry = createCapabilityRegistry([ + ...RESEARCH_CAPABILITY_PLAN.filter((id) => id !== 'research.news').map((id) => fakeCap(id)), + news, + ]); + const runner = new ResearchRunner({ + registry, synthesizer: new LocalResearchSynthesizer(), now: () => 1_700_000_002_000, + }); + + const result = await runner.run({ symbol: 'NVDA.US', runId: 'run-news-snapshot' }); + const snapshot = result.report!.sections.find((section) => section.key === 'research.news')!.evidence[0].sourceSnapshots![0]; + + expect(snapshot).toMatchObject({ + documentId: 'article-1', canonicalUrl: 'https://news.example.test/result', + excerpt: 'Issuer result\n\nRevenue grew 10%.', + retrievedAt: 1_700_000_001_000, + retrieval: { provider: 'controlled-news', method: 'capability:research.news' }, + }); + }); + it('fails when no capability succeeds', async () => { const runner = makeRunner([ ['company.profile', 'fail'], diff --git a/packages/shared/src/research/runner.ts b/packages/shared/src/research/runner.ts index 655bddc..e65943b 100644 --- a/packages/shared/src/research/runner.ts +++ b/packages/shared/src/research/runner.ts @@ -24,6 +24,7 @@ import { planForStrategy, type PlannedCapability, } from './planner.ts'; +import { snapshotCapabilityEvidence } from './source-provenance.ts'; const CONCURRENCY = 4; const TIMEOUT_MS = 20000; @@ -321,7 +322,7 @@ function assembleReport(args: { const sections: ResearchSection[] = synthesis.sections.map((section) => { const outcome = outcomeByCapability.get(section.key); const evidence: EvidenceRef[] = []; - if (outcome && outcome.record.status === 'success') { + if (outcome && outcome.record.status === 'success' && outcome.result) { const instrumentId = outcome.result?.provenance?.instrumentId ?? readInstrumentId(outcome.result?.data); evidence.push({ @@ -331,6 +332,11 @@ function assembleReport(args: { fetchedAt: outcome.record.provenance?.fetchedAt ?? generatedAt, summary: outcome.result?.summary, ...(instrumentId ? { instrumentId } : {}), + sourceSnapshots: snapshotCapabilityEvidence({ + capabilityId: outcome.record.capabilityId, + result: outcome.result, + retrievedAt: outcome.record.provenance?.fetchedAt ?? generatedAt, + }), }); } return { ...section, evidence }; diff --git a/packages/shared/src/research/source-provenance.test.ts b/packages/shared/src/research/source-provenance.test.ts new file mode 100644 index 0000000..62bef25 --- /dev/null +++ b/packages/shared/src/research/source-provenance.test.ts @@ -0,0 +1,119 @@ +import { afterEach, describe, expect, it } from 'bun:test'; +import type { CapabilityResult } from '@finagent/core'; +import { + createSourceProvenanceSnapshot, + detectSourceDrift, + refreshSourceSnapshot, + snapshotCapabilityEvidence, +} from './source-provenance.ts'; + +describe('source provenance snapshots', () => { + it('uses a stable source identity while preserving only the minimum evidence span', () => { + const first = createSourceProvenanceSnapshot({ + canonicalUrl: 'https://example.test/article?utm_source=mail&a=1#section-2', + title: 'Quarterly update', + retrievedAt: 1_700_000_000_000, + provider: 'test-news', + method: 'search', + excerpt: 'Revenue rose 10%.', + sourceContent: 'Quarterly update\nRevenue rose 10%.', + sourceVersion: 'v1', + }); + const second = createSourceProvenanceSnapshot({ + canonicalUrl: 'https://example.test/article?a=1', + title: 'Quarterly update', + retrievedAt: 1_800_000_000_000, + provider: 'test-news', + method: 'search', + excerpt: 'Revenue rose 10%.', + sourceContent: 'Quarterly update\nRevenue rose 10%.', + sourceVersion: 'v1', + }); + + expect(first.sourceId).toBe(second.sourceId); + expect(first.canonicalUrl).toBe('https://example.test/article?a=1'); + expect(first).not.toHaveProperty('sourceContent'); + expect(first.excerptHash).toMatch(/^sha256:/); + expect(first.sourceContentHash).toMatch(/^sha256:/); + }); + + it('snapshots each news source that actually reaches the research report path', () => { + const result: CapabilityResult = { + data: [{ + id: 'wire-42', + title: 'Issuer publishes results', + summary: 'Revenue grew 10%.', + url: 'https://news.example.test/results?utm_campaign=weekly', + timestamp: 1_700_000_000, + symbols: ['NVDA.US'], + }], + provenance: { provider: 'test-wire', fetchedAt: 1_700_000_001_000, stale: false }, + }; + + const snapshots = snapshotCapabilityEvidence({ + capabilityId: 'research.news', result, retrievedAt: 1_700_000_001_000, + }); + + expect(snapshots).toEqual([expect.objectContaining({ + documentId: 'wire-42', + canonicalUrl: 'https://news.example.test/results', + title: 'Issuer publishes results', + publishedAt: 1_700_000_000_000, + retrievedAt: 1_700_000_001_000, + retrieval: { provider: 'test-wire', method: 'capability:research.news' }, + excerpt: 'Issuer publishes results\n\nRevenue grew 10%.', + })]); + }); + + it('reports v1 to v2 content changes and unavailable sources without replacing original evidence', async () => { + let source = 'v1: revenue grew 10%.'; + let online = true; + const server = Bun.serve({ + port: 0, + fetch() { + return online ? new Response(source) : new Response('gone', { status: 410 }); + }, + }); + try { + const original = createSourceProvenanceSnapshot({ + canonicalUrl: `${server.url}article`, + title: 'Issuer update', + retrievedAt: 1_700_000_000_000, + provider: 'controlled-test-server', + method: 'deep-research-retrieval', + excerpt: source, + sourceContent: source, + sourceVersion: 'v1', + }); + const retrieve = async () => { + const response = await fetch(`${server.url}article`); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const text = await response.text(); + return { available: true as const, canonicalUrl: `${server.url}article`, excerpt: text, sourceContent: text, + sourceVersion: text.startsWith('v2') ? 'v2' : 'v1' }; + }; + + expect(await refreshSourceSnapshot(original, retrieve)).toMatchObject({ status: 'current', reasons: [] }); + + source = 'v2: revenue was restated to 6%.'; + const drifted = await refreshSourceSnapshot(original, retrieve); + expect(drifted).toMatchObject({ status: 'source_drifted' }); + expect(drifted.reasons).toEqual(['content_changed', 'version_changed']); + expect(drifted.original.excerpt).toBe('v1: revenue grew 10%.'); + + online = false; + expect(await refreshSourceSnapshot(original, retrieve)).toMatchObject({ + status: 'source_unavailable', reasons: ['unavailable'], original: { excerpt: 'v1: revenue grew 10%.' }, + }); + } finally { + await server.stop(true); + } + }); + + it('does not call an inaccessible live source current', () => { + const original = createSourceProvenanceSnapshot({ + documentId: 'doc-1', retrievedAt: 1, provider: 'test', method: 'search', excerpt: 'original', sourceContent: 'original', + }); + expect(detectSourceDrift(original, { available: false }).status).toBe('source_unavailable'); + }); +}); diff --git a/packages/shared/src/research/source-provenance.ts b/packages/shared/src/research/source-provenance.ts new file mode 100644 index 0000000..fd8e13d --- /dev/null +++ b/packages/shared/src/research/source-provenance.ts @@ -0,0 +1,234 @@ +import { createHash } from 'node:crypto'; +import type { + CapabilityResult, + NewsItem, + SourceDriftStatus, + SourceProvenanceSnapshot, +} from '@finagent/core'; + +const SCHEMA_VERSION = 'folio-source-provenance/v1' as const; + +export interface SourceSnapshotInput { + documentId?: string; + canonicalUrl?: string; + title?: string; + author?: string; + publishedAt?: number; + retrievedAt: number; + provider: string; + method: string; + /** The exact source span supplied to the report/evidence path. */ + excerpt: string; + /** + * The smallest source content from which `excerpt` was selected. It is + * hashed but intentionally never persisted, so a report is auditable without + * becoming a general web archive. + */ + sourceContent?: string; + sourceVersion?: string; +} + +export interface LiveSourceObservation { + available: boolean; + canonicalUrl?: string; + title?: string; + author?: string; + publishedAt?: number; + excerpt?: string; + sourceContent?: string; + sourceVersion?: string; +} + +export interface SourceDriftCheck { + status: SourceDriftStatus; + original: SourceProvenanceSnapshot; + /** Present only when a current source was successfully retrieved. */ + current?: { + canonicalUrl?: string; + excerptHash: string; + sourceContentHash: string; + sourceVersion?: string; + }; + reasons: Array<'content_changed' | 'version_changed' | 'canonical_url_changed' | 'unavailable'>; +} + +/** A privacy-policy-aware live retrieval adapter, supplied by the caller. */ +export type SourceSnapshotRefresher = ( + original: Readonly +) => Promise; + +/** + * Create the durable, minimum-necessary provenance record for one source. + * `sourceId` excludes retrieval time; observations of the same source can + * therefore be compared even after an application restart. + */ +export function createSourceProvenanceSnapshot(input: SourceSnapshotInput): SourceProvenanceSnapshot { + const canonicalUrl = canonicalizeUrl(input.canonicalUrl); + const documentId = cleanOptional(input.documentId); + const title = cleanOptional(input.title); + const author = cleanOptional(input.author); + const provider = cleanRequired(input.provider, 'unknown'); + const method = cleanRequired(input.method, 'unknown'); + const excerpt = cleanText(input.excerpt); + const sourceContent = cleanText(input.sourceContent ?? excerpt); + const sourceIdentity = documentId + ?? canonicalUrl + ?? `${provider}:${title ?? hashText(excerpt)}`; + + return { + schemaVersion: SCHEMA_VERSION, + sourceId: `src_${hashText(`${provider}:${sourceIdentity}`).slice(0, 24)}`, + ...(documentId ? { documentId } : {}), + ...(canonicalUrl ? { canonicalUrl } : {}), + ...(title ? { title } : {}), + ...(author ? { author } : {}), + ...(validTimestamp(input.publishedAt) ? { publishedAt: input.publishedAt } : {}), + retrievedAt: validTimestamp(input.retrievedAt) ? input.retrievedAt : 0, + retrieval: { provider, method }, + excerpt, + excerptHash: hashSourceText(excerpt), + sourceContentHash: hashSourceText(sourceContent), + ...(cleanOptional(input.sourceVersion) ? { sourceVersion: cleanOptional(input.sourceVersion) } : {}), + }; +} + +/** + * Snapshot every report evidence source without storing an entire page or raw + * capability response. News keeps one snapshot per retrieved article; other + * capability results retain the exact result summary that reached the report. + */ +export function snapshotCapabilityEvidence(input: { + capabilityId: string; + result: CapabilityResult; + retrievedAt: number; +}): SourceProvenanceSnapshot[] { + const provider = cleanRequired(input.result.provenance.providerId ?? input.result.provenance.provider, 'unknown'); + if (input.capabilityId === 'research.news' && isNewsItems(input.result.data)) { + const snapshots = input.result.data.map((item) => createSourceProvenanceSnapshot({ + documentId: item.id, + canonicalUrl: item.url, + title: item.title, + publishedAt: toEpochMs(item.timestamp), + retrievedAt: input.retrievedAt, + provider, + method: 'capability:research.news', + excerpt: newsExcerpt(item), + sourceContent: newsExcerpt(item), + })); + if (snapshots.length > 0) return snapshots; + } + + const excerpt = cleanText(input.result.summary ?? `${input.capabilityId} completed.`); + return [createSourceProvenanceSnapshot({ + documentId: `capability:${input.capabilityId}:${input.result.provenance.instrumentId ?? 'global'}`, + title: input.capabilityId, + retrievedAt: input.retrievedAt, + provider, + method: `capability:${input.capabilityId}`, + excerpt, + sourceContent: excerpt, + })]; +} + +/** Compare a newly retrieved source against the immutable report snapshot. */ +export function detectSourceDrift( + original: SourceProvenanceSnapshot, + live: LiveSourceObservation +): SourceDriftCheck { + if (!live.available) { + return { status: 'source_unavailable', original, reasons: ['unavailable'] }; + } + + const canonicalUrl = canonicalizeUrl(live.canonicalUrl); + const excerpt = cleanText(live.excerpt ?? ''); + const sourceContent = cleanText(live.sourceContent ?? excerpt); + const current = { + ...(canonicalUrl ? { canonicalUrl } : {}), + excerptHash: hashSourceText(excerpt), + sourceContentHash: hashSourceText(sourceContent), + ...(cleanOptional(live.sourceVersion) ? { sourceVersion: cleanOptional(live.sourceVersion) } : {}), + }; + const reasons: SourceDriftCheck['reasons'] = []; + if (current.sourceContentHash !== original.sourceContentHash) reasons.push('content_changed'); + if (original.sourceVersion && current.sourceVersion && original.sourceVersion !== current.sourceVersion) { + reasons.push('version_changed'); + } + if (original.canonicalUrl && current.canonicalUrl && original.canonicalUrl !== current.canonicalUrl) { + reasons.push('canonical_url_changed'); + } + return { status: reasons.length > 0 ? 'source_drifted' : 'current', original, current, reasons }; +} + +/** + * Convert retrieval failures into an explicit unavailable state. Callers own + * networking and privacy/redaction policy; this helper never fetches URLs. + */ +export async function refreshSourceSnapshot( + original: SourceProvenanceSnapshot, + refresh: SourceSnapshotRefresher +): Promise { + try { + return detectSourceDrift(original, await refresh(original)); + } catch { + return detectSourceDrift(original, { available: false }); + } +} + +export function hashSourceText(value: string): string { + return `sha256:${hashText(cleanText(value))}`; +} + +function canonicalizeUrl(value: string | undefined): string | undefined { + const candidate = cleanOptional(value); + if (!candidate) return undefined; + try { + const url = new URL(candidate); + if (!/^https?:$/.test(url.protocol)) return undefined; + url.hash = ''; + for (const key of [...url.searchParams.keys()]) { + if (/^utm_/i.test(key) || /^(fbclid|gclid)$/i.test(key)) url.searchParams.delete(key); + } + url.searchParams.sort(); + return url.toString(); + } catch { + return undefined; + } +} + +function newsExcerpt(item: NewsItem): string { + return cleanText([item.title, item.summary].filter(Boolean).join('\n\n')); +} + +function isNewsItems(value: unknown): value is NewsItem[] { + return Array.isArray(value) && value.every((item) => item && typeof item === 'object' + && typeof (item as NewsItem).id === 'string' + && typeof (item as NewsItem).title === 'string' + && typeof (item as NewsItem).summary === 'string' + && typeof (item as NewsItem).url === 'string' + && typeof (item as NewsItem).timestamp === 'number'); +} + +function toEpochMs(value: number): number | undefined { + return Number.isFinite(value) ? value * 1000 : undefined; +} + +function validTimestamp(value: number | undefined): value is number { + return typeof value === 'number' && Number.isFinite(value) && value >= 0; +} + +function cleanText(value: string): string { + return String(value ?? '').replace(/\r\n?/g, '\n').trim(); +} + +function cleanOptional(value: string | undefined): string | undefined { + const cleaned = value === undefined ? undefined : cleanText(value); + return cleaned || undefined; +} + +function cleanRequired(value: string | undefined, fallback: string): string { + return cleanOptional(value) ?? fallback; +} + +function hashText(value: string): string { + return createHash('sha256').update(value).digest('hex'); +}