diff --git a/README.md b/README.md index 27f3f08..e176490 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,12 @@ what couples tool execution to Helio's liveness, and there is no escape hatch to failed decision." **A Helio outage will halt tool calls mid-conversation.** For local development without a Helio proxy, simply do not enable this plugin. +The same posture covers **misconfiguration**: if the adapter token is missing or the plugin +config is invalid, the adapter still registers — but every tool call and install is blocked, with +a loud error log. It never leaves the tool surface ungoverned (it does not rely on the host +aborting startup), so a config mistake fails closed and visibly rather than silently bypassing +governance. + ## Install Install it into your OpenClaw gateway with OpenClaw's plugin installer (it resolves the package @@ -82,6 +88,30 @@ export HELIO_ADAPTER_TOKEN="" The token is the **adapter-scope** token (`HELIO_ADAPTER_TOKEN`), never the SDK token. The adapter never sends an `Origin` header (Helio's browser-forgery guard rejects it). +### Grounding evidence (optional) + +To let a later [evidence-grounded policy](./docs/adapter-api.md#populating-evidence) enforce on +facts a tool produced, configure **success-only** extraction rules keyed by tool name. Each rule +pulls a value out of `event.result` by an **explicit segment path** (array form, not a dotted +string — a key containing `.` stays unambiguous) and grounds it under an `evidence_key`: + +```jsonc +{ + "evidence": { + "send_email": [ + { "key": "recipient", "path": ["to"], "ttlSeconds": 300 }, + { "key": "message_id", "path": ["id"] }, + ], + "http_request": [{ "key": "host", "path": ["url", "host"] }], + }, +} +``` + +Evidence is attached only on a successful outcome; a rule whose path is absent in the result is +silently skipped (never sent as `undefined`). Each `key` **must** be named by a Helio +`evidence.requires` policy rule — an unlisted key is silently not stored by the proxy, so a later +grounded `/evaluate` would fail closed. + ## How it works | OpenClaw hook | Helio call | Result mapping | diff --git a/openclaw.plugin.json b/openclaw.plugin.json index 6de6d96..a2678b7 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -31,6 +31,37 @@ "minimum": 1, "default": 2000, "description": "Bounded timeout for POST /evaluate. On timeout the tool call fails closed (blocked)." + }, + "evidence": { + "type": "object", + "default": {}, + "description": "Success-only evidence-extraction rules keyed by tool name. Each rule pulls a value from the tool's result and grounds it under an evidence key (which must be named by a Helio evidence.requires policy rule, or it is silently not stored).", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": ["key", "path"], + "additionalProperties": false, + "properties": { + "key": { + "type": "string", + "minLength": 1, + "description": "evidence_key sent to POST /audit; must match a policy evidence.requires key." + }, + "path": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1, + "description": "Explicit segment path into event.result (e.g. [\"url\",\"host\"]). Array form, not dotted, so keys containing '.' are unambiguous. A missing path skips the entry." + }, + "ttlSeconds": { + "type": "integer", + "minimum": 1, + "description": "Optional TTL forwarded as ttl_seconds on the evidence entry." + } + } + } + } } } } diff --git a/src/config.test.ts b/src/config.test.ts new file mode 100644 index 0000000..f09d12c --- /dev/null +++ b/src/config.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest' +import { DEFAULT_CONFIG, parseConfig } from './config.js' + +describe('parseConfig', () => { + it('parses a full valid config including evidence rules', () => { + const result = parseConfig({ + helioBaseUrl: 'http://127.0.0.1:9999', + tokenEnv: 'MY_TOKEN', + origin: 'openclaw-test', + evaluateTimeoutMs: 500, + evidence: { + send_email: [ + { key: 'recipient', path: ['to'], ttlSeconds: 300 }, + { key: 'message_id', path: ['id'] }, + ], + }, + }) + + expect(result).toEqual({ + ok: true, + config: { + helioBaseUrl: 'http://127.0.0.1:9999', + tokenEnv: 'MY_TOKEN', + origin: 'openclaw-test', + evaluateTimeoutMs: 500, + evidence: { + send_email: [ + { key: 'recipient', path: ['to'], ttlSeconds: 300 }, + { key: 'message_id', path: ['id'] }, + ], + }, + }, + }) + }) + + it('applies defaults for an empty config object', () => { + expect(parseConfig({})).toEqual({ ok: true, config: DEFAULT_CONFIG }) + }) + + it('treats undefined (no operator config) as all-defaults', () => { + expect(parseConfig(undefined)).toEqual({ ok: true, config: DEFAULT_CONFIG }) + }) + + it('rejects an origin that violates the manifest pattern', () => { + const result = parseConfig({ origin: 'Not A Valid Origin!' }) + expect(result.ok).toBe(false) + }) + + it('rejects a non-positive or non-integer evaluate timeout', () => { + expect(parseConfig({ evaluateTimeoutMs: 0 }).ok).toBe(false) + expect(parseConfig({ evaluateTimeoutMs: 12.5 }).ok).toBe(false) + }) + + it('rejects an invalid base URL', () => { + expect(parseConfig({ helioBaseUrl: 'not-a-url' }).ok).toBe(false) + }) + + it('rejects an evidence rule with an empty path', () => { + const result = parseConfig({ evidence: { send_email: [{ key: 'recipient', path: [] }] } }) + expect(result.ok).toBe(false) + }) + + it('rejects an evidence rule with an empty key', () => { + const result = parseConfig({ evidence: { send_email: [{ key: '', path: ['to'] }] } }) + expect(result.ok).toBe(false) + }) + + it('rejects an evidence rule carrying an unknown field (mirrors manifest additionalProperties:false)', () => { + const result = parseConfig({ + evidence: { send_email: [{ key: 'recipient', path: ['to'], bogus: 1 }] }, + }) + expect(result.ok).toBe(false) + }) +}) diff --git a/src/config.ts b/src/config.ts index 9b3d521..aa48ee3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,9 +1,27 @@ +import { z } from 'zod' + +/** + * One success-only evidence-extraction rule. `path` addresses a value inside `event.result` by + * explicit segments (e.g. `['url', 'host']`) — array form, not a dotted string, so a key that + * itself contains a `.` is unambiguous and there is no escaping to get wrong. The extracted value + * is sent as `evidence_data` under `evidence_key: key`; a missing path skips the entry. + */ +export interface EvidenceRule { + readonly key: string + readonly path: readonly string[] + readonly ttlSeconds?: number +} + +/** Evidence-extraction rules keyed by tool name (extraction is tool-shape-specific). */ +export type EvidenceConfig = Readonly> + // Resolved adapter configuration. Mirrors the manifest `configSchema` in openclaw.plugin.json. export interface AdapterConfig { helioBaseUrl: string tokenEnv: string origin: string evaluateTimeoutMs: number + evidence: EvidenceConfig } export const DEFAULT_CONFIG: AdapterConfig = { @@ -11,10 +29,73 @@ export const DEFAULT_CONFIG: AdapterConfig = { tokenEnv: 'HELIO_ADAPTER_TOKEN', origin: 'openclaw', evaluateTimeoutMs: 2000, + evidence: {}, } -// TODO: validate the raw plugin config against the manifest schema and resolve the -// bearer token from `process.env[tokenEnv]`. Real validation lands with the config.ts TDD. -export function parseConfig(_raw: unknown): AdapterConfig { - return { ...DEFAULT_CONFIG } +// Runtime validation of operator-supplied plugin config. Mirrors the manifest `configSchema` +// (openclaw.plugin.json) as defense-in-depth — the host may validate too, but the adapter does not +// trust that. Each field defaults to DEFAULT_CONFIG when omitted; a present-but-invalid value fails. +const evidenceRuleSchema = z + .object({ + key: z.string().min(1), + path: z.array(z.string()).min(1), + ttlSeconds: z.number().int().min(1).optional(), + }) + .strict() + +const adapterConfigSchema = z.object({ + helioBaseUrl: z + .string() + .refine((s) => URL.canParse(s), 'must be a valid URL') + .default(DEFAULT_CONFIG.helioBaseUrl), + tokenEnv: z.string().min(1).default(DEFAULT_CONFIG.tokenEnv), + origin: z + .string() + .regex(/^[a-z0-9_-]{1,64}$/) + .default(DEFAULT_CONFIG.origin), + evaluateTimeoutMs: z.number().int().min(1).default(DEFAULT_CONFIG.evaluateTimeoutMs), + evidence: z.record(z.string(), z.array(evidenceRuleSchema)).default({}), +}) + +/** Result of resolving plugin config. Fail-closed: the caller blocks all calls when `ok` is false. */ +export type ParseConfigResult = + | { readonly ok: true; readonly config: AdapterConfig } + | { readonly ok: false; readonly error: string } + +/** + * Validate the host-provided plugin config (`api.pluginConfig`) into a resolved `AdapterConfig`. + * `undefined`/`null` means no operator config → all defaults. Returns `{ ok: false }` (never throws) + * on invalid input so the caller can fail closed by registering blocking hooks. + */ +export function parseConfig(raw: unknown): ParseConfigResult { + const input = raw === undefined || raw === null ? {} : raw + const parsed = adapterConfigSchema.safeParse(input) + if (!parsed.success) { + const error = parsed.error.issues + .map((issue) => `${issue.path.join('.') || '(root)'}: ${issue.message}`) + .join('; ') + return { ok: false, error } + } + + const d = parsed.data + // Rebuild evidence entries explicitly so optional `ttlSeconds` honors exactOptionalPropertyTypes. + const evidence: Record = {} + for (const [toolName, rules] of Object.entries(d.evidence)) { + evidence[toolName] = rules.map((rule) => ({ + key: rule.key, + path: rule.path, + ...(rule.ttlSeconds !== undefined ? { ttlSeconds: rule.ttlSeconds } : {}), + })) + } + + return { + ok: true, + config: { + helioBaseUrl: d.helioBaseUrl, + tokenEnv: d.tokenEnv, + origin: d.origin, + evaluateTimeoutMs: d.evaluateTimeoutMs, + evidence, + }, + } } diff --git a/src/hooks/after-tool-call.test.ts b/src/hooks/after-tool-call.test.ts new file mode 100644 index 0000000..93f7781 --- /dev/null +++ b/src/hooks/after-tool-call.test.ts @@ -0,0 +1,491 @@ +import { describe, expect, it, vi } from 'vitest' +import { createAfterToolCallHook } from './after-tool-call.js' +import { CorrelationRegistry } from '../correlation/registry.js' +import type { AuditOutcome, HelioClient } from '../client/helio-client.js' +import type { PluginHookAfterToolCallEvent, PluginHookToolContext } from '../types.js' + +function makeClient(auditImpl?: HelioClient['audit']) { + const audit = vi.fn(auditImpl ?? (() => Promise.resolve({ ok: true }))) + const client: HelioClient = { + evaluate: vi.fn(() => Promise.resolve({ ok: false, reason: 'n/a' })), + audit, + installScan: vi.fn(() => + Promise.resolve({ ok: false, reason: 'n/a' }), + ), + resolveApproval: vi.fn(() => Promise.resolve({ ok: true })), + } + return { client, audit } +} + +// Pre-bind an evaluation in the registry so a matching after_tool_call can claim it. +function bind( + registry: CorrelationRegistry, + evaluationId: string, + meta: Parameters[0], +) { + const reservation = registry.reserve(meta) + if (!reservation.ok) throw new Error('test setup: reserve failed') + registry.bind(reservation.ticket, evaluationId) +} + +const event = (over: Partial = {}): PluginHookAfterToolCallEvent => ({ + toolName: 'send_message', + params: { channel: '#general', text: 'hi' }, + ...over, +}) + +const ctx = (over: Partial = {}): PluginHookToolContext => ({ + toolName: 'send_message', + ...over, +}) + +describe('after_tool_call', () => { + it('claims the evaluation_id and posts a success audit with result + duration', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-1', { session: 'oc:s1', toolName: 'send_message', toolCallId: 'tc-1' }) + const hook = createAfterToolCallHook({ client, registry }) + + await hook( + event({ toolCallId: 'tc-1', result: { ok: true }, durationMs: 412 }), + ctx({ sessionId: 's1', toolCallId: 'tc-1' }), + ) + + expect(audit).toHaveBeenCalledWith({ + evaluation_id: 'eval-1', + status: 'success', + duration_ms: 412, + result: { ok: true }, + }) + }) + + it('posts an error audit carrying the error message when event.error is set', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-2', { session: 'oc:s1', toolName: 'send_message', toolCallId: 'tc-2' }) + const hook = createAfterToolCallHook({ client, registry }) + + await hook( + event({ toolCallId: 'tc-2', error: 'boom', durationMs: 7 }), + ctx({ sessionId: 's1', toolCallId: 'tc-2' }), + ) + + expect(audit).toHaveBeenCalledWith({ + evaluation_id: 'eval-2', + status: 'error', + error: 'boom', + duration_ms: 7, + }) + }) + + it('omits optional fields that are absent on the event', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-3', { session: 'oc:s1', toolName: 'send_message', toolCallId: 'tc-3' }) + const hook = createAfterToolCallHook({ client, registry }) + + await hook(event({ toolCallId: 'tc-3' }), ctx({ sessionId: 's1', toolCallId: 'tc-3' })) + + expect(audit).toHaveBeenCalledWith({ evaluation_id: 'eval-3', status: 'success' }) + }) + + it('does not post when there is no correlated evaluation to audit (claim miss)', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + const hook = createAfterToolCallHook({ client, registry }) + + await hook( + event({ toolCallId: 'never-bound' }), + ctx({ sessionId: 's1', toolCallId: 'never-bound' }), + ) + + expect(audit).not.toHaveBeenCalled() + }) + + it('claims the slot so a replayed after_tool_call no longer audits', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-4', { session: 'oc:s1', toolName: 'send_message', toolCallId: 'tc-4' }) + const hook = createAfterToolCallHook({ client, registry }) + + await hook(event({ toolCallId: 'tc-4' }), ctx({ sessionId: 's1', toolCallId: 'tc-4' })) + await hook(event({ toolCallId: 'tc-4' }), ctx({ sessionId: 's1', toolCallId: 'tc-4' })) + + expect(audit).toHaveBeenCalledTimes(1) + }) + + it('never throws across the hook even when the audit call rejects (best-effort)', async () => { + const { client } = makeClient((): Promise => Promise.reject(new Error('network'))) + const registry = new CorrelationRegistry() + bind(registry, 'eval-5', { session: 'oc:s1', toolName: 'send_message', toolCallId: 'tc-5' }) + const hook = createAfterToolCallHook({ client, registry }) + + await expect( + hook(event({ toolCallId: 'tc-5' }), ctx({ sessionId: 's1', toolCallId: 'tc-5' })), + ).resolves.toBeUndefined() + }) +}) + +describe('after_tool_call evidence extraction', () => { + it('attaches success-only evidence extracted from result by configured key paths', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e1', { session: 'oc:s1', toolName: 'send_email', toolCallId: 'tc-1' }) + const hook = createAfterToolCallHook({ + client, + registry, + evidence: { + send_email: [ + { key: 'recipient', path: ['to'], ttlSeconds: 300 }, + { key: 'message_id', path: ['id'] }, + ], + }, + }) + + await hook( + event({ toolName: 'send_email', toolCallId: 'tc-1', result: { to: 'a@b.com', id: 'm1' } }), + ctx({ toolName: 'send_email', sessionId: 's1', toolCallId: 'tc-1' }), + ) + + expect(audit).toHaveBeenCalledWith({ + evaluation_id: 'eval-e1', + status: 'success', + result: { to: 'a@b.com', id: 'm1' }, + evidence: [ + { evidence_key: 'recipient', evidence_data: 'a@b.com', ttl_seconds: 300 }, + { evidence_key: 'message_id', evidence_data: 'm1' }, + ], + }) + }) + + it('walks nested paths and skips rules whose path is absent or runs through a non-object', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e2', { session: 'oc:s1', toolName: 'http_request', toolCallId: 'tc-2' }) + const hook = createAfterToolCallHook({ + client, + registry, + evidence: { + http_request: [ + { key: 'host', path: ['url', 'host'] }, + { key: 'absent', path: ['url', 'missing'] }, + { key: 'through_scalar', path: ['status', 'deep'] }, + ], + }, + }) + + await hook( + event({ + toolName: 'http_request', + toolCallId: 'tc-2', + result: { url: { host: 'example.com' }, status: 200 }, + }), + ctx({ toolName: 'http_request', sessionId: 's1', toolCallId: 'tc-2' }), + ) + + const body = audit.mock.calls[0]?.[0] + expect(body?.evidence).toEqual([{ evidence_key: 'host', evidence_data: 'example.com' }]) + }) + + it('does not attach evidence on an error outcome (success-only)', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e3', { session: 'oc:s1', toolName: 'send_email', toolCallId: 'tc-3' }) + const hook = createAfterToolCallHook({ + client, + registry, + evidence: { send_email: [{ key: 'recipient', path: ['to'] }] }, + }) + + await hook( + event({ + toolName: 'send_email', + toolCallId: 'tc-3', + error: 'smtp down', + result: { to: 'a@b.com' }, + }), + ctx({ toolName: 'send_email', sessionId: 's1', toolCallId: 'tc-3' }), + ) + + expect(audit).toHaveBeenCalledWith({ + evaluation_id: 'eval-e3', + status: 'error', + error: 'smtp down', + result: { to: 'a@b.com' }, + }) + }) + + it('omits the evidence field entirely when no rule matches the tool', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e4', { session: 'oc:s1', toolName: 'send_message', toolCallId: 'tc-4' }) + const hook = createAfterToolCallHook({ + client, + registry, + evidence: { send_email: [{ key: 'recipient', path: ['to'] }] }, + }) + + await hook( + event({ toolCallId: 'tc-4', result: { to: 'a@b.com' } }), + ctx({ sessionId: 's1', toolCallId: 'tc-4' }), + ) + + expect(audit).toHaveBeenCalledWith({ + evaluation_id: 'eval-e4', + status: 'success', + result: { to: 'a@b.com' }, + }) + }) + + it('omits evidence when the result is absent even though rules are configured', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e5', { session: 'oc:s1', toolName: 'send_email', toolCallId: 'tc-5' }) + const hook = createAfterToolCallHook({ + client, + registry, + evidence: { send_email: [{ key: 'recipient', path: ['to'] }] }, + }) + + await hook( + event({ toolName: 'send_email', toolCallId: 'tc-5' }), + ctx({ toolName: 'send_email', sessionId: 's1', toolCallId: 'tc-5' }), + ) + + expect(audit).toHaveBeenCalledWith({ evaluation_id: 'eval-e5', status: 'success' }) + }) + + it('degrades to auditing without evidence when extraction throws (hostile result value)', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e6', { session: 'oc:s1', toolName: 'send_email', toolCallId: 'tc-6' }) + const hook = createAfterToolCallHook({ + client, + registry, + evidence: { send_email: [{ key: 'recipient', path: ['to'] }] }, + }) + + // A tool-controlled result whose property access throws (getter/proxy) must not escape the + // observational hook — the audit still posts, just without evidence. + const hostile: Record = {} + Object.defineProperty(hostile, 'to', { + enumerable: true, + get() { + throw new Error('hostile getter') + }, + }) + + await expect( + hook( + event({ toolName: 'send_email', toolCallId: 'tc-6', result: hostile }), + ctx({ toolName: 'send_email', sessionId: 's1', toolCallId: 'tc-6' }), + ), + ).resolves.toBeUndefined() + + const body = audit.mock.calls[0]?.[0] + expect(body?.evaluation_id).toBe('eval-e6') + expect(body?.status).toBe('success') + expect(body && 'evidence' in body).toBe(false) + }) + + it('omits a non-JSON-serializable result so the audit still finalizes (no lost audit)', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e7', { session: 'oc:s1', toolName: 'send_message', toolCallId: 'tc-7' }) + const hook = createAfterToolCallHook({ client, registry }) // no evidence rules + + // A result the real client's JSON.stringify cannot encode (BigInt) must not reach the client as + // a body it would choke on — that throw is caught client-side and the POST never goes out, + // losing the audit for a call that ran. The adapter drops the unserializable field instead. + await hook( + event({ toolCallId: 'tc-7', result: { ok: true, big: 10n }, durationMs: 9 }), + ctx({ sessionId: 's1', toolCallId: 'tc-7' }), + ) + + const body = audit.mock.calls[0]?.[0] + expect(body?.evaluation_id).toBe('eval-e7') + expect(body?.status).toBe('success') + expect(body?.duration_ms).toBe(9) + expect(body && 'result' in body).toBe(false) + expect(() => JSON.stringify(body)).not.toThrow() + }) + + it('drops only the unserializable field, keeping serializable evidence', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e8', { session: 'oc:s1', toolName: 'send_email', toolCallId: 'tc-8' }) + const hook = createAfterToolCallHook({ + client, + registry, + evidence: { send_email: [{ key: 'recipient', path: ['to'] }] }, + }) + + // `result` carries an unserializable field (`junk`) that no rule extracts, while the extracted + // evidence (`to`) is a plain string. The result drops; the serializable evidence is kept. + await hook( + event({ toolName: 'send_email', toolCallId: 'tc-8', result: { to: 'a@b.com', junk: 10n } }), + ctx({ toolName: 'send_email', sessionId: 's1', toolCallId: 'tc-8' }), + ) + + const body = audit.mock.calls[0]?.[0] + expect(body && 'result' in body).toBe(false) + expect(body?.evidence).toEqual([{ evidence_key: 'recipient', evidence_data: 'a@b.com' }]) + expect(() => JSON.stringify(body)).not.toThrow() + }) + + it('hands the client a stable snapshot, not the live result (no serialize-twice TOCTOU)', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e9', { session: 'oc:s1', toolName: 'send_message', toolCallId: 'tc-9' }) + const hook = createAfterToolCallHook({ client, registry }) + + // A non-deterministic getter: serializable on the FIRST read, throws on the SECOND. A + // probe-then-resend design passes the probe and throws on the client's later JSON.stringify, + // losing the audit. Snapshotting once must hand the client a plain value it can re-serialize. + let reads = 0 + const flaky: Record = {} + Object.defineProperty(flaky, 'x', { + enumerable: true, + get() { + reads += 1 + if (reads > 1) throw new Error('second read throws') + return 'ok' + }, + }) + + await hook( + event({ toolCallId: 'tc-9', result: flaky }), + ctx({ sessionId: 's1', toolCallId: 'tc-9' }), + ) + + const body = audit.mock.calls[0]?.[0] + expect(body?.result).toEqual({ x: 'ok' }) + // The body the client receives must survive (re-)serialization — i.e. it is not the live getter. + expect(() => JSON.stringify(body)).not.toThrow() + }) + + it('filters evidence per entry, keeping good entries when one is unserializable', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e10', { session: 'oc:s1', toolName: 'send_email', toolCallId: 'tc-10' }) + const hook = createAfterToolCallHook({ + client, + registry, + evidence: { + send_email: [ + { key: 'recipient', path: ['to'] }, + { key: 'count', path: ['big'] }, + ], + }, + }) + + // `big` extracts a BigInt (unserializable) — that one entry drops; `recipient` survives. + await hook( + event({ toolName: 'send_email', toolCallId: 'tc-10', result: { to: 'a@b.com', big: 10n } }), + ctx({ toolName: 'send_email', sessionId: 's1', toolCallId: 'tc-10' }), + ) + + const body = audit.mock.calls[0]?.[0] + expect(body?.evidence).toEqual([{ evidence_key: 'recipient', evidence_data: 'a@b.com' }]) + expect(() => JSON.stringify(body)).not.toThrow() + }) + + // The sideband rejects bodies over 1 MiB (docs/adapter-api.md), so an oversized optional field + // must not 413 the mandatory audit for a call that ran. Drop result first, then trim evidence. + const overOneMiB = 'x'.repeat(1_100_000) + + it('drops an oversized result before evidence so the audit body fits the budget', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e11', { session: 'oc:s1', toolName: 'send_email', toolCallId: 'tc-11' }) + const hook = createAfterToolCallHook({ + client, + registry, + evidence: { send_email: [{ key: 'recipient', path: ['to'] }] }, + }) + + // result is > 1 MiB; the extracted evidence is tiny. Dropping result alone brings it under budget. + await hook( + event({ + toolName: 'send_email', + toolCallId: 'tc-11', + result: { to: 'a@b.com', blob: overOneMiB }, + }), + ctx({ toolName: 'send_email', sessionId: 's1', toolCallId: 'tc-11' }), + ) + + const body = audit.mock.calls[0]?.[0] + expect(body?.evaluation_id).toBe('eval-e11') + expect(body?.status).toBe('success') + expect(body && 'result' in body).toBe(false) + expect(body?.evidence).toEqual([{ evidence_key: 'recipient', evidence_data: 'a@b.com' }]) + expect(new TextEncoder().encode(JSON.stringify(body)).length).toBeLessThanOrEqual(1_048_576) + }) + + it('trims oversized evidence down to the core so the audit still finalizes', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e12', { session: 'oc:s1', toolName: 'send_email', toolCallId: 'tc-12' }) + const hook = createAfterToolCallHook({ + client, + registry, + evidence: { send_email: [{ key: 'blob', path: ['big'] }] }, + }) + + // The single evidence entry is itself > 1 MiB; after dropping result, evidence must be trimmed + // too, leaving the always-serializable core to finalize. + await hook( + event({ toolName: 'send_email', toolCallId: 'tc-12', result: { big: overOneMiB } }), + ctx({ toolName: 'send_email', sessionId: 's1', toolCallId: 'tc-12' }), + ) + + const body = audit.mock.calls[0]?.[0] + expect(body?.evaluation_id).toBe('eval-e12') + expect(body?.status).toBe('success') + expect(body && 'result' in body).toBe(false) + expect(body && 'evidence' in body).toBe(false) + expect(new TextEncoder().encode(JSON.stringify(body)).length).toBeLessThanOrEqual(1_048_576) + }) + + it('truncates an oversized error so the core audit still finalizes under budget', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e13', { session: 'oc:s1', toolName: 'send_message', toolCallId: 'tc-13' }) + const hook = createAfterToolCallHook({ client, registry }) + + // `error` is optional in the wire contract, so an oversized error message must not keep the + // body over budget and 413 the audit — it is truncated (UTF-8 safe, with a marker) to fit. + await hook( + event({ toolCallId: 'tc-13', error: 'e'.repeat(1_200_000), durationMs: 3 }), + ctx({ sessionId: 's1', toolCallId: 'tc-13' }), + ) + + const body = audit.mock.calls[0]?.[0] + expect(body?.evaluation_id).toBe('eval-e13') + expect(body?.status).toBe('error') + expect(body?.duration_ms).toBe(3) + expect(typeof body?.error).toBe('string') + expect(body?.error?.endsWith('...[truncated]')).toBe(true) + expect(new TextEncoder().encode(JSON.stringify(body)).length).toBeLessThanOrEqual(1_048_576) + }) + + it('does not split a multi-byte code point when truncating an oversized error', async () => { + const { client, audit } = makeClient() + const registry = new CorrelationRegistry() + bind(registry, 'eval-e14', { session: 'oc:s1', toolName: 'send_message', toolCallId: 'tc-14' }) + const hook = createAfterToolCallHook({ client, registry }) + + // '😀' is 4 UTF-8 bytes; a naive byte cut could leave a lone surrogate / replacement char. + await hook( + event({ toolCallId: 'tc-14', error: '😀'.repeat(400_000) }), + ctx({ sessionId: 's1', toolCallId: 'tc-14' }), + ) + + const body = audit.mock.calls[0]?.[0] + const error = body?.error ?? '' + expect(error.endsWith('...[truncated]')).toBe(true) + // The kept prefix is whole emoji only — no U+FFFD replacement char from a split sequence. + expect(error).not.toContain('�') + expect(new TextEncoder().encode(JSON.stringify(body)).length).toBeLessThanOrEqual(1_048_576) + }) +}) diff --git a/src/hooks/after-tool-call.ts b/src/hooks/after-tool-call.ts index 586b60d..eaa8089 100644 --- a/src/hooks/after-tool-call.ts +++ b/src/hooks/after-tool-call.ts @@ -1,10 +1,203 @@ -import type { HelioClient } from '../client/helio-client.js' +import { mapSession } from '../session/mapping.js' +import type { AuditEvidenceEntry, AuditRequest, HelioClient } from '../client/helio-client.js' +import type { EvidenceConfig, EvidenceRule } from '../config.js' +import type { CorrelationRegistry } from '../correlation/registry.js' import type { PluginHookAfterToolCallEvent, PluginHookToolContext } from '../types.js' -// after_tool_call → POST /audit (+ success-only evidence). The host ignores the return value. -export function createAfterToolCallHook(_client: HelioClient) { - return (_event: PluginHookAfterToolCallEvent, _ctx: PluginHookToolContext): void => { - // TODO: POST /audit; treat 200 already_finalized as success; attach evidence on success. - throw new Error('not implemented') +export interface AfterToolCallDeps { + readonly client: HelioClient + readonly registry: CorrelationRegistry + /** Success-only evidence-extraction rules, keyed by tool name. Absent → no evidence. */ + readonly evidence?: EvidenceConfig +} + +// Walk `result` by explicit path segments. Returns `found: false` if the path runs off the object +// or through a non-object — a missing value must skip the entry, never emit `undefined`. +function extractByPath(root: unknown, path: readonly string[]): { found: boolean; value: unknown } { + let current: unknown = root + for (const segment of path) { + if (typeof current !== 'object' || current === null) return { found: false, value: undefined } + current = (current as Record)[segment] + if (current === undefined) return { found: false, value: undefined } + } + return { found: true, value: current } +} + +// `result` and extracted evidence are tool-controlled and may not be JSON-encodable (BigInt, +// circular refs, throwing getters). The client serializes the body with JSON.stringify and a throw +// there is caught as a failed POST — losing the audit for a call that ran. Snapshot each risky field +// to a stable plain-JSON clone *once* here, so the value the client later re-serializes is exactly +// what we validated — no second, possibly-divergent evaluation of a hostile/non-deterministic getter +// or proxy (TOCTOU). On failure the field is dropped; the always-serializable core +// (evaluation_id/status/error/duration_ms) still finalizes. +function safeJsonSnapshot(value: T): { ok: true; value: T } | { ok: false } { + try { + return { ok: true, value: JSON.parse(JSON.stringify(value)) as T } + } catch { + return { ok: false } + } +} + +// The sideband rejects request bodies over 1 MiB (docs/adapter-api.md) with 413. Keep headroom so a +// near-limit body can't tip over and lose the audit for a call that ran. +const MAX_AUDIT_BODY_BYTES = 1_048_576 - 16_384 + +function bodyBytes(req: AuditRequest): number { + return new TextEncoder().encode(JSON.stringify(req)).length +} + +// Truncate `str` to at most `maxBytes` UTF-8 bytes without splitting a multi-byte code point (back +// off the cut while it lands on a continuation byte, 0b10xxxxxx). +function truncateUtf8(str: string, maxBytes: number): string { + const bytes = new TextEncoder().encode(str) + if (bytes.length <= maxBytes) return str + let end = maxBytes + while (end > 0 && ((bytes[end] ?? 0) & 0xc0) === 0x80) end -= 1 + return new TextDecoder().decode(bytes.subarray(0, end)) +} + +// Fit the audit body under the size budget by shedding fields in priority order so a call that ran +// always finalizes rather than 413-ing into a false `evaluation_expired`. The truly unsheddable +// minimal core (evaluation_id/status/duration_ms) is tiny and bounded; everything else degrades: +// drop the (unbounded) `result`, then trim `evidence` from the end, then — since `error` is optional +// in the wire contract — truncate an oversized `error` (UTF-8 safe, with a marker) and finally omit +// it entirely. +function fitToBudget( + base: AuditRequest, + error: string | undefined, + result: { value: unknown } | undefined, + evidence: readonly AuditEvidenceEntry[], +): AuditRequest { + const build = ( + err: string | undefined, + includeResult: boolean, + ev: readonly AuditEvidenceEntry[], + ): AuditRequest => ({ + ...base, + ...(err !== undefined ? { error: err } : {}), + ...(includeResult && result !== undefined ? { result: result.value } : {}), + ...(ev.length > 0 ? { evidence: ev } : {}), + }) + const fits = (req: AuditRequest): boolean => bodyBytes(req) <= MAX_AUDIT_BODY_BYTES + + let candidate = build(error, true, evidence) + if (fits(candidate)) return candidate + + candidate = build(error, false, evidence) + if (fits(candidate)) return candidate + + let kept = evidence.slice(0, -1) + while (kept.length > 0) { + candidate = build(error, false, kept) + if (fits(candidate)) return candidate + kept = kept.slice(0, -1) + } + + candidate = build(error, false, []) + if (fits(candidate) || error === undefined) return candidate + + // Only an oversized `error` remains. Keep the largest UTF-8-safe prefix (with a marker) that fits; + // omitting it (best = no error) always fits since the minimal core is tiny. + const marker = '...[truncated]' + const totalBytes = new TextEncoder().encode(error).length + let best = build(undefined, false, []) + let lo = 0 + let hi = totalBytes + while (lo <= hi) { + const mid = (lo + hi) >> 1 + const trial = build(truncateUtf8(error, mid) + marker, false, []) + if (fits(trial)) { + best = trial + lo = mid + 1 + } else { + hi = mid - 1 + } + } + return best +} + +// Apply a tool's rules to `result`, dropping any rule whose path is absent. Success-only: the +// caller must not invoke this for an error/not-executed outcome. +function extractEvidence( + rules: readonly EvidenceRule[], + result: unknown, +): readonly AuditEvidenceEntry[] { + const entries: AuditEvidenceEntry[] = [] + for (const rule of rules) { + const { found, value } = extractByPath(result, rule.path) + if (!found) continue + entries.push({ + evidence_key: rule.key, + evidence_data: value, + ...(rule.ttlSeconds !== undefined ? { ttl_seconds: rule.ttlSeconds } : {}), + }) + } + return entries +} + +// after_tool_call → POST /audit. The host ignores the return value (observational), so this is +// best-effort: a failed audit is never thrown across the hook. A 200 `already_finalized` replay +// (terminal-at-evaluate) is treated as success by the client, so we audit unconditionally. +export function createAfterToolCallHook(deps: AfterToolCallDeps) { + const { client, registry } = deps + + return async (event: PluginHookAfterToolCallEvent, ctx: PluginHookToolContext): Promise => { + const session = mapSession(ctx) + + // Claim the evaluation_id bound by the matching before_tool_call. A miss means there is no id + // to audit — the before-hook released the slot for a terminal decision, or it was never bound + // (e.g. fail-closed block). Nothing to do; the pending evaluation expires server-side. + const evaluationId = registry.claim({ + session, + toolName: event.toolName, + ...(event.toolCallId !== undefined ? { toolCallId: event.toolCallId } : {}), + ...(event.runId !== undefined ? { runId: event.runId } : {}), + }) + if (evaluationId === undefined) return + + const isError = event.error !== undefined + // Evidence is success-only (a failed tool must not ground later calls) and config-driven. + // `event.result` is tool-controlled `unknown`, so a hostile property access (throwing getter / + // proxy) during the path walk must never escape this observational hook — degrade to auditing + // without evidence so the audit record (and its counter commit) for a call that ran is kept. + let evidence: readonly AuditEvidenceEntry[] = [] + if (!isError) { + try { + evidence = extractEvidence(deps.evidence?.[event.toolName] ?? [], event.result) + } catch { + evidence = [] + } + } + + const result = event.result !== undefined ? safeJsonSnapshot(event.result) : undefined + + // Snapshot evidence per entry so one unserializable extracted value drops only its own entry, + // not the valid grounding facts beside it. + const safeEvidence: AuditEvidenceEntry[] = [] + for (const entry of evidence) { + const snapshot = safeJsonSnapshot(entry) + if (snapshot.ok) safeEvidence.push(snapshot.value) + } + + const base: AuditRequest = { + evaluation_id: evaluationId, + status: isError ? 'error' : 'success', + ...(event.durationMs !== undefined ? { duration_ms: event.durationMs } : {}), + } + const request = fitToBudget( + base, + event.error, + result?.ok ? { value: result.value } : undefined, + safeEvidence, + ) + + // Best-effort: the client never throws (it returns an outcome), but guard anyway so an + // unexpected failure can never escape an observational hook. + try { + await client.audit(request) + } catch { + // Swallowed deliberately — the call already ran; a lost audit surfaces server-side as an + // `evaluation_expired` record, which Helio monitors as a tamper/bypass signal. + } } } diff --git a/src/index.test.ts b/src/index.test.ts index 88e9a0e..f3314be 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -1,20 +1,154 @@ -import { describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import definition from './index.js' -import type { OpenClawPluginApi } from './types.js' +import type { AuditRequest } from './client/helio-client.js' +import type { + OpenClawPluginApi, + PluginHookAfterToolCallEvent, + PluginHookBeforeInstallResult, + PluginHookBeforeToolCallEvent, + PluginHookBeforeToolCallResult, + PluginHookToolContext, +} from './types.js' + +const TOKEN_ENV = 'HELIO_ADAPTER_TOKEN' + +interface Harness { + readonly handlers: Map + readonly on: ReturnType + readonly logger: { + error: ReturnType + warn: ReturnType + info: ReturnType + } + readonly api: OpenClawPluginApi +} + +function harness(pluginConfig: unknown): Harness { + const handlers = new Map() + const on = vi.fn((name: string, handler: unknown) => { + handlers.set(name, handler) + }) + const logger = { error: vi.fn(), warn: vi.fn(), info: vi.fn() } + const api = { on, logger, pluginConfig } as unknown as OpenClawPluginApi + return { handlers, on, logger, api } +} + +const beforeHook = (h: Harness) => + h.handlers.get('before_tool_call') as ( + event: PluginHookBeforeToolCallEvent, + ctx: PluginHookToolContext, + ) => Promise | PluginHookBeforeToolCallResult + +const afterHook = (h: Harness) => + h.handlers.get('after_tool_call') as ( + event: PluginHookAfterToolCallEvent, + ctx: PluginHookToolContext, + ) => Promise + +// In degraded mode the install gate ignores its arguments and always blocks. +const installHook = (h: Harness) => + h.handlers.get('before_install') as () => PluginHookBeforeInstallResult + +let savedToken: string | undefined + +beforeEach(() => { + savedToken = process.env[TOKEN_ENV] +}) + +afterEach(() => { + if (savedToken === undefined) Reflect.deleteProperty(process.env, TOKEN_ENV) + else process.env[TOKEN_ENV] = savedToken + vi.unstubAllGlobals() +}) describe('plugin definition', () => { - it('registers the three governance hooks under the "helio" id', () => { - const on = vi.fn() - const api = { on } as unknown as OpenClawPluginApi + it('registers the three governance hooks when configured with a token', () => { + process.env[TOKEN_ENV] = 'tok' + const h = harness({}) expect(definition.id).toBe('helio') - definition.register?.(api) + definition.register?.(h.api) - expect(on).toHaveBeenCalledTimes(3) - expect(on.mock.calls.map((call) => call[0])).toStrictEqual([ + expect(h.on.mock.calls.map((call) => call[0] as string)).toStrictEqual([ 'before_tool_call', 'after_tool_call', 'before_install', ]) }) + + it('fails closed (blocks all tool calls) when the adapter token is missing', () => { + Reflect.deleteProperty(process.env, TOKEN_ENV) + const h = harness({}) + + definition.register?.(h.api) + + const result = beforeHook(h)( + { toolName: 'send_message', params: {} }, + { toolName: 'send_message' }, + ) + expect(result).toEqual({ block: true, blockReason: 'Helio governance misconfigured' }) + // installs are gated too — governance misconfig must not let an install through + expect(installHook(h)()).toEqual({ block: true, blockReason: 'Helio governance misconfigured' }) + expect(h.logger.error).toHaveBeenCalledOnce() + // no audit path in degraded mode — only the blocking gates are registered + expect(h.handlers.has('after_tool_call')).toBe(false) + }) + + it('fails closed when the plugin config is invalid', () => { + process.env[TOKEN_ENV] = 'tok' + const h = harness({ origin: 'Not A Valid Origin!' }) + + definition.register?.(h.api) + + const result = beforeHook(h)( + { toolName: 'send_message', params: {} }, + { toolName: 'send_message' }, + ) + expect(result).toEqual({ block: true, blockReason: 'Helio governance misconfigured' }) + expect(h.logger.error).toHaveBeenCalledOnce() + }) + + it('passes configured evidence rules end-to-end into the /audit body', async () => { + process.env[TOKEN_ENV] = 'tok' + const auditBodies: AuditRequest[] = [] + const fetchMock = vi.fn((input: string | URL, init?: RequestInit) => { + const url = typeof input === 'string' ? input : input.toString() + if (url.endsWith('/evaluate')) { + return Promise.resolve( + new Response(JSON.stringify({ evaluation_id: 'ev1', decision: 'allow' }), { + status: 200, + }), + ) + } + if (url.endsWith('/audit')) { + const body = init?.body as string | undefined + auditBodies.push(JSON.parse(body ?? '{}') as AuditRequest) + return Promise.resolve(new Response(JSON.stringify({ ok: true }), { status: 200 })) + } + return Promise.resolve(new Response('{}', { status: 200 })) + }) + vi.stubGlobal('fetch', fetchMock) + + const h = harness({ evidence: { send_email: [{ key: 'recipient', path: ['to'] }] } }) + definition.register?.(h.api) + + const ctx: PluginHookToolContext = { + toolName: 'send_email', + sessionId: 's1', + toolCallId: 'tc1', + } + await beforeHook(h)( + { toolName: 'send_email', params: { to: 'a@b.com' }, toolCallId: 'tc1' }, + ctx, + ) + await afterHook(h)( + { toolName: 'send_email', params: {}, toolCallId: 'tc1', result: { to: 'a@b.com' } }, + ctx, + ) + + expect(auditBodies).toHaveLength(1) + expect(auditBodies[0]?.evidence).toEqual([ + { evidence_key: 'recipient', evidence_data: 'a@b.com' }, + ]) + }) }) diff --git a/src/index.ts b/src/index.ts index 39cd57f..b1914ec 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,27 +6,47 @@ import { createBeforeInstallHook } from './hooks/before-install.js' import { createBeforeToolCallHook } from './hooks/before-tool-call.js' import type { OpenClawPluginApi, OpenClawPluginDefinition } from './types.js' +const MISCONFIGURED_REASON = 'Helio governance misconfigured' + const definition: OpenClawPluginDefinition = { id: 'helio', name: 'Helio Governance', register(api: OpenClawPluginApi): void { - // TODO(config step): resolve + validate the real plugin config from the host, and - // fail-closed when the token is missing. Defaults + env lookup are a placeholder. - const adapterConfig = parseConfig(undefined) + const parsed = parseConfig(api.pluginConfig) + const token = parsed.ok ? process.env[parsed.config.tokenEnv] : undefined + + // Fail closed WITHOUT depending on the host's register()-throw semantics: if the config is + // invalid or the adapter token is missing, register hooks that BLOCK every governed action + // rather than leaving the tool surface ungoverned (which is what skipping registration would + // do). The operator sees everything blocked + a loud log, and fixes the config. + if (!parsed.ok || token === undefined || token === '') { + const reason = !parsed.ok + ? `Helio adapter config is invalid: ${parsed.error}` + : `Helio adapter token ($${parsed.config.tokenEnv}) is not set` + api.logger.error(`${reason} — blocking all governed tool calls and installs.`) + api.on('before_tool_call', () => ({ block: true, blockReason: MISCONFIGURED_REASON })) + api.on('before_install', () => ({ block: true, blockReason: MISCONFIGURED_REASON })) + return + } + + const { config } = parsed const clientConfig: HelioClientConfig = { - baseUrl: adapterConfig.helioBaseUrl, - token: process.env[adapterConfig.tokenEnv] ?? '', - origin: adapterConfig.origin, - evaluateTimeoutMs: adapterConfig.evaluateTimeoutMs, + baseUrl: config.helioBaseUrl, + token, + origin: config.origin, + evaluateTimeoutMs: config.evaluateTimeoutMs, } const client = createHelioClient(clientConfig) const registry = new CorrelationRegistry() api.on( 'before_tool_call', - createBeforeToolCallHook({ client, registry, origin: adapterConfig.origin }), + createBeforeToolCallHook({ client, registry, origin: config.origin }), + ) + api.on( + 'after_tool_call', + createAfterToolCallHook({ client, registry, evidence: config.evidence }), ) - api.on('after_tool_call', createAfterToolCallHook(client)) api.on('before_install', createBeforeInstallHook(client)) }, }