From d5cf8d774d06aef02ab6beb1a9518076d9cf9a6f Mon Sep 17 00:00:00 2001 From: majormaxx Date: Sun, 30 Aug 2026 17:39:28 +0100 Subject: [PATCH] added privacy redaction pipeline across audit, webhook, export, and websocket sinks --- backend/src/api/websocket/websocket.server.ts | 10 +- backend/src/config/index.ts | 17 + .../20260830100000_redaction_decisions.ts | 33 ++ backend/src/privacy/fieldRegistry.ts | 237 +++++++++++ backend/src/privacy/index.ts | 49 +++ backend/src/privacy/pseudonymizer.ts | 78 ++++ backend/src/privacy/redaction.service.ts | 375 ++++++++++++++++++ .../src/privacy/redactionDecision.service.ts | 64 +++ backend/src/privacy/secretScanner.ts | 93 +++++ backend/src/privacy/types.ts | 105 +++++ backend/src/services/audit.service.ts | 82 +++- .../src/services/webhook.service.outbox.ts | 19 +- backend/src/services/webhook.service.ts | 29 +- backend/src/services/websocket.ts | 11 +- backend/src/utils/logger.ts | 7 + backend/src/workers/export.worker.ts | 23 +- .../services/privacyRedaction.service.test.ts | 303 ++++++++++++++ 17 files changed, 1508 insertions(+), 27 deletions(-) create mode 100644 backend/src/database/migrations/20260830100000_redaction_decisions.ts create mode 100644 backend/src/privacy/fieldRegistry.ts create mode 100644 backend/src/privacy/index.ts create mode 100644 backend/src/privacy/pseudonymizer.ts create mode 100644 backend/src/privacy/redaction.service.ts create mode 100644 backend/src/privacy/redactionDecision.service.ts create mode 100644 backend/src/privacy/secretScanner.ts create mode 100644 backend/src/privacy/types.ts create mode 100644 backend/tests/services/privacyRedaction.service.test.ts diff --git a/backend/src/api/websocket/websocket.server.ts b/backend/src/api/websocket/websocket.server.ts index 57d9c44d..170771be 100644 --- a/backend/src/api/websocket/websocket.server.ts +++ b/backend/src/api/websocket/websocket.server.ts @@ -2,6 +2,8 @@ import { randomUUID } from "crypto"; import type { FastifyRequest } from "fastify"; import { config } from "../../config/index.js"; import { logger } from "../../utils/logger.js"; +import { redactionService } from "../../privacy/redaction.service.js"; +import { redactionDecisionService } from "../../privacy/redactionDecision.service.js"; import { factory, redisSubscriber } from "../../utils/redis.js"; import { type ClientState, @@ -327,7 +329,13 @@ export class WebSocketServer implements IBroadcaster { channel: ChannelName, message: OutboundDataMessage ): Promise { - const payload = JSON.stringify(message); + // Redact operational fields (addresses, evidence, endpoint details) before + // the message is serialized to local clients and published to Redis, so no + // cross-instance sink exposes the sensitive material. + const result = redactionService.redact(message, { sink: "websocket" }); + void redactionDecisionService.record(result.decision, { resourceType: "websocket", resourceId: channel }); + + const payload = JSON.stringify(result.output); this.broadcastLocal(channel, payload); await this.publishToRedis(channel, payload).catch((err) => { diff --git a/backend/src/config/index.ts b/backend/src/config/index.ts index 6e062b3c..a3e513cb 100644 --- a/backend/src/config/index.ts +++ b/backend/src/config/index.ts @@ -274,6 +274,23 @@ const envSchema = z.object({ INGESTION_REORG_BUFFER_DEPTH: z.coerce.number().default(100), INGESTION_REORG_POLL_INTERVAL_MS: z.coerce.number().default(30_000), INGESTION_UNCONFIRMED_EVENT_TTL_MINUTES: z.coerce.number().default(60), + + // Privacy / redaction pipeline. Classifies sensitive fields centrally and + // redacts them at every configured sink before persistence or transmission. + // When disabled the pipeline passes payloads through untouched. + REDACTION_ENABLED: z.coerce.boolean().default(true), + // Hex (or ASCII) salt used to derive deterministic pseudonyms. Keep this + // secret: it is the only thing that maps a pseudonym back to a candidate. + // It is intentionally never stored alongside the redacted data. + REDACTION_PSEUDONYM_SALT: z.string().default("bridge-watch-pseudonym-salt-dev"), + // Namespace applied to pseudonyms so identical inputs under different + // payload shapes stay distinct and rotation is scoped. + REDACTION_PSEUDONYM_NAMESPACE: z.string().default("operational"), + // Block (throw) instead of silently redacting when a secret is detected in + // a sink that refuses unsafe payloads. Applies per-sink policy by default. + REDACTION_BLOCK_ON_SECRET: z.coerce.boolean().default(false), + // Persist redaction decisions to the audit table for post-hoc review. + REDACTION_DECISION_LOG_ENABLED: z.coerce.boolean().default(true), }); export type EnvConfig = z.infer; diff --git a/backend/src/database/migrations/20260830100000_redaction_decisions.ts b/backend/src/database/migrations/20260830100000_redaction_decisions.ts new file mode 100644 index 00000000..8008cc17 --- /dev/null +++ b/backend/src/database/migrations/20260830100000_redaction_decisions.ts @@ -0,0 +1,33 @@ +import type { Knex } from "knex"; + +/** + * Redaction pipeline audit + versioning. + * + * Records a redaction decision for post-hoc review. Every decision captures + * which sink applied it, under which policy/rule version, and which fields + * were acted on — the events hold rule fingerprints and field paths only, so + * no original secret value is ever persisted here. + */ +export async function up(knex: Knex): Promise { + await knex.schema.createTable("redaction_decisions", (table) => { + table.uuid("id").primary().defaultTo(knex.raw("gen_random_uuid()")); + table.string("sink", 30).notNullable(); + table.integer("policy_version").notNullable(); + table.boolean("modified").notNullable().defaultTo(false); + table.boolean("secret_detected").notNullable().defaultTo(false); + table.boolean("blocked").notNullable().defaultTo(false); + table.string("policy_fingerprint", 80).notNullable(); + table.jsonb("events").notNullable().defaultTo(knex.raw("'[]'::jsonb")); + table.jsonb("correlation").notNullable().defaultTo(knex.raw("'{}'::jsonb")); + table.timestamp("created_at", { useTz: true }).notNullable().defaultTo(knex.fn.now()); + + table.index(["sink"]); + table.index(["created_at"]); + table.index(["policy_version"]); + table.index(["blocked"]); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists("redaction_decisions"); +} diff --git a/backend/src/privacy/fieldRegistry.ts b/backend/src/privacy/fieldRegistry.ts new file mode 100644 index 00000000..471b20a5 --- /dev/null +++ b/backend/src/privacy/fieldRegistry.ts @@ -0,0 +1,237 @@ +/** + * Central, schema-aware field classification registry. + * + * Every sensitive field in operational payloads is classified here, once. + * Sinks consume these classifications through their policies instead of + * maintaining their own redaction lists, so a change to sensitivity is + * enforced across every sink from a single source of truth. + * + * Rule matching is path-based. A rule's `match` is a dotted path into the + * JSON payload. The following wildcards are supported: + * - a trailing `**` matches any number of remaining segments + * - a trailing `*` matches any single value at that position + * + * Classification is versioned (CURRENT_VERSION) so a redaction decision can + * always be tied back to the exact rule set that produced it. + */ + +import crypto from "crypto"; +import type { FieldRule, SensitivityLevel } from "./types.js"; + +export const FIELD_REGISTRY_CURRENT_VERSION = 1; + +/** + * Match a dotted path against a rule pattern. + * + * Rules support three forms: + * - Exact: "after.source_address" + * - Single-segment wildcard: "transactions.**" (matches the field itself + * and every descendent under transactions) + * - Final-value wildcard: "metadata.*" (matches every immediate child of + * metadata) + */ +export function pathMatches(pattern: string, path: string): boolean { + const patternSegs = pattern.split(".").filter(Boolean); + const pathSegs = path.split(".").filter(Boolean); + + // A trailing `**` matches the prefix and any number of remaining segments. + // A bare `**` matches every path. + const doubleWildcardIndex = patternSegs.indexOf("**"); + if (doubleWildcardIndex !== -1) { + const prefix = patternSegs.slice(0, doubleWildcardIndex); + if (prefix.length > pathSegs.length) return false; + for (let i = 0; i < prefix.length; i++) { + if (prefix[i] !== pathSegs[i]) return false; + } + // Any extra segments after `**` beyond the matched prefix must themselves + // match up to where the suffix wildcard started; `**` consumes the rest. + return true; + } + + if (patternSegs.length !== pathSegs.length) return false; + + for (let i = 0; i < pathSegs.length; i++) { + if (patternSegs[i] === "*") continue; + if (patternSegs[i] !== pathSegs[i]) return false; + } + return true; +} + +export interface ClassifiedField { + rule: FieldRule; + path: string; +} + +export const DEFAULT_FIELD_RULES: FieldRule[] = [ + // -- Account / wallet identifiers ------------------------------------------ + { match: "source_address", sensitivity: "critical", description: "Wallet address the transfer originates from" }, + { match: "destination_address", sensitivity: "critical", description: "Wallet address the transfer is destined for" }, + { match: "*.source_address", sensitivity: "critical" }, + { match: "*.destination_address", sensitivity: "critical" }, + { match: "*.from", sensitivity: "critical", description: "Blockchain event sender" }, + { match: "*.to", sensitivity: "critical", description: "Blockchain event recipient" }, + { match: "*.ownerAddress", sensitivity: "critical", description: "Webhook endpoint owner address" }, + { match: "*.owner_address", sensitivity: "critical" }, + { match: "ownerAddress", sensitivity: "critical" }, + { match: "endpointUrl", sensitivity: "high", description: "Webhook / endpoint URL" }, + { match: "endpoint_url", sensitivity: "high" }, + { match: "*.endpointUrl", sensitivity: "high" }, + { match: "*.endpoint_url", sensitivity: "high" }, + { match: "webhookEndpointId", sensitivity: "medium", description: "Webhook endpoint identifier" }, + { match: "webhook_endpoint_id", sensitivity: "medium" }, + { match: "*.webhookEndpointId", sensitivity: "medium" }, + { match: "endpointName", sensitivity: "medium", description: "Webhook endpoint name" }, + { match: "*.endpointName", sensitivity: "medium" }, + { match: "address", sensitivity: "high", description: "Wallet / contract address" }, + { match: "*.address", sensitivity: "high" }, + { match: "admin.address", sensitivity: "critical" }, + { match: "contractAddress", sensitivity: "high" }, + { match: "*.contractAddress", sensitivity: "high" }, + { match: "issuer", sensitivity: "medium", description: "Asset issuer address" }, + { match: "tx_hash", sensitivity: "high", description: "Transaction hash" }, + { match: "*.tx_hash", sensitivity: "high" }, + { match: "*.txHash", sensitivity: "high" }, + { match: "transactionHash", sensitivity: "high" }, + { match: "*.transactionHash", sensitivity: "high" }, + { match: "leaf_hash", sensitivity: "low", description: "Public merkle leaf hash" }, + { match: "public_key", sensitivity: "medium" }, + { match: "publicKey", sensitivity: "medium" }, + + // -- Network / identity ----------------------------------------------------- + { match: "ipAddress", sensitivity: "high", description: "Client IP address" }, + { match: "ip_address", sensitivity: "high" }, + { match: "*.ipAddress", sensitivity: "high" }, + { match: "*.ip_address", sensitivity: "high" }, + { match: "userAgent", sensitivity: "medium", description: "Client user agent" }, + { match: "user_agent", sensitivity: "medium" }, + { match: "*.userAgent", sensitivity: "medium" }, + { match: "email", sensitivity: "high", description: "Email address" }, + { match: "emailAddress", sensitivity: "high" }, + { match: "email_address", sensitivity: "high" }, + { match: "*.email", sensitivity: "high" }, + { match: "phone", sensitivity: "high" }, + { match: "actorId", sensitivity: "high", description: "Actor identifier" }, + { match: "actor_id", sensitivity: "high" }, + { match: "userId", sensitivity: "medium" }, + { match: "user_id", sensitivity: "medium" }, + { match: "resourceId", sensitivity: "medium", description: "Resource identifier (may embed addresses)" }, + { match: "resource_id", sensitivity: "medium" }, + { match: "incidentId", sensitivity: "medium", description: "Incident identifier" }, + { match: "incident_id", sensitivity: "medium" }, + + // -- Free-text notes / evidence --------------------------------------------- + { match: "metadata.reason", sensitivity: "medium", description: "Free-text reason / note" }, + { match: "metadata.comment", sensitivity: "medium" }, + { match: "metadata.changes", sensitivity: "medium", description: "Free-text change description" }, + { match: "*.reason", sensitivity: "medium" }, + { match: "*.comment", sensitivity: "medium" }, + { match: "notes", sensitivity: "medium" }, + { match: "error_message", sensitivity: "medium", description: "Error text, may embed payloads" }, + { match: "errorMessage", sensitivity: "medium" }, + { match: "*.error_message", sensitivity: "medium" }, + + // -- Raw third-party evidence ----------------------------------------------- + { match: "raw", sensitivity: "critical", description: "Verbatim third-party payload" }, + { match: "*.raw", sensitivity: "critical" }, + { match: "rawPayload", sensitivity: "critical" }, + { match: "request_body", sensitivity: "high", description: "Serialized outbound request body" }, + { match: "requestBody", sensitivity: "high" }, + { match: "request_headers", sensitivity: "high" }, + { match: "requestHeaders", sensitivity: "high" }, + { match: "response_body", sensitivity: "medium", description: "Serialized response body" }, + { match: "responseBody", sensitivity: "medium" }, + + // -- Credentials / secrets --------------------------------------------------- + { match: "secret", sensitivity: "critical" }, + { match: "*.secret", sensitivity: "critical" }, + { match: "password", sensitivity: "critical" }, + { match: "passwordHash", sensitivity: "critical" }, + { match: "token", sensitivity: "critical" }, + { match: "*.token", sensitivity: "critical" }, + { match: "apiKey", sensitivity: "critical" }, + { match: "api_key", sensitivity: "critical" }, + { match: "apikey", sensitivity: "critical" }, + { match: "private_key", sensitivity: "critical" }, + { match: "privateKey", sensitivity: "critical" }, + { match: "signature", sensitivity: "critical" }, + { match: "authorization", sensitivity: "critical" }, + { match: "cookie", sensitivity: "critical" }, + { match: "memo", sensitivity: "medium", description: "Stellar transaction note" }, + + // -- Security / admin -------------------------------------------------------- + { match: "before", sensitivity: "medium", description: "Pre-mutation snapshot" }, + { match: "after", sensitivity: "medium", description: "Post-mutation snapshot" }, +]; + +export class FieldRegistry { + private readonly rules: FieldRule[]; + readonly version: number; + + constructor(rules: FieldRule[] = DEFAULT_FIELD_RULES, version: number = FIELD_REGISTRY_CURRENT_VERSION) { + this.rules = rules; + this.version = version; + } + + get signatures(): FieldRule[] { + return this.rules; + } + + /** + * Classify a field path. Returns undefined when the path does not match any + * classified rule. When multiple rules match, the most specific (longest + * pattern) wins. + */ + classify(path: string): ClassifiedField | undefined { + let best: ClassifiedField | undefined; + for (const rule of this.rules) { + if (!pathMatches(rule.match, path)) continue; + if (!best || rule.match.length > best.rule.match.length) { + best = { rule, path }; + } + } + return best; + } + + /** + * Stable fingerprint covering the rule set. Used so a redaction decision + * can be audited against the exact rules that produced it without storing + * the rules or the values. SHA-256 over the serialized rule set. + */ + fingerprint(): string { + const sorted = [...this.rules].sort((a, b) => (a.match < b.match ? -1 : 1)); + return crypto.createHash("sha256").update(JSON.stringify(sorted)).digest("hex"); + } + + sensitivityLevel(path: string): SensitivityLevel | undefined { + return this.classify(path)?.rule.sensitivity; + } + + /** + * Return the set of sensitive leaf key names across all rules plus common + * snake_case / camelCase variants. Used to derive the flat key list for + * loggers that redact by key name (cheap at runtime, single source of + * truth with the registry). + */ + sensitiveKeyNames(): string[] { + const names = new Set(); + for (const rule of this.rules) { + const segs = rule.match.split("."); + const leaf = segs[segs.length - 1]; + if (leaf === "*" || leaf === "**") continue; + if (leaf.startsWith("~")) continue; + names.add(leaf); + // seed common casing variants + names.add(snakeToCamel(leaf)); + names.add(leaf.replace(/([a-z0-9])([A-Z])/g, "$1_$2").toLowerCase()); + } + return Array.from(names); + } +} + +/** Convert snake_case to camelCase: "api_key" -> "apiKey". */ +function snakeToCamel(value: string): string { + if (!value.includes("_")) return value; + return value.replace(/_([a-z])/g, (_m, c: string) => c.toUpperCase()); +} + +export const fieldRegistry = new FieldRegistry(); diff --git a/backend/src/privacy/index.ts b/backend/src/privacy/index.ts new file mode 100644 index 00000000..e127d065 --- /dev/null +++ b/backend/src/privacy/index.ts @@ -0,0 +1,49 @@ +/** + * Privacy-preserving operational data redaction pipeline. + * + * Public entry point for the redaction module. Wire sinks through these + * exports; the engine classifies fields centrally, applies the sink's policy, + * scans for secrets, and emits versioned decisions. + */ + +export { + redactionService, + RedactionService, + DEFAULT_SINK_POLICIES, + REDACTION_MARKER, + type RedactOptions, +} from "./redaction.service.js"; + +export { + fieldRegistry, + FieldRegistry, + DEFAULT_FIELD_RULES, + FIELD_REGISTRY_CURRENT_VERSION, + pathMatches, + type ClassifiedField, +} from "./fieldRegistry.js"; + +export { + scanValue, + scanString, + DEFAULT_SECRET_PATTERNS, + type SecretMatch, + type SecretPattern, +} from "./secretScanner.js"; + +export { + pseudonymize, + pseudonymizeValue, + hmacDigest, + isPseudonym, + type PseudonymOptions, +} from "./pseudonymizer.js"; + +export { + redactionDecisionService, + RedactionDecisionService, + scrubDecisionForLog, + type DecisionCorrelation, +} from "./redactionDecision.service.js"; + +export * from "./types.js"; diff --git a/backend/src/privacy/pseudonymizer.ts b/backend/src/privacy/pseudonymizer.ts new file mode 100644 index 00000000..48b398ff --- /dev/null +++ b/backend/src/privacy/pseudonymizer.ts @@ -0,0 +1,78 @@ +/** + * Deterministic pseudonymization. + * + * Values are transformed with an HMAC-SHA-256 keyed by a salt that lives in + * configuration (never in the persisted data). The same input always maps to + * the same pseudonym within a namespace, so related records stay correlatable + * across sinks during an investigation, while the original value cannot be + * recovered from the pseudonym. + * + * Separating namespaces (one per distinct payload shape) prevents cross-sink + * correlation outside the intended scope and lets a namespace be rotated + * without disturbing the others. + */ + +import crypto from "crypto"; + +export interface PseudonymOptions { + /** Hex salt. Derived from config; keeps pseudonyms deterministic per deployment. */ + salt: string; + /** Namespace the pseudonym is scoped to. */ + namespace: string; + /** Prefix that makes the output obviously a pseudonym. */ + prefix?: string; + /** Number of hex characters of the HMAC digest to keep (min 16). */ + length?: number; +} + +const DEFAULT_LENGTH = 24; + +/** Compute the HMAC digest for a value under a namespace + salt. */ +export function hmacDigest(value: string, salt: string, namespace: string): string { + return crypto + .createHmac("sha256", salt) + .update(`${namespace}:${value}`) + .digest("hex"); +} + +/** Whether a value already looks like one of our pseudonyms. */ +export function isPseudonym(value: string, prefix = "psn:"): boolean { + return typeof value === "string" && value.startsWith(prefix); +} + +/** + * Deterministically pseudonymize a string. Returns a `psn:`-prefixed value + * that is stable for the same (value, namespace, salt). + */ +export function pseudonymize(value: string, options: PseudonymOptions): string { + const length = Math.max(typeof options.length === "number" ? options.length : DEFAULT_LENGTH, 16); + const prefix = options.prefix ?? "psn:"; + if (isPseudonym(value, prefix)) return value; + const digest = hmacDigest(value, options.salt, options.namespace).slice(0, length); + return `${prefix}${digest}`; +} + +/** + * Walk a value and deterministically pseudonymize every string leaf under a + * namespace. Objects and arrays are cloned so the caller's value is never + * mutated. + */ +export function pseudonymizeValue( + value: unknown, + options: PseudonymOptions, +): unknown { + if (value === null || value === undefined) return value; + if (typeof value === "string") return pseudonymize(value, options); + if (typeof value === "number" || typeof value === "boolean") return value; + if (Array.isArray(value)) { + return value.map((item) => pseudonymizeValue(item, options)); + } + if (typeof value === "object") { + const out: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + out[key] = pseudonymizeValue(item, options); + } + return out; + } + return value; +} diff --git a/backend/src/privacy/redaction.service.ts b/backend/src/privacy/redaction.service.ts new file mode 100644 index 00000000..c16df2ef --- /dev/null +++ b/backend/src/privacy/redaction.service.ts @@ -0,0 +1,375 @@ +/** + * Schema-aware redaction pipeline. + * + * Orchestrates the full flow: walk an operational payload against the central + * field registry, apply the selected sink's policy (keep / redact / + * pseudonymize / obfuscate), run secret detection, and emit a versioned + * decision. The same payload shape therefore gets the same treatment at every + * configured sink, and decisions stay auditable without storing the values. + */ + +import crypto from "crypto"; +import { config } from "../config/index.js"; +import { fieldRegistry as defaultRegistry, pathMatches } from "./fieldRegistry.js"; +import { DEFAULT_SECRET_PATTERNS, scanValue, type SecretPattern } from "./secretScanner.js"; +import { pseudonymizeValue, type PseudonymOptions } from "./pseudonymizer.js"; +import type { + FieldRule, + RedactedResult, + RedactionAction, + RedactionDecision, + RedactionEvent, + SensitivityLevel, + SinkName, + SinkPolicy, +} from "./types.js"; + +export const REDACTION_MARKER = "[REDACTED]"; + +/** + * Default sink policies. A sink's policy decides how each sensitivity level + * is treated and whether secrets are allowed to pass. Sinks that persist + * operational payloads are stricter; sinks that only surface aggregate + * signals are more permissive. + */ +export const DEFAULT_SINK_POLICIES: Record = { + log: { + sink: "log", + version: 1, + enabled: true, + levelActions: { public: "keep", low: "keep", medium: "redact", high: "redact", critical: "redact" }, + pseudonymizeLevels: [], + allowList: [], + blockOnSecret: true, + }, + audit: { + sink: "audit", + version: 1, + enabled: true, + levelActions: { public: "keep", low: "keep", medium: "redact", high: "pseudonymize", critical: "pseudonymize" }, + pseudonymizeLevels: ["high", "critical"], + allowList: ["actorType", "severity", "action", "resourceType"], + blockOnSecret: false, + }, + webhook: { + sink: "webhook", + version: 1, + enabled: true, + levelActions: { public: "keep", low: "keep", medium: "redact", high: "pseudonymize", critical: "redact" }, + pseudonymizeLevels: ["high"], + allowList: ["eventType", "timestamp", "ruleId", "ruleName", "assetCode", "metric", "threshold", "priority", "alertType"], + blockOnSecret: false, + }, + pdf: { + sink: "pdf", + version: 1, + enabled: true, + levelActions: { public: "keep", low: "keep", medium: "redact", high: "redact", critical: "redact" }, + pseudonymizeLevels: [], + allowList: [], + blockOnSecret: true, + }, + csv: { + sink: "csv", + version: 1, + enabled: true, + levelActions: { public: "keep", low: "keep", medium: "redact", high: "pseudonymize", critical: "redact" }, + pseudonymizeLevels: ["high"], + allowList: [], + blockOnSecret: true, + }, + json: { + sink: "json", + version: 1, + enabled: true, + levelActions: { public: "keep", low: "keep", medium: "redact", high: "pseudonymize", critical: "redact" }, + pseudonymizeLevels: ["high"], + allowList: [], + blockOnSecret: true, + }, + websocket: { + sink: "websocket", + version: 1, + enabled: true, + levelActions: { public: "keep", low: "keep", medium: "redact", high: "pseudonymize", critical: "redact" }, + pseudonymizeLevels: ["high"], + allowList: ["type", "topic", "sequence", "timestamp", "priority", "ruleId", "assetCode", "metric", "threshold", "severity"], + blockOnSecret: true, + }, + export: { + sink: "export", + version: 1, + enabled: true, + levelActions: { public: "keep", low: "keep", medium: "redact", high: "pseudonymize", critical: "redact" }, + pseudonymizeLevels: ["high"], + allowList: [], + blockOnSecret: true, + }, +}; + +export interface RedactOptions { + sink: SinkName; + policy?: SinkPolicy; + /** Overrides for the default sink policy (merges level actions/regex). */ + registry?: ReturnType; + secretPatterns?: SecretPattern[]; + /** Namespace for pseudonyms (defaults to config namespace). */ + namespace?: string; +} + +function getDefaultRegistry() { + return defaultRegistry; +} + +export class RedactionService { + private readonly policies: Record; + private readonly registry; + private readonly secretPatterns: SecretPattern[]; + + constructor( + policies: Record = DEFAULT_SINK_POLICIES, + registry = defaultRegistry, + secretPatterns: SecretPattern[] = DEFAULT_SECRET_PATTERNS, + ) { + this.policies = policies; + this.registry = registry; + this.secretPatterns = secretPatterns; + } + + getPolicy(sink: SinkName): SinkPolicy { + return this.policies[sink]; + } + + /** + * Redact a structured payload for a sink. Returns the transformed output + * and a versioned decision. The decision never contains original values. + */ + redact(input: unknown, opts: RedactOptions): RedactedResult { + const policy = opts.policy ?? this.getPolicy(opts.sink); + if (!config.REDACTION_ENABLED || !policy.enabled) { + const decision = this.emptyDecision(policy, opts.sink); + return { output: input, decision }; + } + + const events: RedactionEvent[] = []; + const secretPaths: string[] = []; + const output = this.walk(input, { + path: "", + policy, + events, + secretPaths, + namespace: opts.namespace ?? config.REDACTION_PSEUDONYM_NAMESPACE, + registry: opts.registry ?? this.registry, + secretPatterns: opts.secretPatterns ?? this.secretPatterns, + }); + + const secretDetected = secretPaths.length > 0; + const blocked = secretDetected && policy.blockOnSecret; + + if (blocked) { + throw new Error( + `Redaction blocked payload for sink "${opts.sink}": secret content is not allowed. Paths: ${secretPaths.join(", ")}`, + ); + } + + return { + output, + decision: { + sink: opts.sink, + policyVersion: policy.version, + modified: events.length > 0, + secretDetected, + blocked, + events, + policyFingerprint: this.registryFingerprint(policy), + timestamp: new Date().toISOString(), + }, + }; + } + + /** + * Scan a plain string (e.g. a message or a serialized body) for secrets, + * and if the policy blocks on secrets, throw. Returns the original string + * when allowed. Used when there is no structured object to walk. + */ + classifyString(input: string, opts: { sink: SinkName; policy?: SinkPolicy }): { allowed: boolean; decision: RedactionDecision } { + const policy = opts.policy ?? this.getPolicy(opts.sink); + const matches = scanValue(input, "$", this.secretPatterns); + const secretDetected = matches.length > 0; + const blocked = secretDetected && policy.blockOnSecret; + const events: RedactionEvent[] = secretDetected + ? matches.map((m) => ({ field: "$", action: blocked ? "redact" : "keep", sensitivity: "critical" })) + : []; + return { + allowed: !blocked, + decision: { + sink: opts.sink, + policyVersion: policy.version, + modified: blocked, + secretDetected, + blocked, + events, + policyFingerprint: this.registryFingerprint(policy), + timestamp: new Date().toISOString(), + }, + }; + } + + private emptyDecision(policy: SinkPolicy, sink: SinkName): RedactionDecision { + return { + sink, + policyVersion: policy.version, + modified: false, + secretDetected: false, + blocked: false, + events: [], + policyFingerprint: this.registryFingerprint(policy), + timestamp: new Date().toISOString(), + }; + } + + private registryFingerprint(policy: SinkPolicy): string { + return `${policy.version}:${this.registry.fingerprint()}`; + } + + private walk( + value: unknown, + ctx: { + path: string; + policy: SinkPolicy; + events: RedactionEvent[]; + secretPaths: string[]; + namespace: string; + registry: ReturnType; + secretPatterns: SecretPattern[]; + }, + ): unknown { + if (value === null || value === undefined) return value; + if (typeof value === "string") { + return this.handleString(value, ctx); + } + if (typeof value === "number" || typeof value === "boolean") return value; + if (Array.isArray(value)) { + return value.map((item, idx) => this.walk(item, { ...ctx, path: `${ctx.path}[${idx}]` })); + } + if (typeof value === "object") { + const out: Record = {}; + for (const [key, item] of Object.entries(value as Record)) { + const childPath = ctx.path ? `${ctx.path}.${key}` : key; + out[key] = this.walk(item, { ...ctx, path: childPath }); + } + return out; + } + return value; + } + + private handleString( + value: string, + ctx: { + path: string; + policy: SinkPolicy; + events: RedactionEvent[]; + secretPaths: string[]; + namespace: string; + registry: ReturnType; + secretPatterns: SecretPattern[]; + }, + ): string { + const classified = ctx.registry.classify(ctx.path); + + if (classified && !this.isAllowed(ctx.policy.allowList, ctx.path)) { + const rule = classified.rule; + const action = this.resolveAction(rule, classified.rule.sensitivity, ctx.policy); + const event: RedactionEvent = { + field: ctx.path, + action, + sensitivity: rule.sensitivity, + ruleFingerprint: this.ruleFingerprint(rule), + }; + + if (action === "pseudonymize" || ctx.policy.pseudonymizeLevels.includes(rule.sensitivity)) { + ctx.events.push({ ...event, action: "pseudonymize" }); + return pseudoOf(value, ctx.namespace); + } + if (action === "redact") { + ctx.events.push({ ...event, action: "redact" }); + return REDACTION_MARKER; + } + if (action === "obfuscate") { + ctx.events.push({ ...event, action: "obfuscate" }); + return obfuscate(value); + } + // keep + return value; + } + + // Unclassified string: still run secret detection. If a secret is found, + // respect the policy's secret handling. + const secretMatches = scanValue(value, ctx.path, ctx.secretPatterns); + if (secretMatches.length > 0) { + if (!secretMatches.some((m) => m.path && ctx.policy.allowList.includes(m.path.replace(/^\$\./, "")))) { + ctx.secretPaths.push(ctx.path || "$"); + if (ctx.policy.blockOnSecret) { + return value; // value preserved; the caller walks decision.blocked to reject + } + } + } + + // String values that are serialized JSON may contain nested sensitive + // fields: parse, walk, and re-serialize. + const parsed = tryParseJson(value); + if (parsed !== undefined) { + const walked = this.walk(parsed, { ...ctx, path: ctx.path ? `${ctx.path}.~` : "~" }); + const reserialized = JSON.stringify(walked); + if (reserialized !== value) { + ctx.events.push({ field: ctx.path, action: "redact", sensitivity: "high" }); + return reserialized; + } + } + return value; + } + + private isAllowed(allowList: string[], path: string): boolean { + for (const allow of allowList) { + if (pathMatches(allow, path)) return true; + } + return false; + } + + private resolveAction(rule: FieldRule, level: SensitivityLevel, policy: SinkPolicy): RedactionAction { + if (rule.action) return rule.action; + return policy.levelActions[level] ?? "redact"; + } + + private ruleFingerprint(rule: FieldRule): string { + // Fingerprint identifies the rule without leaking any value. + const payload = `${this.registry.version}:${rule.match}:${rule.sensitivity}`; + return crypto.createHash("sha256").update(payload).digest("hex").slice(0, 16); + } +} + +function pseudoOf(value: string, namespace: string): string { + const opts: PseudonymOptions = { + salt: config.REDACTION_PSEUDONYM_SALT, + namespace, + }; + return pseudonymizeValue(value, opts) as string; +} + +function obfuscate(value: string): string { + if (value.length <= 4) return REDACTION_MARKER; + const head = value.slice(0, 2); + const tail = value.slice(-4); + return `${head}****${tail}`; +} + +function tryParseJson(value: string): unknown | undefined { + if (value.length < 2 || (value[0] !== "{" && value[0] !== "[")) return undefined; + try { + return JSON.parse(value); + } catch { + return undefined; + } +} + +export const redactionService = new RedactionService(); diff --git a/backend/src/privacy/redactionDecision.service.ts b/backend/src/privacy/redactionDecision.service.ts new file mode 100644 index 00000000..7b32a60b --- /dev/null +++ b/backend/src/privacy/redactionDecision.service.ts @@ -0,0 +1,64 @@ +/** + * Persistence layer for redaction decisions. + * + * A redaction decision is written after a payload is processed so that what + * was redacted, under which policy version, and which fields were touched can + * be audited later. The decision is already scrubbed by the redaction engine + * (rule fingerprints + field paths only), so writing it stores no secrets. + * + * Persistence is best-effort: a decision must never break the operational + * path that produced it, so failures here are logged and swallowed. + */ + +import { getDatabase } from "../database/connection.js"; +import { logger } from "../utils/logger.js"; +import { config } from "../config/index.js"; +import type { RedactionDecision } from "./types.js"; + +export interface DecisionCorrelation { + /** Incident / request / delivery id the decision belongs to, if any. */ + incidentId?: string; + /** Resource type this decision was about, if known. */ + resourceType?: string; + /** Resource id this decision was about, if known. */ + resourceId?: string; +} + +export function scrubDecisionForLog(decision: RedactionDecision) { + return { + sink: decision.sink, + policyVersion: decision.policyVersion, + modified: decision.modified, + secretDetected: decision.secretDetected, + blocked: decision.blocked, + eventCount: decision.events.length, + policyFingerprint: decision.policyFingerprint, + }; +} + +export class RedactionDecisionService { + async record(decision: RedactionDecision, correlation: DecisionCorrelation = {}): Promise { + if (!config.REDACTION_DECISION_LOG_ENABLED) return; + try { + const db = getDatabase(); + await db("redaction_decisions").insert({ + sink: decision.sink, + policy_version: decision.policyVersion, + modified: decision.modified, + secret_detected: decision.secretDetected, + blocked: decision.blocked, + policy_fingerprint: decision.policyFingerprint, + events: JSON.stringify(decision.events), + correlation: JSON.stringify(correlation), + created_at: new Date(), + }); + } catch (error) { + logger.warn( + { error: error instanceof Error ? error.message : String(error) }, + "Failed to persist redaction decision", + ); + } + } +} + +export const redactionDecisionService = new RedactionDecisionService(); diff --git a/backend/src/privacy/secretScanner.ts b/backend/src/privacy/secretScanner.ts new file mode 100644 index 00000000..44b0d073 --- /dev/null +++ b/backend/src/privacy/secretScanner.ts @@ -0,0 +1,93 @@ +/** + * Secret detection for the redaction pipeline. + * + * Scans string values (including values that are stringified JSON) for + * high-confidence secret patterns: private keys, seed phrases, bearer / + * JWT tokens, API keys, and identity credentials. Detection is intentionally + * conservative to avoid blocking benign operational data; only well-formed + * patterns that match real secret formats are flagged. + */ + +export interface SecretMatch { + /** Which secret pattern matched. */ + type: string; + /** Human readable description of the matched pattern. */ + description: string; + /** Field path where the secret was found, when scanning structured data. */ + path?: string; +} + +export interface SecretPattern { + type: string; + description: string; + regex: RegExp; +} + +const ED25519_SECRET = /S[A-Z2-7]{54,56}/; +const EVM_PRIVATE_KEY = /0x[0-9a-fA-F]{64}/; +const BITCOIN_WIF = /5[HJK][1-9A-HJ-NP-Za-km-z]{49,51}/; +const STELLAR_MNEMONIC = /\b([a-z]+(?: [a-z]+){11,23})\b/; +const JW_TOKEN = /eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/; +const BEARER = /Bearer\s+[A-Za-z0-9._~+/-]{20,}/i; +const GENERIC_KEY = /(?:api[_-]?key|secret|private[_-]?key|access[_-]?token)\s*[:=]\s*["']?[A-Za-z0-9._~+/-]{16,}["']?/i; +const AWS_ACCESS_KEY = /AKIA[0-9A-Z]{16}/; +const GITHUB_TOKEN = /gh[pousr]_[A-Za-z0-9_]{30,}/; +const STELLAR_SECRET_ISSUER = /G[A-Z2-7]{55}/; + +export const DEFAULT_SECRET_PATTERNS: SecretPattern[] = [ + { type: "stellar_secret", description: "Stellar ed25519 secret key (S...)", regex: ED25519_SECRET }, + { type: "evm_private_key", description: "EVM private key (0x + 64 hex)", regex: EVM_PRIVATE_KEY }, + { type: "bitcoin_wif", description: "Bitcoin WIF private key", regex: BITCOIN_WIF }, + { type: "bip39_mnemonic", description: "BIP-39 seed phrase (12-24 words)", regex: STELLAR_MNEMONIC }, + { type: "jwt", description: "JWT / bearer token", regex: JW_TOKEN }, + { type: "bearer_token", description: "Bearer credential", regex: BEARER }, + { type: "named_secret", description: "Named api key / secret / token assignment", regex: GENERIC_KEY }, + { type: "aws_access_key", description: "AWS access key id", regex: AWS_ACCESS_KEY }, + { type: "github_token", description: "GitHub token", regex: GITHUB_TOKEN }, + { type: "stellar_issuer", description: "Stellar public issuer address (G...)", regex: STELLAR_SECRET_ISSUER }, +]; + +/** + * Detect whether a string contains one or more secrets. Returns a list of + * matches; an empty list means no secret was found. + */ +export function scanString(value: string, patterns: SecretPattern[] = DEFAULT_SECRET_PATTERNS): SecretMatch[] { + const matches: SecretMatch[] = []; + for (const pattern of patterns) { + pattern.regex.lastIndex = 0; + if (pattern.regex.test(value)) { + matches.push({ type: pattern.type, description: pattern.description }); + } + pattern.regex.lastIndex = 0; + } + return matches; +} + +function isSensitiveString(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +/** + * Recursively scan a structured value for secrets. The optional `path` + * tracks the current field path for reporting. + */ +export function scanValue(value: unknown, path = "$", patterns: SecretPattern[] = DEFAULT_SECRET_PATTERNS): SecretMatch[] { + if (value === null || value === undefined) return []; + if (typeof value === "string") { + return scanString(value, patterns).map((m) => (path === "$" ? m : { ...m, path })); + } + if (typeof value === "object") { + const results: SecretMatch[] = []; + if (Array.isArray(value)) { + value.forEach((item, idx) => { + results.push(...scanValue(item, `${path}[${idx}]`, patterns)); + }); + return results; + } + for (const [key, item] of Object.entries(value as Record)) { + results.push(...scanValue(item, `${path}.${key}`, patterns)); + } + return results; + } + return []; +} diff --git a/backend/src/privacy/types.ts b/backend/src/privacy/types.ts new file mode 100644 index 00000000..0e7ecd12 --- /dev/null +++ b/backend/src/privacy/types.ts @@ -0,0 +1,105 @@ +/** + * Privacy redaction pipeline types. + * + * The pipeline classifies fields against a central registry, applies a + * per-sink policy (keep / redact / pseudonymize), runs secret detection, + * and emits a versioned, auditable redaction decision. Decisions record + * what changed and under which policy version, never the original value. + */ + +export type SensitivityLevel = "public" | "low" | "medium" | "high" | "critical"; + +export type RedactionAction = "keep" | "redact" | "pseudonymize" | "obfuscate"; + +export type SinkName = + | "log" + | "audit" + | "webhook" + | "pdf" + | "csv" + | "json" + | "websocket" + | "export"; + +/** + * A single classified field rule. `match` is a field path (dotted) or a + * path pattern. Patterns support the glob suffix `**` (matches any number + * of segments) and a trailing `*` (matches any single segment value at the + * final position, e.g. `metadata.*` covers every immediate child). A plain + * path without wildcards matches exactly. + */ +export interface FieldRule { + /** Dotted field path or path pattern, e.g. "after.source_address". */ + match: string; + /** Central sensitivity classification. */ + sensitivity: SensitivityLevel; + /** + * Optional per-field action that overrides the sink level action. When + * omitted the sink policy decides based on sensitivity. + */ + action?: RedactionAction; + /** Optional free-text description of why this field is classified. */ + description?: string; +} + +/** + * Semantic classification shared by sinks that do not have a field-precise + * contract (JSON export, webhook payloads). Sinks apply this to any key + * whose name matches a sensitive token. + */ +export interface SinkPolicy { + sink: SinkName; + /** Monotonic policy version. Bumped when rules change. */ + version: number; + enabled: boolean; + /** + * Default action per sensitivity level when a classified field has no + * explicit `action`. + */ + levelActions: Record; + /** Levels that should always be pseudonymized, regardless of level action. */ + pseudonymizeLevels: SensitivityLevel[]; + /** + * Keys this sink keeps verbatim even if secret detection flags them. + */ + allowList: string[]; + /** Throw when a secret is detected (sink refuses the payload). */ + blockOnSecret: boolean; +} + +export interface RedactionEvent { + /** Field path that was acted on. */ + field: string; + /** Action applied. */ + action: RedactionAction; + /** Central sensitivity of the field, if classified. */ + sensitivity?: SensitivityLevel; + /** SHA-256 fingerprint of the matched rule (identity, not the value). */ + ruleFingerprint?: string; +} + +export interface RedactionDecision { + /** Sink that produced this decision. */ + sink: SinkName; + /** Policy version that applied. */ + policyVersion: number; + /** Whether the payload was modified. */ + modified: boolean; + /** Whether a secret was detected. */ + secretDetected: boolean; + /** Whether a secret caused the payload to be blocked. */ + blocked: boolean; + /** Ordered list of redaction events. */ + events: RedactionEvent[]; + /** Policy rule set fingerprint for audit (version + rules hash). */ + policyFingerprint: string; + /** ISO timestamp. */ + timestamp: string; +} + +export interface RedactedResult { + /** The redacted payload. */ + output: T; + /** Machine-readable decision for audit / telemetry. */ + decision: RedactionDecision; +} diff --git a/backend/src/services/audit.service.ts b/backend/src/services/audit.service.ts index 6b252df7..afe00c12 100644 --- a/backend/src/services/audit.service.ts +++ b/backend/src/services/audit.service.ts @@ -1,6 +1,10 @@ import crypto from "crypto"; import { getDatabase } from "../database/connection.js"; import { logger } from "../utils/logger.js"; +import { config } from "../config/index.js"; +import { redactionService } from "../privacy/redaction.service.js"; +import { redactionDecisionService } from "../privacy/redactionDecision.service.js"; +import type { RedactionDecision } from "../privacy/types.js"; // ============================================================================= // TYPES @@ -157,29 +161,38 @@ export class AuditService { severity: params.severity ?? this.inferSeverity(params.action), }; - const checksum = this.computeChecksum(draft); + // Redact sensitive operational fields (IP, addresses, notes, evidence) + // before the checksum is computed and before persistence, so the stored + // values are already scrubbed and the checksum covers the scrubbed state. + const redaction = this.applyRedaction(draft); + + const checksum = this.computeChecksum(redaction.draft); const [row] = await db("audit_logs") .insert({ id: crypto.randomUUID(), - action: draft.action, - actor_id: draft.actorId, - actor_type: draft.actorType, - ip_address: draft.ipAddress, - user_agent: draft.userAgent, - resource_type: draft.resourceType, - resource_id: draft.resourceId, - before: draft.before ? JSON.stringify(draft.before) : null, - after: draft.after ? JSON.stringify(draft.after) : null, - metadata: JSON.stringify(draft.metadata), - severity: draft.severity, + action: redaction.draft.action, + actor_id: redaction.draft.actorId, + actor_type: redaction.draft.actorType, + ip_address: redaction.draft.ipAddress, + user_agent: redaction.draft.userAgent, + resource_type: redaction.draft.resourceType, + resource_id: redaction.draft.resourceId, + before: redaction.draft.before ? JSON.stringify(redaction.draft.before) : null, + after: redaction.draft.after ? JSON.stringify(redaction.draft.after) : null, + metadata: JSON.stringify(redaction.draft.metadata), + severity: redaction.draft.severity, checksum, created_at: new Date(), }) .returning("*"); + if (redaction.decision) { + this.recordRedactionDecision(redaction.decision, params.resourceType ?? null, params.resourceId ?? null); + } + logger.info( - { auditId: row.id, action: draft.action, actorId: draft.actorId, severity: draft.severity }, + { auditId: row.id, action: redaction.draft.action, actorId: redaction.draft.actorId, severity: redaction.draft.severity }, "Audit event recorded" ); @@ -338,6 +351,49 @@ export class AuditService { // HELPERS // --------------------------------------------------------------------------- + private applyRedaction(draft: { + action: AuditAction; + actorId: string; + actorType: "user" | "api_key" | "system" | "admin"; + ipAddress: string | null; + userAgent: string | null; + resourceType: string | null; + resourceId: string | null; + before: Record | null; + after: Record | null; + metadata: Record; + severity: AuditSeverity; + }): { draft: typeof draft; decision: RedactionDecision | null } { + if (!config.REDACTION_ENABLED) { + return { draft, decision: null }; + } + + const composite = { + actorId: draft.actorId, + ipAddress: draft.ipAddress, + userAgent: draft.userAgent, + resourceId: draft.resourceId, + before: draft.before, + after: draft.after, + metadata: draft.metadata, + }; + + const result = redactionService.redact(composite, { sink: "audit" }); + const red = result.output as typeof composite; + + return { + draft: { ...draft, ...red }, + decision: result.decision, + }; + } + + private recordRedactionDecision(decision: RedactionDecision, resourceType: string | null, resourceId: string | null): void { + void redactionDecisionService.record(decision, { + resourceType: resourceType ?? undefined, + resourceId: resourceId ?? undefined, + }); + } + private inferSeverity(action: AuditAction): AuditSeverity { if ( action === "admin.user_permission_changed" || diff --git a/backend/src/services/webhook.service.outbox.ts b/backend/src/services/webhook.service.outbox.ts index 72636464..49f7a6cb 100644 --- a/backend/src/services/webhook.service.outbox.ts +++ b/backend/src/services/webhook.service.outbox.ts @@ -3,6 +3,8 @@ import { randomBytes } from "crypto"; import { getDatabase } from "../database/connection.js"; import { logger } from "../utils/logger.js"; import { OutboxProducer } from "../outbox/eventProducer.js"; +import { redactionService } from "../privacy/redaction.service.js"; +import { redactionDecisionService } from "../privacy/redactionDecision.service.js"; // Import existing types from original webhook service import type { @@ -43,6 +45,8 @@ export class OutboxWebhookService { throw new Error(`Rate limit exceeded for webhook endpoint: ${params.webhookEndpointId}`); } + const payload = this.redactWebhookPayload(params.payload); + // Create delivery record in webhook_deliveries table (existing behavior) const deliveryId = crypto.randomUUID(); const [delivery] = await tx("webhook_deliveries") @@ -50,7 +54,7 @@ export class OutboxWebhookService { id: deliveryId, webhook_endpoint_id: params.webhookEndpointId, event_type: params.eventType, - payload: JSON.stringify(params.payload), + payload: JSON.stringify(payload), status: "pending", attempts: 0, created_at: new Date(), @@ -66,7 +70,7 @@ export class OutboxWebhookService { deliveryId, webhookEndpointId: params.webhookEndpointId, eventType: params.eventType, - payload: params.payload, + payload, url: endpoint.url, secret: endpoint.secret, customHeaders: endpoint.customHeaders, @@ -111,12 +115,13 @@ export class OutboxWebhookService { } const batchId = crypto.randomUUID(); + const events = params.events.map((event) => this.redactWebhookPayload(event)); const batchPayload = { batch: true, batchId, eventType: params.eventType, - count: params.events.length, - events: params.events, + count: events.length, + events, timestamp: new Date().toISOString(), }; @@ -350,6 +355,12 @@ export class OutboxWebhookService { }; } + private redactWebhookPayload(payload: Record): Record { + const result = redactionService.redact(payload, { sink: "webhook" }); + void redactionDecisionService.record(result.decision); + return result.output as Record; + } + private mapToDelivery(row: any): WebhookDelivery { return { id: row.id, diff --git a/backend/src/services/webhook.service.ts b/backend/src/services/webhook.service.ts index f64948e7..1a9c9137 100644 --- a/backend/src/services/webhook.service.ts +++ b/backend/src/services/webhook.service.ts @@ -6,6 +6,8 @@ import { Queue, Job, ConnectionOptions } from "bullmq"; import { config } from "../config/index.js"; import { WebhookBatchBufferService } from "./webhookBatchBuffer.service.js"; import type { BatchBufferStatus } from "./webhookBatchBuffer.service.js"; +import { redactionService } from "../privacy/redaction.service.js"; +import { redactionDecisionService } from "../privacy/redactionDecision.service.js"; const fetch = globalThis.fetch; @@ -715,6 +717,8 @@ export class WebhookService extends EventEmitter { }): Promise { const db = getDatabase(); + const payload = this.redactWebhookPayload(params.payload); + const endpoint = await this.getEndpoint(params.webhookEndpointId); if (!endpoint) { throw new Error(`Webhook endpoint not found: ${params.webhookEndpointId}`); @@ -731,7 +735,7 @@ export class WebhookService extends EventEmitter { id: crypto.randomUUID(), webhook_endpoint_id: params.webhookEndpointId, event_type: params.eventType, - payload: JSON.stringify(params.payload), + payload: JSON.stringify(payload), status: "buffered", attempts: 0, created_at: new Date(), @@ -742,7 +746,7 @@ export class WebhookService extends EventEmitter { endpointId: params.webhookEndpointId, deliveryId: delivery.id, eventType: params.eventType, - payload: params.payload, + payload, windowMs: endpoint.batchWindowMs, }); @@ -760,7 +764,7 @@ export class WebhookService extends EventEmitter { id: crypto.randomUUID(), webhook_endpoint_id: params.webhookEndpointId, event_type: params.eventType, - payload: JSON.stringify(params.payload), + payload: JSON.stringify(payload), status: "pending", attempts: 0, created_at: new Date(), @@ -771,7 +775,7 @@ export class WebhookService extends EventEmitter { deliveryId: delivery.id, webhookEndpointId: params.webhookEndpointId, eventType: params.eventType, - payload: params.payload, + payload, attemptNumber: 0, }; @@ -834,11 +838,13 @@ export class WebhookService extends EventEmitter { throw new Error("Batch delivery is not enabled for this endpoint"); } + const events = params.events.map((event) => this.redactWebhookPayload(event)); + const batchPayload = { batch: true, eventType: params.eventType, - count: params.events.length, - events: params.events, + count: events.length, + events, timestamp: new Date().toISOString(), }; @@ -871,6 +877,17 @@ export class WebhookService extends EventEmitter { return [this.mapToDelivery(delivery)]; } + /** + * Redact a webhook payload through the webhook sink policy and record the + * resulting decision. Runs before persistence and queueing so every stored + * and transmitted copy (deliveries, job data, logs, outbox) is scrubbed. + */ + private redactWebhookPayload(payload: Record): Record { + const result = redactionService.redact(payload, { sink: "webhook" }); + void redactionDecisionService.record(result.decision); + return result.output as Record; + } + public async processDelivery(job: Job): Promise<{ status: number; body: string }> { const { deliveryId, webhookEndpointId, eventType, payload } = job.data; diff --git a/backend/src/services/websocket.ts b/backend/src/services/websocket.ts index 63d80b4a..af062782 100644 --- a/backend/src/services/websocket.ts +++ b/backend/src/services/websocket.ts @@ -1,5 +1,7 @@ import { randomUUID } from "crypto"; import { config } from "../config/index.js"; +import { redactionService } from "../privacy/redaction.service.js"; +import { redactionDecisionService } from "../privacy/redactionDecision.service.js"; const MAX_HISTORY_PER_TOPIC = config.WS_MAX_HISTORY_PER_TOPIC; const MAX_HISTORY_AGE_MS = config.WS_MAX_HISTORY_AGE_MS; @@ -264,6 +266,13 @@ export class WebsocketService { payload: unknown, options: WebsocketPublishOptions = {}, ): void { + // Redact operational payloads before they enter replay history or are + // delivered, so addresses, hashes, and evidence are not exposed over the + // socket or replayed on reconnect. + const result = redactionService.redact(payload, { sink: "websocket" }); + void redactionDecisionService.record(result.decision); + const safePayload = result.output; + const timestamp = options.timestamp ?? new Date().toISOString(); const createdAtMs = Date.parse(timestamp); const expiresAtMs = @@ -275,7 +284,7 @@ export class WebsocketService { type, topic, priority: options.priority ?? "medium", - payload, + payload: safePayload, timestamp, expiresAt: new Date(expiresAtMs).toISOString(), ackRequired: options.ackRequired ?? false, diff --git a/backend/src/utils/logger.ts b/backend/src/utils/logger.ts index 5dbd6381..a549aff1 100644 --- a/backend/src/utils/logger.ts +++ b/backend/src/utils/logger.ts @@ -1,9 +1,15 @@ import os from "os"; import pino from "pino"; import { config } from "../config/index.js"; +import { fieldRegistry } from "../privacy/fieldRegistry.js"; type LogMeta = Record; +// Extend pino's exact-key redaction with the fields classified as sensitive by +// the central registry. Deriving from the registry keeps the log sink's +// enforcement tied to the same source of truth as every other sink. +const REGISTRY_SENSITIVE_KEYS = fieldRegistry.sensitiveKeyNames(); + export interface FlexibleLogger { trace: (...args: unknown[]) => void; debug: (...args: unknown[]) => void; @@ -88,6 +94,7 @@ const baseConfig = { // Custom redaction for sensitive fields redact: { paths: [ + ...REGISTRY_SENSITIVE_KEYS, 'password', 'token', 'secret', diff --git a/backend/src/workers/export.worker.ts b/backend/src/workers/export.worker.ts index 0bdae607..0001684c 100644 --- a/backend/src/workers/export.worker.ts +++ b/backend/src/workers/export.worker.ts @@ -9,6 +9,8 @@ import { sendExportEmail } from "../utils/email.js"; import { CSVHandler } from "../services/formatHandlers/csv.handler.js"; import { JSONHandler } from "../services/formatHandlers/json.handler.js"; import { PDFHandler } from "../services/formatHandlers/pdf.handler.js"; +import { redactionService } from "../privacy/redaction.service.js"; +import { redactionDecisionService } from "../privacy/redactionDecision.service.js"; import type { ExportJobPayload } from "../types/export.types.js"; import { getDatabase } from "../database/connection.js"; import path from "path"; @@ -48,6 +50,21 @@ function getFileExtension(format: string, compressed: boolean): string { return format; } +/** + * Wrap a record stream so every exported row passes through the export + * redaction policy before it reaches a format handler. Each decision is + * recorded for audit (fingerprints only; no original values). + */ +async function* redactExportStream( + source: AsyncGenerator, +): AsyncGenerator { + for await (const record of source) { + const result = redactionService.redact(record, { sink: "export" }); + void redactionDecisionService.record(result.decision); + yield result.output; + } +} + /** * Export Queue Worker * @@ -84,8 +101,10 @@ export const exportWorker = new Worker( const fileName = `${payload.exportId}.${extension}`; const filePath = path.join(config.EXPORT_STORAGE_PATH, fileName); - // Stream data from database - const dataStream = streamData(payload.dataType, payload.filters); + // Stream data from database, redacting operational fields through the + // export policy before any format handler (PDF, CSV, JSON) serializes a + // row, so the persisted artifact never contains sensitive material. + const dataStream = redactExportStream(streamData(payload.dataType, payload.filters)); // Generate output using appropriate format handler let outputStream: NodeJS.ReadableStream; diff --git a/backend/tests/services/privacyRedaction.service.test.ts b/backend/tests/services/privacyRedaction.service.test.ts new file mode 100644 index 00000000..d01c8e37 --- /dev/null +++ b/backend/tests/services/privacyRedaction.service.test.ts @@ -0,0 +1,303 @@ +import { describe, it, expect } from "vitest"; +import { + fieldRegistry, + FieldRegistry, + pathMatches, + DEFAULT_FIELD_RULES, + redactionService, + RedactionService, + DEFAULT_SINK_POLICIES, + REDACTION_MARKER, + scanValue, + scanString, + pseudonymize, + pseudonymizeValue, + isPseudonym, + scrubDecisionForLog, +} from "../../src/privacy/index.js"; +import { redactionDecisionService } from "../../src/privacy/redactionDecision.service.js"; +import type { RedactionDecision, SinkPolicy } from "../../src/privacy/types.js"; + +describe("fieldRegistry.pathMatches", () => { + it("matches exact dotted paths", () => { + expect(pathMatches("after.source_address", "after.source_address")).toBe(true); + expect(pathMatches("after.source_address", "after.destination_address")).toBe(false); + }); + + it("matches single-segment wildcard *", () => { + expect(pathMatches("*.source_address", "data.source_address")).toBe(true); + expect(pathMatches("*.source_address", "data.nested.source_address")).toBe(false); + }); + + it("matches ** suffix across any depth", () => { + expect(pathMatches("transactions.**", "transactions")).toBe(true); + expect(pathMatches("transactions.**", "transactions.0.source_address")).toBe(true); + expect(pathMatches("transactions.**", "other.source_address")).toBe(false); + }); +}); + +describe("FieldRegistry", () => { + it("classifies sensitive paths centrally", () => { + const classified = fieldRegistry.classify("after.source_address"); + expect(classified).toBeDefined(); + expect(classified?.rule.sensitivity).toBe("critical"); + }); + + it("classifies nested evidence paths", () => { + expect(fieldRegistry.sensitivityLevel("data.raw")).toBe("critical"); + expect(fieldRegistry.sensitivityLevel("transaction.source_address")).toBe("critical"); + expect(fieldRegistry.sensitivityLevel("ownerAddress")).toBe("critical"); + }); + + it("most specific rule wins", () => { + const registry = new FieldRegistry([ + { match: "**", sensitivity: "medium" }, + { match: "account.address", sensitivity: "critical" }, + ]); + expect(registry.classify("account.address")?.rule.sensitivity).toBe("critical"); + expect(registry.classify("account.other")?.rule.sensitivity).toBe("medium"); + }); + + it("fingerprint is stable and changes with rules", () => { + const a = new FieldRegistry(DEFAULT_FIELD_RULES, 1).fingerprint(); + const b = new FieldRegistry(DEFAULT_FIELD_RULES, 1).fingerprint(); + const c = new FieldRegistry([{ match: "x", sensitivity: "low" }], 1).fingerprint(); + expect(a).toBe(b); + expect(a).not.toBe(c); + }); +}); + +describe("secretScanner", () => { + it("detects a Stellar ed25519 secret key", () => { + const matches = scanString("SDR5FQGCNVP5KU2YGW5IOKRRRRMPKXA5ZQ3XO7Q4N7GNRYVUZROXXJUVK"); + expect(matches.some((m) => m.type === "stellar_secret")).toBe(true); + }); + + it("detects an EVM private key", () => { + const matches = scanValue({ + privateKey: "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + }); + expect(matches.some((m) => m.type === "evm_private_key")).toBe(true); + }); + + it("detects a BIP-39 seed phrase", () => { + const phrase = + "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about"; + const matches = scanString(phrase); + expect(matches.some((m) => m.type === "bip39_mnemonic")).toBe(true); + }); + + it("detects a JWT / bearer token", () => { + const jwt = + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"; + expect(scanString(jwt).some((m) => m.type === "jwt")).toBe(true); + }); + + it("detects named secret assignments", () => { + const matches = scanString("apiKey = 'sk-abcdef0123456789'"); + expect(matches.some((m) => m.type === "named_secret")).toBe(true); + }); + + it("does not flag benign operational text", () => { + const matches = scanString("Bridge USDC supply increased by 12.4% this week"); + expect(matches).toEqual([]); + }); + + it("returns empty when no secret is present", () => { + expect(scanValue({ symbol: "USDC", price: 1.0 })).toEqual([]); + }); +}); + +describe("pseudonymizer", () => { + const opts = { salt: "test-salt", namespace: "incident-1" }; + + it("is deterministic for the same value + namespace", () => { + const a = pseudonymize("GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN", opts); + const b = pseudonymize("GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN", opts); + expect(a).toBe(b); + expect(a.startsWith("psn:")).toBe(true); + }); + + it("differs across namespaces for the same value", () => { + const a = pseudonymize("GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN", opts); + const b = pseudonymize("GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN", { ...opts, namespace: "incident-2" }); + expect(a).not.toBe(b); + }); + + it("is irreversible (pseudonym is a truncated digest, not the value)", () => { + const value = "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"; + const result = pseudonymize(value, opts); + expect(result).not.toContain(value); + }); + + it("preserves shape when walking structured values", () => { + const out = pseudonymizeValue( + { from: "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN", amount: 100 }, + opts, + ); + expect(isPseudonym(out.from)).toBe(true); + expect(out.amount).toBe(100); + }); +}); + +describe("RedactionService - audit policy", () => { + it("pseudonymizes high/critical classified fields (addresses)", () => { + const result = redactionService.redact( + { actorId: "user-1", after: { source_address: "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN" } }, + { sink: "audit" }, + ); + const out = result.output as any; + expect(out.after.source_address.startsWith("psn:")).toBe(true); + expect(result.decision.modified).toBe(true); + }); + + it("keeps benign audit metadata", () => { + const result = redactionService.redact( + { action: "auth.login", severity: "info", actorType: "user", before: {} }, + { sink: "audit" }, + ); + expect(result.output).toEqual({ + action: "auth.login", + severity: "info", + actorType: "user", + before: {}, + }); + }); + + it("redacts free-text notes at persistence", () => { + const result = redactionService.redact( + { metadata: { reason: "manual override for account remediation" } }, + { sink: "audit" }, + ); + expect(result.output.metadata.reason).toBe(REDACTION_MARKER); + }); +}); + +describe("RedactionService - webhook policy", () => { + it("redacts critical fields and pseudonymizes high fields", () => { + const result = redactionService.redact( + { + ruleId: "r1", + assetCode: "USDC", + ownerAddress: "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN", + memo: "release team", + }, + { sink: "webhook" }, + ); + const out = result.output as any; + expect(out.ownerAddress).toBe(REDACTION_MARKER); // critical -> redact + expect(out.ruleId).toBe("r1"); // allowlisted + expect(out.assetCode).toBe("USDC"); + }); +}); + +describe("RedactionService - sink-specific policies", () => { + it("applies different actions per sink for the same payload", () => { + const payload = { source_address: "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN" }; + const audit = redactionService.redact(payload, { sink: "audit" }).output as any; + const webhook = redactionService.redact(payload, { sink: "webhook" }).output as any; + const pdf = redactionService.redact(payload, { sink: "pdf" }).output as any; + expect(audit.source_address.startsWith("psn:")).toBe(true); // audit pseudonymizes critical? no -> redact + expect(webhook.source_address).toBe(REDACTION_MARKER); + expect(pdf.source_address).toBe(REDACTION_MARKER); + }); +}); + +describe("RedactionService - secret blocking", () => { + it("blocks payloads that contain secrets when the sink blocks on secret", () => { + const policy: SinkPolicy = { + ...DEFAULT_SINK_POLICIES.export, + blockOnSecret: true, + sink: "export", + }; + expect(() => + redactionService.redact( + { evidence: "key material SDR5FQGCNVP5KU2YGW5IOKRRRRMPKXA5ZQ3XO7Q4N7GNRYVUZROXXJUVK in the bundle" }, + { sink: "export", policy }, + ), + ).toThrow(); + }); + + it("records secretDetected without throwing when the sink does not block", () => { + const result = redactionService.redact( + { evidence: "witness statement, verify 0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" }, + { sink: "audit" }, + ); + expect(result.decision.secretDetected).toBe(true); + expect(result.decision.blocked).toBe(false); + }); +}); + +describe("RedactionService - decisions are auditable without secrets", () => { + it("decision events carry fingerprints and paths, never original values", () => { + const secret = "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"; + const result = redactionService.redact( + { after: { source_address: secret } }, + { sink: "audit" }, + ); + const serialized = JSON.stringify(result.decision); + expect(serialized).not.toContain(secret); + expect(serialized).not.toContain("GA5ZSEJ"); + const ev = result.decision.events.find((e) => e.field === "after.source_address"); + expect(ev).toBeDefined(); + expect(ev!.ruleFingerprint).toBeTruthy(); + }); + + it("decisions are versioned with a policy fingerprint", () => { + const result = redactionService.redact({ after: { source_address: "x" } }, { sink: "audit" }); + expect(result.decision.policyVersion).toBe(DEFAULT_SINK_POLICIES.audit.version); + expect(result.decision.policyFingerprint).toContain(String(DEFAULT_SINK_POLICIES.audit.version)); + }); +}); + +describe("RedactionService - nested / stringified JSON", () => { + it("walks and redacts sensitive fields inside a stringified JSON body", () => { + const body = JSON.stringify({ requestor: "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN" }); + const result = redactionService.redact({ request_body: body }, { sink: "webhook" }); + const out = result.output as any; + expect(out.request_body).not.toContain("GA5ZSEJ"); + }); + + it("redacts obfuscate-mode fields in place (shape preserved)", () => { + const customService = new RedactionService({ + ...DEFAULT_SINK_POLICIES, + csv: { + ...DEFAULT_SINK_POLICIES.csv, + levelActions: { public: "keep", low: "keep", medium: "obfuscate", high: "redact", critical: "redact" }, + }, + }); + const result = customService.redact({ metadata: { reason: "a reasonably long note" } }, { sink: "csv" }); + expect(result.output.metadata.reason).toBe("a ****note"); + }); +}); + +describe("RedactionService - disabled", () => { + it("passes payload through untouched when a sink policy is disabled", () => { + const disabledPolicy: SinkPolicy = { ...DEFAULT_SINK_POLICIES.audit, enabled: false }; + const result = redactionService.redact( + { after: { source_address: "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN" } }, + { sink: "audit", policy: disabledPolicy }, + ); + expect(result.output.after.source_address).toBe("GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"); + expect(result.decision.modified).toBe(false); + }); +}); + +describe("redactionDecisionService.log scrub", () => { + it("scrubDecisionForLog exposes only safe telemetry", () => { + const decision: RedactionDecision = { + sink: "audit", + policyVersion: 1, + modified: true, + secretDetected: false, + blocked: false, + events: [{ field: "after.source_address", action: "redact" }], + policyFingerprint: "1:abc", + timestamp: new Date().toISOString(), + }; + const scrubbed = scrubDecisionForLog(decision); + expect(scrubbed.eventCount).toBe(1); + expect(scrubbed.sink).toBe("audit"); + expect(JSON.stringify(scrubbed)).not.toContain("source_address"); + }); +});