From bef8ed6bcdc5e141011df1f8a1957f2dceb96e01 Mon Sep 17 00:00:00 2001 From: Matteo Date: Fri, 2 Oct 2026 16:05:41 +0200 Subject: [PATCH] Shared outbound fetch helper with connect-time SSRF checks fetchOutbound() downloads a third-party URL for a user: every hop (the URL and each redirect target) passes the SSRF guard, the socket connects only to addresses the guard checked, no env proxy, only the caller's headers (Authorization/Cookie dropped on a cross-origin redirect), one deadline for the whole exchange, a byte cap on Content-Length and on the bytes read, non-2xx as errors, and URLs without query strings in messages. ssrf.util gains ssrfGuardedLookup() and createSsrfGuardedAgents() so other callers can close the same gaps; assertSafeOutboundHost keeps its behaviour and messages. Needed by #645 (multipart file parts fetched from a URL). --- .../src/common/outbound-fetch.util.spec.ts | 256 ++++++++++++++++ .../backend/src/common/outbound-fetch.util.ts | 290 ++++++++++++++++++ packages/backend/src/common/ssrf.util.ts | 92 +++++- 3 files changed, 631 insertions(+), 7 deletions(-) create mode 100644 packages/backend/src/common/outbound-fetch.util.spec.ts create mode 100644 packages/backend/src/common/outbound-fetch.util.ts diff --git a/packages/backend/src/common/outbound-fetch.util.spec.ts b/packages/backend/src/common/outbound-fetch.util.spec.ts new file mode 100644 index 00000000..0f6ecae4 --- /dev/null +++ b/packages/backend/src/common/outbound-fetch.util.spec.ts @@ -0,0 +1,256 @@ +import * as dns from 'dns'; +import * as http from 'http'; +import { AddressInfo } from 'net'; +import { + fetchOutbound, + OutboundFetchError, + redactUrl, +} from './outbound-fetch.util'; +import { SsrfBlockedError, ssrfGuardedLookup } from './ssrf.util'; + +// The guard is on, and `localhost` stands in for a public host: it is the only +// allowlisted name. Literal 127.0.0.1 stays blocked, like an internal address. +const ENV = { SSRF_GUARD: 'enabled', SSRF_ALLOWED_HOSTS: 'localhost' } as NodeJS.ProcessEnv; + +type Handler = (req: http.IncomingMessage, res: http.ServerResponse) => void; + +interface TestServer { + port: number; + hits: http.IncomingMessage[]; + close: () => Promise; +} + +async function serve(handler: Handler): Promise { + const hits: http.IncomingMessage[] = []; + const server = http.createServer((req, res) => { + hits.push(req); + handler(req, res); + }); + await new Promise((ok) => server.listen(0, '127.0.0.1', ok)); + return { + port: (server.address() as AddressInfo).port, + hits, + close: () => + new Promise((ok) => { + server.closeAllConnections(); + server.close(() => ok()); + }), + }; +} + +describe('fetchOutbound', () => { + const servers: TestServer[] = []; + const start = async (h: Handler) => { + const s = await serve(h); + servers.push(s); + return s; + }; + + afterEach(async () => { + jest.restoreAllMocks(); + await Promise.all(servers.splice(0).map((s) => s.close())); + }); + + it('returns the body, headers and final URL', async () => { + const s = await start((_req, res) => { + res.writeHead(200, { 'Content-Type': 'image/png' }); + res.end('PNGDATA'); + }); + const out = await fetchOutbound(`http://localhost:${s.port}/a.png`, { env: ENV }); + expect(out.status).toBe(200); + expect(out.body.toString()).toBe('PNGDATA'); + expect(out.headers['content-type']).toBe('image/png'); + expect(out.finalUrl).toBe(`http://localhost:${s.port}/a.png`); + }); + + it('sends only the caller headers: no cookie, no authorization', async () => { + const s = await start((_req, res) => res.end('ok')); + await fetchOutbound(`http://localhost:${s.port}/`, { + env: ENV, + headers: { Accept: 'image/*' }, + }); + const sent = s.hits[0].headers; + expect(sent.accept).toBe('image/*'); + expect(sent.authorization).toBeUndefined(); + expect(sent.cookie).toBeUndefined(); + }); + + it('follows a redirect to an allowed host', async () => { + const s = await start((req, res) => { + if (req.url === '/old') { + res.writeHead(302, { Location: '/new' }); + return res.end(); + } + res.end('moved here'); + }); + const out = await fetchOutbound(`http://localhost:${s.port}/old`, { env: ENV }); + expect(out.body.toString()).toBe('moved here'); + expect(out.finalUrl).toBe(`http://localhost:${s.port}/new`); + }); + + it('refuses a redirect to an internal address and never contacts it', async () => { + const internal = await start((_req, res) => res.end('INTERNAL')); + const s = await start((_req, res) => { + res.writeHead(302, { Location: `http://127.0.0.1:${internal.port}/meta` }); + res.end(); + }); + await expect( + fetchOutbound(`http://localhost:${s.port}/`, { env: ENV }), + ).rejects.toBeInstanceOf(SsrfBlockedError); + expect(internal.hits).toHaveLength(0); + }); + + it('checks the address the socket connects to, not an earlier DNS answer', async () => { + const internal = await start((_req, res) => res.end('INTERNAL')); + // A rebinding name: public when the URL is checked, loopback when the + // client connects. + const answers = [ + [{ address: '93.184.216.34', family: 4 }], + [{ address: '127.0.0.1', family: 4 }], + ]; + const real = dns.promises.lookup; + jest.spyOn(dns.promises, 'lookup').mockImplementation(((host: string, opts: any) => + host === 'rebind.test' + ? Promise.resolve(answers.shift() ?? [{ address: '127.0.0.1', family: 4 }]) + : real(host, opts)) as any); + + await expect( + fetchOutbound(`http://rebind.test:${internal.port}/`, { env: ENV }), + ).rejects.toThrow(/non-public address '127\.0\.0\.1'/); + expect(internal.hits).toHaveLength(0); + }); + + it('refuses a redirect to a non-http scheme', async () => { + const s = await start((_req, res) => { + res.writeHead(302, { Location: 'file:///etc/passwd' }); + res.end(); + }); + await expect( + fetchOutbound(`http://localhost:${s.port}/`, { env: ENV }), + ).rejects.toMatchObject({ reason: 'invalid_url' }); + }); + + it('stops after maxRedirects', async () => { + const s = await start((_req, res) => { + res.writeHead(302, { Location: '/again' }); + res.end(); + }); + await expect( + fetchOutbound(`http://localhost:${s.port}/`, { env: ENV, maxRedirects: 2 }), + ).rejects.toMatchObject({ reason: 'too_many_redirects' }); + expect(s.hits).toHaveLength(3); + }); + + it('drops Authorization and Cookie on a redirect to another origin', async () => { + const other = await start((_req, res) => res.end('other')); + const s = await start((_req, res) => { + res.writeHead(302, { Location: `http://localhost:${other.port}/` }); + res.end(); + }); + await fetchOutbound(`http://localhost:${s.port}/`, { + env: ENV, + headers: { Authorization: 'Bearer t', Cookie: 'a=b', Accept: 'text/plain' }, + }); + expect(s.hits[0].headers.authorization).toBe('Bearer t'); + const sent = other.hits[0].headers; + expect(sent.authorization).toBeUndefined(); + expect(sent.cookie).toBeUndefined(); + expect(sent.accept).toBe('text/plain'); + }); + + it('treats a non-2xx answer as an error and keeps the query string out of it', async () => { + const s = await start((_req, res) => { + res.writeHead(403, { 'Content-Type': 'text/html' }); + res.end('

Forbidden

'); + }); + const err = await fetchOutbound( + `http://localhost:${s.port}/f.pdf?X-Amz-Signature=secret`, + { env: ENV }, + ).catch((e) => e); + expect(err).toBeInstanceOf(OutboundFetchError); + expect(err).toMatchObject({ reason: 'status', status: 403 }); + expect(err.message).toContain('/f.pdf'); + expect(err.message).not.toContain('secret'); + }); + + it('rejects a body whose Content-Length is over the cap without reading it', async () => { + const s = await start((_req, res) => { + res.writeHead(200, { 'Content-Length': '2048' }); + res.end(Buffer.alloc(2048)); + }); + await expect( + fetchOutbound(`http://localhost:${s.port}/`, { env: ENV, maxBytes: 1024 }), + ).rejects.toMatchObject({ reason: 'too_large' }); + }); + + it('aborts a streamed body as soon as it crosses the cap', async () => { + let written = 0; + const s = await start((_req, res) => { + // Chunked, no Content-Length: the cap must hold on the bytes read. + res.writeHead(200); + const timer = setInterval(() => { + written += 512; + res.write(Buffer.alloc(512)); + if (written >= 64 * 1024) { + clearInterval(timer); + res.end(); + } + }, 1); + res.on('close', () => clearInterval(timer)); + }); + await expect( + fetchOutbound(`http://localhost:${s.port}/`, { env: ENV, maxBytes: 2048 }), + ).rejects.toMatchObject({ reason: 'too_large' }); + expect(written).toBeLessThan(64 * 1024); + }); + + it('gives up at the deadline', async () => { + const s = await start(() => { + /* never answers */ + }); + await expect( + fetchOutbound(`http://localhost:${s.port}/`, { env: ENV, timeoutMs: 200 }), + ).rejects.toMatchObject({ reason: 'timeout' }); + }); + + it('ignores an env proxy', async () => { + const s = await start((_req, res) => res.end('direct')); + const saved = process.env.HTTP_PROXY; + process.env.HTTP_PROXY = 'http://127.0.0.1:9'; + try { + const out = await fetchOutbound(`http://localhost:${s.port}/`, { env: ENV }); + expect(out.body.toString()).toBe('direct'); + } finally { + if (saved === undefined) delete process.env.HTTP_PROXY; + else process.env.HTTP_PROXY = saved; + } + }); +}); + +describe('ssrfGuardedLookup', () => { + const lookup = (host: string, env: NodeJS.ProcessEnv) => + new Promise((resolve, reject) => + ssrfGuardedLookup(env)(host, { all: true }, (err, addrs) => + err ? reject(err) : resolve(addrs), + ), + ); + + it('blocks a loopback name that is not allowlisted', async () => { + await expect( + lookup('localhost', { SSRF_GUARD: 'enabled' } as NodeJS.ProcessEnv), + ).rejects.toBeInstanceOf(SsrfBlockedError); + }); + + it('resolves an allowlisted name', async () => { + const addrs = (await lookup('localhost', ENV)) as Array<{ address: string }>; + expect(addrs.length).toBeGreaterThan(0); + }); +}); + +describe('redactUrl', () => { + it('keeps origin and path only', () => { + expect(redactUrl('https://user:pw@cdn.example.com/a/b.png?sig=x#f')).toBe( + 'https://cdn.example.com/a/b.png', + ); + }); +}); diff --git a/packages/backend/src/common/outbound-fetch.util.ts b/packages/backend/src/common/outbound-fetch.util.ts new file mode 100644 index 00000000..e8ad5568 --- /dev/null +++ b/packages/backend/src/common/outbound-fetch.util.ts @@ -0,0 +1,290 @@ +import axios, { AxiosResponse } from 'axios'; +import { Readable } from 'stream'; +import { + assertSafeOutboundUrl, + createSsrfGuardedAgents, + SsrfBlockedError, +} from './ssrf.util'; + +/** + * Download a third-party URL on a user's behalf: a file to attach to a + * request, a spec to import, anything whose address comes from a user or a + * model rather than from the connector's own configuration. + * + * What it guarantees, so callers don't each re-implement it: + * - SSRF: every hop (the URL and each redirect target) passes + * assertSafeOutboundUrl, and the socket connects only to addresses the + * guard checked (ssrfGuardedLookup), so neither a redirect nor a DNS + * answer that changes after the check reaches an internal address. + * - A bare request: only the headers the caller passes, no env proxy, no + * cookies. Never pass connector credentials here; on a redirect to + * another origin, Authorization and Cookie are dropped anyway. + * - Bounded: one deadline for the whole exchange (redirects and body + * included) and a byte cap, enforced on Content-Length up front and on + * the bytes actually read. The body is buffered, so it can be sent again + * (e.g. on a 401 retry). + * - Non-2xx answers are errors, so an error page never passes as content. + * - Errors name the URL without its query string: presigned links carry + * their credentials there. + */ + +export const OUTBOUND_FETCH_DEFAULTS = { + maxBytes: 10 * 1024 * 1024, + timeoutMs: 30_000, + maxRedirects: 5, +}; + +export interface OutboundFetchOptions { + /** Byte cap on the (decompressed) body. Default 10 MB. */ + maxBytes?: number; + /** Deadline for the whole exchange, redirects included. Default 30 s. */ + timeoutMs?: number; + /** Redirects to follow; each target is checked again. Default 5. */ + maxRedirects?: number; + /** Request headers. Nothing else is sent besides axios' own defaults. */ + headers?: Record; + /** Policy env, for tests. Defaults to process.env. */ + env?: NodeJS.ProcessEnv; +} + +export interface OutboundFetchResult { + status: number; + /** Response headers, lower-cased names. */ + headers: Record; + body: Buffer; + /** The URL that answered, after redirects. */ + finalUrl: string; +} + +export type OutboundFetchFailure = + | 'invalid_url' + | 'status' + | 'too_large' + | 'timeout' + | 'too_many_redirects' + | 'network'; + +export class OutboundFetchError extends Error { + constructor( + message: string, + readonly reason: OutboundFetchFailure, + readonly status?: number, + ) { + super(message); + this.name = 'OutboundFetchError'; + } +} + +/** `origin + path` of a URL, for messages and logs: no query, no fragment, no userinfo. */ +export function redactUrl(url: string): string { + try { + const u = new URL(url); + return `${u.origin}${u.pathname}`; + } catch { + return '(invalid URL)'; + } +} + +const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]); +const CROSS_ORIGIN_DROPPED_HEADERS = new Set(['authorization', 'cookie']); + +/** + * GET `url` with the guarantees described above. Throws SsrfBlockedError when + * a hop is not allowed (so callers can keep showing the allowlist hint) and + * OutboundFetchError for everything else. + */ +export async function fetchOutbound( + url: string, + options: OutboundFetchOptions = {}, +): Promise { + const maxBytes = options.maxBytes ?? OUTBOUND_FETCH_DEFAULTS.maxBytes; + const timeoutMs = options.timeoutMs ?? OUTBOUND_FETCH_DEFAULTS.timeoutMs; + const maxRedirects = options.maxRedirects ?? OUTBOUND_FETCH_DEFAULTS.maxRedirects; + const env = options.env ?? process.env; + const { httpAgent, httpsAgent } = createSsrfGuardedAgents(env); + let headers: Record = { ...(options.headers ?? {}) }; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + let current = url; + try { + for (let redirects = 0; ; redirects++) { + await assertHop(current, env); + const res = await request(current, headers, controller.signal, httpAgent, httpsAgent); + const stream = res.data as Readable; + + const location = res.headers['location']; + if (REDIRECT_STATUSES.has(res.status) && typeof location === 'string') { + stream.destroy(); + if (redirects >= maxRedirects) { + throw new OutboundFetchError( + `${redactUrl(url)} redirected more than ${maxRedirects} times.`, + 'too_many_redirects', + ); + } + let next: string; + try { + next = new URL(location, current).toString(); + } catch { + throw new OutboundFetchError( + `${redactUrl(current)} redirected to an invalid URL.`, + 'invalid_url', + ); + } + if (new URL(next).origin !== new URL(current).origin) { + headers = Object.fromEntries( + Object.entries(headers).filter( + ([k]) => !CROSS_ORIGIN_DROPPED_HEADERS.has(k.toLowerCase()), + ), + ); + } + current = next; + continue; + } + + if (res.status < 200 || res.status >= 300) { + stream.destroy(); + throw new OutboundFetchError( + `${redactUrl(current)} answered HTTP ${res.status}.`, + 'status', + res.status, + ); + } + + const declared = Number(res.headers['content-length']); + if (Number.isFinite(declared) && declared > maxBytes) { + stream.destroy(); + throw tooLarge(current, maxBytes); + } + const body = await readWithLimit(stream, maxBytes, current, controller.signal); + return { + status: res.status, + headers: flattenHeaders(res.headers), + body, + finalUrl: current, + }; + } + } catch (err) { + if (err instanceof OutboundFetchError || err instanceof SsrfBlockedError) throw err; + // The guarded lookup rejects inside the HTTP client; surface its error. + const cause = (err as { cause?: unknown })?.cause; + if (cause instanceof SsrfBlockedError) throw cause; + if (controller.signal.aborted) { + throw new OutboundFetchError( + `${redactUrl(current)} did not answer within ${Math.round(timeoutMs / 1000)} s.`, + 'timeout', + ); + } + const code = (err as { code?: string })?.code; + throw new OutboundFetchError( + `Could not fetch ${redactUrl(current)}${code ? ` (${code})` : ''}.`, + 'network', + ); + } finally { + clearTimeout(timer); + httpAgent.destroy(); + httpsAgent.destroy(); + } +} + +async function assertHop(url: string, env: NodeJS.ProcessEnv): Promise { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw new OutboundFetchError('Not a valid URL.', 'invalid_url'); + } + // Checked here as well, because the guard itself can be disabled by env. + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new OutboundFetchError( + `Only http and https URLs can be fetched, not '${parsed.protocol}'.`, + 'invalid_url', + ); + } + await assertSafeOutboundUrl(url, env); +} + +function request( + url: string, + headers: Record, + signal: AbortSignal, + httpAgent: unknown, + httpsAgent: unknown, +): Promise { + return axios.request({ + url, + method: 'GET', + headers, + responseType: 'stream', + // Redirects are followed by hand above, so each target is checked. + maxRedirects: 0, + // An env proxy would resolve the proxy instead of the target and skip + // the guarded lookup. + proxy: false, + httpAgent, + httpsAgent, + signal, + validateStatus: () => true, + }); +} + +function readWithLimit( + stream: Readable, + maxBytes: number, + url: string, + signal: AbortSignal, +): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let total = 0; + let settled = false; + const finish = (err: Error | null) => { + if (settled) return; + settled = true; + signal.removeEventListener('abort', onAbort); + stream.removeListener('data', onData); + if (err) { + stream.destroy(); + reject(err); + } else { + resolve(Buffer.concat(chunks, total)); + } + }; + const onData = (chunk: Buffer) => { + total += chunk.length; + if (total > maxBytes) return finish(tooLarge(url, maxBytes)); + chunks.push(chunk); + }; + const onAbort = () => + finish( + new OutboundFetchError(`${redactUrl(url)} did not finish in time.`, 'timeout'), + ); + if (signal.aborted) return onAbort(); + signal.addEventListener('abort', onAbort); + stream.on('data', onData); + stream.once('end', () => finish(null)); + stream.once('error', (e) => finish(e)); + }); +} + +function tooLarge(url: string, maxBytes: number): OutboundFetchError { + return new OutboundFetchError( + `${redactUrl(url)} is larger than ${formatBytes(maxBytes)}.`, + 'too_large', + ); +} + +function formatBytes(n: number): string { + if (n >= 1024 * 1024) return `${Math.round((n / (1024 * 1024)) * 10) / 10} MB`; + if (n >= 1024) return `${Math.round(n / 1024)} KB`; + return `${n} bytes`; +} + +function flattenHeaders(raw: AxiosResponse['headers']): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(raw ?? {})) { + if (v === undefined || v === null) continue; + out[k.toLowerCase()] = Array.isArray(v) ? v.join(', ') : String(v); + } + return out; +} diff --git a/packages/backend/src/common/ssrf.util.ts b/packages/backend/src/common/ssrf.util.ts index 9d47e197..60ed4da1 100644 --- a/packages/backend/src/common/ssrf.util.ts +++ b/packages/backend/src/common/ssrf.util.ts @@ -1,5 +1,7 @@ -import { promises as dns } from 'dns'; -import { isIP } from 'net'; +import { promises as dns, LookupAddress } from 'dns'; +import * as http from 'http'; +import * as https from 'https'; +import { isIP, LookupFunction } from 'net'; /** * SSRF guard for outbound HTTP/S calls performed on behalf of users. @@ -178,20 +180,31 @@ export async function assertSafeOutboundHost( ): Promise { const policy = readPolicy(env); if (!policy.enabled) return; + await vetHost(hostname, policy); +} +/** + * Apply the policy to `hostname`. Returns null when the host is allowlisted + * (env or DB), otherwise the addresses it resolves to, every one of them + * checked. Throws SsrfBlockedError when the host is not allowed. + */ +async function vetHost( + hostname: string, + policy: SsrfPolicy, +): Promise { if (!hostname) { throw new SsrfBlockedError('SSRF guard: empty hostname'); } // Env-driven allowlist (synchronous). - if (hostMatchesAllowlist(hostname, policy.allowedHosts)) return; + if (hostMatchesAllowlist(hostname, policy.allowedHosts)) return null; // DB-driven allowlist (admin-configured, async). The provider caches // internally so this is effectively a Map lookup after the first call. if (dbAllowedHostsProvider) { try { const dbHosts = await dbAllowedHostsProvider(); - if (hostMatchesAllowlist(hostname, dbHosts)) return; + if (hostMatchesAllowlist(hostname, dbHosts)) return null; } catch { // Provider failure: fall through to IP-based checks rather than // hard-failing every outbound call. @@ -199,13 +212,14 @@ export async function assertSafeOutboundHost( } // If the host is already a literal IP, check it directly. - if (isIP(hostname)) { + const literalFamily = isIP(hostname); + if (literalFamily) { if (!isPublicIp(hostname, policy)) { throw new SsrfBlockedError( `SSRF guard: address '${hostname}' is not a public IP`, ); } - return; + return [{ address: hostname, family: literalFamily }]; } // Block 'localhost' and friends explicitly — DNS may not resolve them @@ -222,7 +236,7 @@ export async function assertSafeOutboundHost( ); } - let resolved: { address: string; family: number }[]; + let resolved: LookupAddress[]; try { resolved = await dns.lookup(hostname, { all: true }); } catch (e: any) { @@ -238,6 +252,70 @@ export async function assertSafeOutboundHost( ); } } + return resolved; +} + +/** + * A `lookup` for http(s).Agent that applies the guard when the socket + * connects, and connects only to the addresses it checked. + * + * {@link assertSafeOutboundUrl} on its own checks the URL once, before the + * request: the HTTP client then resolves the name again (a DNS answer can + * change in between) and follows redirects to hosts nobody checked. With this + * lookup the check and the connection use the same answer, on every hop. + * + * Node does not call `lookup` for a literal IP, so callers must still run + * {@link assertSafeOutboundUrl} on each URL they request (including every + * redirect target): that is what covers `http://169.254.169.254/`. + */ +export function ssrfGuardedLookup( + env: NodeJS.ProcessEnv = process.env, +): LookupFunction { + return (hostname, options, callback) => { + const policy = readPolicy(env); + const vetted = policy.enabled + ? vetHost(hostname, policy) + : Promise.resolve(null); + vetted + .then((addrs) => addrs ?? dns.lookup(hostname, { all: true })) + .then((addrs) => { + const wanted = + options?.family === 4 || options?.family === 'IPv4' + ? 4 + : options?.family === 6 || options?.family === 'IPv6' + ? 6 + : 0; + const usable = wanted + ? addrs.filter((a) => a.family === wanted) + : addrs; + if (usable.length === 0) { + const err: NodeJS.ErrnoException = new Error( + `getaddrinfo ENOTFOUND ${hostname}`, + ); + err.code = 'ENOTFOUND'; + throw err; + } + if (options?.all) callback(null, usable); + else callback(null, usable[0].address, usable[0].family); + }) + .catch((err) => callback(err, '', 0)); + }; +} + +/** + * http and https agents whose connections go through {@link ssrfGuardedLookup}. + * Pass both to axios (`httpAgent`, `httpsAgent`) together with `proxy: false`: + * an env proxy would make the agent resolve the proxy, not the target. + */ +export function createSsrfGuardedAgents(env: NodeJS.ProcessEnv = process.env): { + httpAgent: http.Agent; + httpsAgent: https.Agent; +} { + const lookup = ssrfGuardedLookup(env); + return { + httpAgent: new http.Agent({ lookup }), + httpsAgent: new https.Agent({ lookup }), + }; } /**