Feat/data api (#522)

.env.example

@@ -31,3 +31,7 @@ OPENAI_BASE_URL=xxx
 GOOGLE_CLIENT_ID=xxx
 GOOGLE_CLIENT_SECRET=xxx
 GOOGLE_HOSTED_DOMAIN=
+
+# Extraction APIs
+EXTRACT_API_ALLOWED_IPS=127.0.0.1,::1
+EXTRACT_API_KEY=dev-extract-key-for-testing-only

.gitignore

@@ -23,3 +23,6 @@ out
 # Ignore environment variables.
 *.env*
 !.env.example
+
+# test
+coverage/*

src/lib/server/middleware/index.test.ts

@@ -0,0 +1,202 @@
+import type { RequestHandler } from '@sveltejs/kit';
+import { beforeEach, describe, expect, test, vi } from 'vitest';
+
+const { fallbackChildLogger, fallbackLogger } = vi.hoisted(() => {
+  const childLogger = { warn: vi.fn(), error: vi.fn() };
+  const logger = { child: vi.fn().mockReturnValue(childLogger) };
+
+  return {
+    fallbackChildLogger: childLogger,
+    fallbackLogger: logger,
+  };
+});
+
+vi.mock('$env/dynamic/private', () => ({
+  env: new Proxy(
+    {},
+    {
+      get: (_target, prop) => process.env[prop as string],
+    },
+  ),
+}));
+
+vi.mock('$lib/server/logger.js', () => ({
+  logger: fallbackLogger,
+}));
+
+import {
+  composeMiddleware,
+  type Middleware,
+  withInternalApiKey,
+  withIpWhitelist,
+} from './index.js';
+
+function makeEvent({
+  headers = {},
+  clientIp = '127.0.0.1',
+  includeRequestLogger = true,
+}: {
+  headers?: Record<string, string>;
+  clientIp?: string;
+  includeRequestLogger?: boolean;
+} = {}) {
+  const childLogger = { warn: vi.fn(), error: vi.fn() };
+  const requestLogger = {
+    warn: vi.fn(),
+    error: vi.fn(),
+    child: vi.fn().mockReturnValue(childLogger),
+  };
+
+  return {
+    request: new Request('http://localhost/api/extract/test', { headers }),
+    getClientAddress: () => clientIp,
+    url: new URL('http://localhost/api/extract/test'),
+    locals: includeRequestLogger ? { logger: requestLogger } : {},
+  } as never;
+}
+
+beforeEach(() => {
+  vi.unstubAllEnvs();
+  fallbackLogger.child.mockClear();
+  fallbackChildLogger.warn.mockReset();
+  fallbackChildLogger.error.mockReset();
+});
+
+describe('composeMiddleware', () => {
+  test('composes in expected order', async () => {
+    const calls: string[] = [];
+
+    const m1: Middleware = (handler) =>
+      (async (event) => {
+        calls.push('m1:before');
+        const response = await handler(event);
+        calls.push('m1:after');
+        return response;
+      }) satisfies RequestHandler;
+
+    const m2: Middleware = (handler) =>
+      (async (event) => {
+        calls.push('m2:before');
+        const response = await handler(event);
+        calls.push('m2:after');
+        return response;
+      }) satisfies RequestHandler;
+
+    const handler: RequestHandler = async () => {
+      calls.push('handler');
+      return new Response(null, { status: 204 });
+    };
+
+    const composed = composeMiddleware([m1, m2])(handler);
+    const response = await composed(makeEvent());
+
+    expect(response.status).toBe(204);
+    expect(calls).toEqual(['m1:before', 'm2:before', 'handler', 'm2:after', 'm1:after']);
+  });
+});
+
+describe('withInternalApiKey', () => {
+  test('returns 500 when EXTRACT_API_KEY is not configured (fallback logger path)', async () => {
+    const wrapped = withInternalApiKey(async () => new Response(null, { status: 204 }));
+
+    const response = await wrapped(makeEvent({ includeRequestLogger: false }));
+    const body = await response.json();
+
+    expect(response.status).toBe(500);
+    expect(body).toEqual({ message: 'Internal API key is not configured.' });
+    expect(fallbackLogger.child).toHaveBeenCalled();
+    expect(fallbackChildLogger.error).toHaveBeenCalledWith('EXTRACT_API_KEY is not configured');
+  });
+
+  test('returns 401 when API key header is missing', async () => {
+    vi.stubEnv('EXTRACT_API_KEY', 'test-api-key');
+    const handler = vi.fn(async () => new Response(null, { status: 204 }));
+    const wrapped = withInternalApiKey(handler as never);
+
+    const response = await wrapped(makeEvent());
+    const body = await response.json();
+
+    expect(response.status).toBe(401);
+    expect(body).toEqual({ message: 'Unauthorized' });
+    expect(handler).not.toHaveBeenCalled();
+  });
+
+  test('returns 401 when API key length differs', async () => {
+    vi.stubEnv('EXTRACT_API_KEY', 'test-api-key');
+    const handler = vi.fn(async () => new Response(null, { status: 204 }));
+    const wrapped = withInternalApiKey(handler as never);
+
+    const response = await wrapped(makeEvent({ headers: { 'x-api-key': 'short' } }));
+    const body = await response.json();
+
+    expect(response.status).toBe(401);
+    expect(body).toEqual({ message: 'Unauthorized' });
+    expect(handler).not.toHaveBeenCalled();
+  });
+
+  test('returns 401 when API key has same length but wrong value', async () => {
+    vi.stubEnv('EXTRACT_API_KEY', 'test-api-key');
+    const handler = vi.fn(async () => new Response(null, { status: 204 }));
+    const wrapped = withInternalApiKey(handler as never);
+
+    const response = await wrapped(makeEvent({ headers: { 'x-api-key': 'test-api-kex' } }));
+    const body = await response.json();
+
+    expect(response.status).toBe(401);
+    expect(body).toEqual({ message: 'Unauthorized' });
+    expect(handler).not.toHaveBeenCalled();
+  });
+
+  test('calls handler when API key is valid', async () => {
+    vi.stubEnv('EXTRACT_API_KEY', 'test-api-key');
+    const handler = vi.fn(async () => new Response(JSON.stringify({ ok: true }), { status: 200 }));
+    const wrapped = withInternalApiKey(handler as never);
+
+    const response = await wrapped(makeEvent({ headers: { 'x-api-key': 'test-api-key' } }));
+    const body = await response.json();
+
+    expect(response.status).toBe(200);
+    expect(body).toEqual({ ok: true });
+    expect(handler).toHaveBeenCalledOnce();
+  });
+});
+
+describe('withIpWhitelist', () => {
+  test('returns 500 when EXTRACT_API_ALLOWED_IPS is not configured', async () => {
+    const handler = vi.fn(async () => new Response(null, { status: 204 }));
+    const wrapped = withIpWhitelist(handler as never);
+
+    const response = await wrapped(makeEvent());
+    const body = await response.json();
+
+    expect(response.status).toBe(500);
+    expect(body).toEqual({ message: 'IP whitelist is not configured.' });
+    expect(handler).not.toHaveBeenCalled();
+  });
+
+  test('returns 403 when client IP is not in whitelist', async () => {
+    vi.stubEnv('EXTRACT_API_ALLOWED_IPS', '127.0.0.1,::1');
+    const handler = vi.fn(async () => new Response(null, { status: 204 }));
+    const wrapped = withIpWhitelist(handler as never);
+
+    const response = await wrapped(makeEvent({ clientIp: '10.0.0.1' }));
+    const body = await response.json();
+
+    expect(response.status).toBe(403);
+    expect(body).toEqual({ message: 'Forbidden' });
+    expect(handler).not.toHaveBeenCalled();
+  });
+
+  test('calls handler when client IP is in whitelist (trimmed list)', async () => {
+    vi.stubEnv('EXTRACT_API_ALLOWED_IPS', ' 127.0.0.1 , ::1 ');
+    const handler = vi.fn(async () => new Response(JSON.stringify({ ok: true }), { status: 200 }));
+    const wrapped = withIpWhitelist(handler as never);
+
+    const response = await wrapped(makeEvent({ clientIp: '::1' }));
+    const body = await response.json();
+
+    expect(response.status).toBe(200);
+    expect(body).toEqual({ ok: true });
+    expect(handler).toHaveBeenCalledOnce();
+  });
+});

src/lib/server/middleware/index.ts

@@ -0,0 +1,81 @@
+import { timingSafeEqual } from 'node:crypto';
+
+import type { RequestHandler } from '@sveltejs/kit';
+import { json } from '@sveltejs/kit';
+
+import { env } from '$env/dynamic/private';
+import { logger as baseLogger } from '$lib/server/logger.js';
+
+const API_KEY_HEADER = 'x-api-key';
+
+export type Middleware = (handler: RequestHandler) => RequestHandler;
+
+export function composeMiddleware(middlewares: Middleware[]) {
+  return (handler: RequestHandler) =>
+    middlewares.reduceRight((currentHandler, middleware) => middleware(currentHandler), handler);
+}
+
+function getRequestLogger(event: Parameters<RequestHandler>[0], middleware: string) {
+  return event.locals.logger?.child({ middleware }) ?? baseLogger.child({ middleware });
+}
+
+function getInternalApiKey() {
+  return env.EXTRACT_API_KEY ?? '';
+}
+
+function isValidApiKey(apiKey: string, expectedApiKey: string) {
+  if (apiKey.length !== expectedApiKey.length) {
+    return false;
+  }
+
+  return timingSafeEqual(Buffer.from(apiKey), Buffer.from(expectedApiKey));
+}
+
+export function withInternalApiKey(handler: RequestHandler): RequestHandler {
+  return async (event) => {
+    const logger = getRequestLogger(event, 'withInternalApiKey');
+    const expectedApiKey = getInternalApiKey();
+
+    if (!expectedApiKey) {
+      logger.error('EXTRACT_API_KEY is not configured');
+      return json({ message: 'Internal API key is not configured.' }, { status: 500 });
+    }
+
+    const apiKey = event.request.headers.get(API_KEY_HEADER);
+    if (!apiKey || !isValidApiKey(apiKey, expectedApiKey)) {
+      logger.warn(
+        { method: event.request.method, path: event.url.pathname },
+        'Internal API request rejected: invalid or missing API key',
+      );
+      return json({ message: 'Unauthorized' }, { status: 401 });
+    }
+
+    return handler(event);
+  };
+}
+
+export function withIpWhitelist(handler: RequestHandler): RequestHandler {
+  return async (event) => {
+    const logger = getRequestLogger(event, 'withIpWhitelist');
+    const allowedIps = (env.EXTRACT_API_ALLOWED_IPS ?? '')
+      .split(',')
+      .map((ip) => ip.trim())
+      .filter(Boolean);
+
+    if (allowedIps.length === 0) {
+      logger.error('EXTRACT_API_ALLOWED_IPS is not configured');
+      return json({ message: 'IP whitelist is not configured.' }, { status: 500 });
+    }
+
+    const clientIp = event.getClientAddress();
+    if (!allowedIps.includes(clientIp)) {
+      logger.warn(
+        { clientIp, method: event.request.method, path: event.url.pathname },
+        'Internal API request rejected: IP not in whitelist',
+      );
+      return json({ message: 'Forbidden' }, { status: 403 });
+    }
+
+    return handler(event);
+  };
+}

src/routes/(main)/api/extract/collections/+server.ts

@@ -0,0 +1,90 @@
+import type { RequestHandler } from '@sveltejs/kit';
+import { json } from '@sveltejs/kit';
+
+import type { CollectionFindManyArgs } from '$lib/server/db.js';
+import { db } from '$lib/server/db.js';
+import {
+  composeMiddleware,
+  withInternalApiKey,
+  withIpWhitelist,
+} from '$lib/server/middleware/index.js';
+
+import {
+  buildPagination,
+  buildUpdatedAtFilters,
+  buildUpdatedAtWhere,
+  DEFAULT_PAGE,
+  DEFAULT_PAGE_SIZE,
+  MAX_PAGE_SIZE,
+  parseDate,
+  parsePositiveInteger,
+} from '../helpers.js';
+
+const getCollectionsApi: RequestHandler = async (event) => {
+  const logger = event.locals.logger.child({ handler: 'api_extract_collections' });
+
+  const pageResult = parsePositiveInteger(event.url.searchParams.get('page'), DEFAULT_PAGE, 'page');
+  if ('error' in pageResult) {
+    return json({ message: pageResult.error }, { status: 400 });
+  }
+
+  const pageSizeResult = parsePositiveInteger(
+    event.url.searchParams.get('pageSize'),
+    DEFAULT_PAGE_SIZE,
+    'pageSize',
+  );
+  if ('error' in pageSizeResult) {
+    return json({ message: pageSizeResult.error }, { status: 400 });
+  }
+
+  const lastUpdatedStartResult = parseDate(
+    event.url.searchParams.get('lastUpdatedStart'),
+    'lastUpdatedStart',
+  );
+  if ('error' in lastUpdatedStartResult) {
+    return json({ message: lastUpdatedStartResult.error }, { status: 400 });
+  }
+
+  const lastUpdatedEndResult = parseDate(
+    event.url.searchParams.get('lastUpdatedEnd'),
+    'lastUpdatedEnd',
+  );
+  if ('error' in lastUpdatedEndResult) {
+    return json({ message: lastUpdatedEndResult.error }, { status: 400 });
+  }
+
+  const page = pageResult.value;
+  const pageSize = Math.min(pageSizeResult.value, MAX_PAGE_SIZE);
+  const skip = (page - 1) * pageSize;
+
+  const where = buildUpdatedAtWhere(lastUpdatedStartResult.value, lastUpdatedEndResult.value);
+
+  const collectionArgs = {
+    where,
+    orderBy: {
+      updatedAt: 'asc',
+    },
+    skip,
+    take: pageSize,
+  } satisfies CollectionFindManyArgs;
+
+  try {
+    const [collections, totalCount] = await Promise.all([
+      db.collection.findMany(collectionArgs),
+      db.collection.count({ where }),
+    ]);
+
+    return json({
+      data: collections,
+      pagination: buildPagination(page, pageSize, totalCount),
+      filters: buildUpdatedAtFilters(lastUpdatedStartResult.value, lastUpdatedEndResult.value),
+    });
+  } catch (err) {
+    logger.error({ err }, 'Failed to extract collections');
+    return json({ message: 'Failed to extract collections.' }, { status: 500 });
+  }
+};
+
+export const GET: RequestHandler = composeMiddleware([withIpWhitelist, withInternalApiKey])(
+  getCollectionsApi,
+);