feat: live structured server-log console (closes #178)

backend/package.json

@@ -22,7 +22,8 @@
     "cookie-parser": "^1.4.7",
     "cors": "^2.8.5",
     "dotenv": "^17.2.3",
-    "express": "^4.18.2"
+    "express": "^4.18.2",
+    "rotating-file-stream": "^2.1.6"
   },
   "devDependencies": {
     "@types/cookie-parser": "^1.4.10",

backend/src/accessLog.ts

@@ -0,0 +1,35 @@
+// Express access-log middleware (#178 Phase 2).
+//
+// Emits exactly one structured entry per handled request through the Phase-1
+// structured logger (`log.*`), so the admin Server Logs panel fills with real
+// traffic — on success too, not just error branches. Level tracks status:
+// >=500 error, >=400 warn, else info.
+//
+// SECURITY (load-bearing): logs `method path status durationMs` ONLY. It uses
+// `req.path` — never `req.originalUrl`/query string, never headers, never the
+// body — so the `X-Admin-Key` header, the `POST /api/admin/login` body
+// password, and any `?token=`/`?key=` query value cannot reach the
+// admin-readable (and, from Phase 3, disk-persisted) ring. "Path only" is the
+// redaction mechanism by construction.
+
+import type { Request, Response, NextFunction } from 'express';
+import { log } from './log';
+
+function levelFor(status: number): 'info' | 'warn' | 'error' {
+  if (status >= 500) return 'error';
+  if (status >= 400) return 'warn';
+  return 'info';
+}
+
+export function accessLog(req: Request, res: Response, next: NextFunction): void {
+  const start = process.hrtime.bigint();
+  // `finish` fires once when the response has been fully flushed — for normal
+  // requests at completion, for a long-lived SSE stream only at disconnect
+  // (one entry, large duration; it issues no request, so it cannot loop).
+  res.once('finish', () => {
+    const ms = Number(process.hrtime.bigint() - start) / 1e6;
+    const msg = `${req.method} ${req.path} ${res.statusCode} ${ms.toFixed(1)}ms`;
+    log[levelFor(res.statusCode)](msg);
+  });
+  next();
+}

backend/src/app.ts

@@ -1,11 +1,14 @@
+import { Readable, pipeline } from 'node:stream';
 import express from 'express';
 import cors from 'cors';
 import cookieParser from 'cookie-parser';
 import { tryParseModuleId } from '@highfive/contracts';
 import type { ServerLogsResponse } from '@highfive/contracts';
 import { db } from './database';
 import { verifyApiKey, getApiKey } from './auth';
-import { getRecentLogLines } from './logRing';
+import { accessLog } from './accessLog';
+import { getRecentEntries } from './logRing';
+import { streamBackendRing, writeSseHeaders } from './logStream';
 import {
   SESSION_COOKIE,
   issueSessionToken,
@@ -57,6 +60,12 @@ app.use(cors(corsOptions));
 app.use(express.json());
 app.use(cookieParser());
 
+// Access logging (#178): one structured entry per request into the admin log
+// ring. Mounted here so it wraps every route below (health + public + admin).
+// Logs method+path+status+duration only — never headers/body/query — so no
+// secret can reach the ring. See accessLog.ts.
+app.use(accessLog);
+
 // Health check (public, no auth required)
 app.get('/api/health', (req, res) => {
   res.json({ status: 'ok', timestamp: new Date().toISOString() });
@@ -604,8 +613,8 @@ app.get('/api/admin/logs', requireAdmin, async (req, res) => {
     : LOG_LINES_DEFAULT;
 
   if (service === 'backend') {
-    const { lines: out, truncated } = getRecentLogLines(lines);
-    const payload: ServerLogsResponse = { service: 'backend', lines: out, truncated };
+    const { entries, truncated } = getRecentEntries(lines);
+    const payload: ServerLogsResponse = { service: 'backend', entries, truncated };
     res.json(payload);
     return;
   }
@@ -626,7 +635,12 @@ app.get('/api/admin/logs', requireAdmin, async (req, res) => {
     // Don't forward a drifted wire shape typed as valid: a service that
     // changed its /logs envelope should surface as a clear 502, not as
     // `undefined` fields reaching the UI.
-    if (!payload || typeof payload.service !== 'string' || !Array.isArray(payload.lines)) {
+    if (
+      !payload ||
+      typeof payload.service !== 'string' ||
+      !Array.isArray(payload.entries) ||
+      typeof payload.truncated !== 'boolean'
+    ) {
       res.status(502).json({ error: `malformed logs response from ${service}` });
       return;
     }
@@ -636,3 +650,61 @@ app.get('/api/admin/logs', requireAdmin, async (req, res) => {
     res.status(502).json({ error: `${service} unreachable` });
   }
 });
+
+// SSE live tail (#178 Phase 4). One `LogEntry` JSON per `data:` event. The panel
+// fetches GET /api/admin/logs once for backfill, then opens this for live tail.
+// `backend` streams its own ring; the two Flask services are piped from their
+// internal `/logs/stream` (X-Admin-Key forwarded). See ADR-023 / logStream.ts.
+app.get('/api/admin/logs/stream', requireAdmin, async (req, res) => {
+  const service = String(req.query.service ?? '');
+  if (!(LOG_SERVICES as readonly string[]).includes(service)) {
+    res.status(400).json({
+      error: `invalid service; expected one of: ${LOG_SERVICES.join(', ')}`,
+    });
+    return;
+  }
+
+  if (service === 'backend') {
+    writeSseHeaders(res);
+    const cleanup = streamBackendRing(res);
+    req.on('close', cleanup);
+    return;
+  }
+
+  // Proxy the Flask service's SSE stream. Connect FIRST so a failure still
+  // surfaces as 502 before we commit to a 200 event-stream response.
+  const base = service === 'duckdb-service' ? DUCKDB_URL : IMAGE_SERVICE_URL;
+  const controller = new AbortController();
+  req.on('close', () => controller.abort());
+  try {
+    const upstream = await fetch(`${base}/logs/stream`, {
+      headers: { 'X-Admin-Key': getApiKey() },
+      signal: controller.signal,
+    });
+    if (!upstream.ok || !upstream.body) {
+      res.status(502).json({ error: `Failed to open ${service} log stream` });
+      return;
+    }
+    writeSseHeaders(res);
+    // Pipe the upstream SSE bytes straight through (Flask emits the same
+    // `data:`/keepalive framing and its own keepalives). Use `pipeline`, not a
+    // bare `.pipe()`: on client disconnect `controller.abort()` makes the
+    // source emit an AbortError, and an unhandled stream 'error' would crash
+    // the process — pipeline routes it to the callback and destroys both ends.
+    pipeline(
+      Readable.fromWeb(upstream.body as Parameters<typeof Readable.fromWeb>[0]),
+      res,
+      (err) => {
+        // Abort on client disconnect is the normal teardown path, not an error.
+        if (err && !controller.signal.aborted) {
+          console.error('[GET /api/admin/logs/stream] pipe', { service, error: String(err) });
+        }
+      },
+    );
+  } catch (error) {
+    if (controller.signal.aborted) return; // client went away mid-connect
+    console.error('[GET /api/admin/logs/stream]', { service, error: String(error) });
+    if (!res.headersSent) res.status(502).json({ error: `${service} unreachable` });
+    else res.end();
+  }
+});

backend/src/log.ts

@@ -0,0 +1,34 @@
+// Structured logger for the backend (#178).
+//
+// Each call pushes a `LogEntry` straight into the in-memory ring AND writes a
+// formatted human line to the *saved original* stdout/stderr (via logRing's
+// `writeStdout`/`writeStderr`), so:
+//   - the admin Server Logs panel sees a structured `{ ts, level, msg }`, and
+//   - `docker logs` / PM2 see a readable line, exactly once (the original
+//     writer bypasses the tee, so the line is not re-captured as a duplicate
+//     entry).
+//
+// New call sites — the access-log middleware and converted boot banners — use
+// this. Stray `console.*` and third-party output still land in the ring via
+// the tee fallback (see logRing.ts).
+//
+// SECURITY: never pass secrets, auth headers, request bodies, or the admin
+// password to these functions — entries are admin-readable and (ADR-023)
+// persisted to disk. See accessLog.ts for the redaction rules on request data.
+
+import type { LogEntry, LogLevel } from '@highfive/contracts';
+import { pushEntry, writeStderr, writeStdout } from './logRing';
+
+function emit(level: LogLevel, msg: string): void {
+  const entry: LogEntry = { ts: new Date().toISOString(), level, msg };
+  pushEntry(entry);
+  const line = `${entry.ts} ${level.toUpperCase()} ${msg}\n`;
+  if (level === 'error') writeStderr(line);
+  else writeStdout(line);
+}
+
+export const log = {
+  info: (msg: string): void => emit('info', msg),
+  warn: (msg: string): void => emit('warn', msg),
+  error: (msg: string): void => emit('error', msg),
+};