server.js

// Tokenoscope server — static host + token-analysis API.
//   GET  /api/health   -> { apiAvailable, models }
//   POST /api/analyze  -> full token analysis for a prompt
// No framework: native node:http + a tiny static handler. The analysis logic is
// shared with the browser build (analysis.js); here we additionally call the
// Anthropic count_tokens API for exact Claude counts when a key is present.

import { createServer } from "node:http";
import { readFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { dirname, join, extname, normalize, sep } from "node:path";
import { tokenize } from "./tokenizer.js";
import { MODELS, MAX_CHARS, buildAnalysis } from "./analysis.js";

const __dirname = dirname(fileURLToPath(import.meta.url));
const PUBLIC_DIR = join(__dirname, "public");
const PORT = Number(process.env.PORT) || 5050;

const HAS_CREDS = Boolean(process.env.ANTHROPIC_API_KEY || process.env.ANTHROPIC_AUTH_TOKEN);

// ── Anthropic client (lazy; only when a credential is present) ────
let _client = null;
async function getClient() {
  if (!HAS_CREDS) return null;
  if (!_client) {
    const mod = await import("@anthropic-ai/sdk");
    const Anthropic = mod.default || mod;
    _client = new Anthropic();
  }
  return _client;
}

async function fetchApiCounts(prompt) {
  const client = await getClient();
  if (!client) return {};
  // Only Anthropic's API can count Claude tokens; GPT models use the (exact) local tiktoken.
  const targets = MODELS.filter((m) => m.provider === "anthropic");
  const settled = await Promise.allSettled(
    targets.map((m) =>
      client.messages.countTokens({
        model: m.id,
        messages: [{ role: "user", content: prompt }],
      })
    )
  );
  const counts = {};
  settled.forEach((r, i) => {
    counts[targets[i].id] = r.status === "fulfilled" ? r.value.input_tokens : null;
  });
  return counts;
}

async function analyze(prompt) {
  const apiCounts = await fetchApiCounts(prompt);
  return buildAnalysis(prompt, tokenize, { apiCounts, apiAvailable: HAS_CREDS });
}

// ── http plumbing ────────────────────────────────────────────────
const MIME = {
  ".html": "text/html; charset=utf-8",
  ".css": "text/css; charset=utf-8",
  ".js": "text/javascript; charset=utf-8",
  ".json": "application/json; charset=utf-8",
  ".svg": "image/svg+xml",
  ".ico": "image/x-icon",
};

function sendJSON(res, status, obj) {
  res.writeHead(status, { "Content-Type": "application/json; charset=utf-8" });
  res.end(JSON.stringify(obj));
}

function readBody(req, limitBytes) {
  return new Promise((resolve, reject) => {
    let size = 0;
    const chunks = [];
    req.on("data", (c) => {
      size += c.length;
      if (size > limitBytes) {
        reject(Object.assign(new Error("payload too large"), { status: 413 }));
        req.destroy();
        return;
      }
      chunks.push(c);
    });
    req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
    req.on("error", reject);
  });
}

async function serveStatic(req, res) {
  let urlPath = decodeURIComponent(new URL(req.url, "http://localhost").pathname);
  if (urlPath === "/") urlPath = "/index.html";
  const filePath = normalize(join(PUBLIC_DIR, urlPath));
  if (filePath !== PUBLIC_DIR && !filePath.startsWith(PUBLIC_DIR + sep)) {
    res.writeHead(403).end("Forbidden");
    return;
  }
  try {
    const data = await readFile(filePath);
    res.writeHead(200, { "Content-Type": MIME[extname(filePath)] || "application/octet-stream" });
    res.end(data);
  } catch {
    res.writeHead(404, { "Content-Type": "text/plain" }).end("Not found");
  }
}

const server = createServer(async (req, res) => {
  try {
    const { pathname } = new URL(req.url, "http://localhost");

    if (req.method === "GET" && pathname === "/api/health") {
      return sendJSON(res, 200, { apiAvailable: HAS_CREDS, models: MODELS.map((m) => m.id) });
    }

    if (req.method === "POST" && pathname === "/api/analyze") {
      let raw;
      try {
        raw = await readBody(req, 2 * 1024 * 1024);
      } catch (e) {
        return sendJSON(res, e.status || 400, { error: "Request body too large." });
      }
      let prompt;
      try {
        ({ prompt } = JSON.parse(raw));
      } catch {
        return sendJSON(res, 400, { error: "Invalid JSON." });
      }
      if (typeof prompt !== "string" || prompt.length === 0) {
        return sendJSON(res, 400, { error: "Provide a non-empty `prompt` string." });
      }
      if (prompt.length > MAX_CHARS) {
        return sendJSON(res, 413, {
          error: `Prompt too long for the visualizer (${prompt.length.toLocaleString()} chars). Limit is ${MAX_CHARS.toLocaleString()}.`,
        });
      }
      const result = await analyze(prompt);
      return sendJSON(res, 200, result);
    }

    if (req.method === "GET") return serveStatic(req, res);

    res.writeHead(405, { "Content-Type": "text/plain" }).end("Method not allowed");
  } catch (err) {
    console.error("server error:", err);
    if (!res.headersSent) sendJSON(res, 500, { error: "Internal server error." });
  }
});

server.listen(PORT, () => {
  console.log(`\n  ◐  Tokenoscope running → http://localhost:${PORT}`);
  console.log(`     Claude counts: ${HAS_CREDS ? "exact (Anthropic count_tokens API)" : "local approximation (set ANTHROPIC_API_KEY for exact)"}`);
  console.log(`     GPT counts:    exact (tiktoken — OpenAI's own tokenizer)`);
  console.log(`     Token tiles:   local tokenizer, exact for GPT / approximate for Claude\n`);
});