analysis.js

// Shared analysis core — runs in BOTH the Node server and the browser bundle.
// Pure: no node-only APIs (TextEncoder is available in both). The caller supplies
// a `tokenize(text, vocab)` function and, optionally, exact Anthropic API counts.

export const MODELS = [
  // Anthropic — Claude's tokenizer is unpublished, so local tiles are an APPROXIMATION
  // (exact counts come from the count_tokens API when a key is present).
  { id: "claude-opus-4-8", label: "Claude Opus 4.8", provider: "anthropic", family: "modern" },
  { id: "claude-opus-4-7", label: "Claude Opus 4.7", provider: "anthropic", family: "modern" },
  { id: "claude-opus-4-5", label: "Claude Opus 4.5", provider: "anthropic", family: "legacy" },
  { id: "claude-sonnet-4-6", label: "Claude Sonnet 4.6", provider: "anthropic", family: "legacy" },
  { id: "claude-haiku-4-5", label: "Claude Haiku 4.5", provider: "anthropic", family: "legacy" },
  // OpenAI — tiktoken IS OpenAI's real tokenizer, so these counts/boundaries are EXACT for the
  // text (the chat API adds a few message-framing tokens on top). Encoding per encoding_for_model.
  { id: "gpt-5", label: "GPT-5", provider: "openai", vocab: "o200k_base" },
  { id: "gpt-4o", label: "GPT-4o", provider: "openai", vocab: "o200k_base" },
  { id: "gpt-4o-mini", label: "GPT-4o mini", provider: "openai", vocab: "o200k_base" },
  { id: "gpt-4-1", label: "GPT-4.1", provider: "openai", vocab: "o200k_base" },
  { id: "gpt-4-turbo", label: "GPT-4 Turbo", provider: "openai", vocab: "cl100k_base" },
  { id: "gpt-3-5-turbo", label: "GPT-3.5 Turbo", provider: "openai", vocab: "cl100k_base" },
];

export const DEFAULT_MODEL = "claude-opus-4-8";
export const FAMILY_VOCAB = { modern: "o200k_base", legacy: "cl100k_base" };
export const VOCAB_SHORT = { o200k_base: "o200k", cl100k_base: "cl100k" };
export const PROVIDER_LABEL = { anthropic: "Anthropic · Claude", openai: "OpenAI · ChatGPT" };
export const MAX_CHARS = 25_000;

export const vocabFor = (m) => (m.provider === "openai" ? m.vocab : FAMILY_VOCAB[m.family]);

export function segmentWords(text) {
  const words = [];
  const re = /\S+/g;
  let m;
  while ((m = re.exec(text)) !== null) {
    words.push({ text: m[0], start: m.index, end: m.index + m[0].length });
  }
  return words;
}

/**
 * Build the full analysis payload.
 * @param {string} prompt
 * @param {(text:string, vocab:string)=>{tokenCount:number, tokens:object[]}} tokenize
 * @param {{ apiCounts?: Record<string, number|null>, apiAvailable?: boolean }} [opts]
 */
export function buildAnalysis(prompt, tokenize, { apiCounts = {}, apiAvailable = false } = {}) {
  const usedVocabs = [...new Set(MODELS.map(vocabFor))];
  const tokenizations = {};
  for (const vocab of usedVocabs) {
    const { tokenCount, tokens } = tokenize(prompt, vocab);
    tokenizations[vocab] = { label: VOCAB_SHORT[vocab], tokenCount, tokens };
  }

  const models = MODELS.map((m) => {
    const isOpenAI = m.provider === "openai";
    const vocab = vocabFor(m);
    const localCount = tokenizations[vocab].tokenCount;
    // tiktoken is OpenAI's real tokenizer → exact for GPT. For Claude it's an approximation
    // unless the count_tokens API supplied an exact number.
    const apiCount = isOpenAI ? null : apiCounts[m.id] ?? null;
    const exact = isOpenAI || apiCount != null;
    const count = apiCount != null ? apiCount : localCount;
    return {
      id: m.id,
      label: m.label,
      provider: m.provider,
      family: m.family || null,
      tokenizerId: vocab,
      tokenizerLabel: `${VOCAB_SHORT[vocab]} · ${exact ? "exact" : "approx"}`,
      apiCount,
      localCount,
      count,
      source: isOpenAI ? "exact-tokenizer" : apiCount != null ? "api" : "local-approx",
    };
  });

  const def = models.find((m) => m.id === DEFAULT_MODEL) || models[0];
  const words = segmentWords(prompt);

  return {
    prompt,
    charCount: prompt.length,
    byteCount: new TextEncoder().encode(prompt).length,
    wordCount: words.length,
    words,
    apiAvailable,
    bestGuess: { model: def.id, tokens: def.count, source: def.source },
    models,
    tokenizations,
  };
}