tokenizer.js

// Local byte-pair tokenizer used for the *visual* breakdown.
//
// IMPORTANT: these are OpenAI BPE vocabularies (o200k_base, cl100k_base) used as
// an APPROXIMATION of Claude's tokenizer — Anthropic does not publish the real
// Claude 3/4 tokenizer. The authoritative per-model counts come from the
// count_tokens API (see server.js). Here we only need plausible token
// *boundaries* to illustrate how subword tokenization works.
//
// js-tiktoken@1.0.21 is pure JS and exposes per-token raw UTF-8 bytes via the
// (internal-but-stable) `textMap: Map<id, Uint8Array>` field, which lets us map
// each token to an exact character span — correct even when a single emoji/CJK
// character is split across several tokens.

import { Tiktoken } from "js-tiktoken/lite";
import o200k_base from "js-tiktoken/ranks/o200k_base";
import cl100k_base from "js-tiktoken/ranks/cl100k_base";

const RANKS = { o200k_base, cl100k_base };
const encoders = new Map();

function getEncoder(vocab) {
  let enc = encoders.get(vocab);
  if (!enc) {
    const ranks = RANKS[vocab];
    if (!ranks) throw new Error(`unknown vocab: ${vocab}`);
    enc = new Tiktoken(ranks);
    if (!enc.textMap || typeof enc.textMap.get !== "function") {
      throw new Error("js-tiktoken `textMap` unavailable — unexpected version");
    }
    encoders.set(vocab, enc);
  }
  return enc;
}

const nonFatal = new TextDecoder("utf-8", { fatal: false });
const toHex = (bytes) => Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join(" ");

/**
 * Tokenize `text` with the given vocab and return token entries with offsets.
 * @returns {{ tokenCount: number, tokens: Array<{
 *   tokenId: number, text: string, bytes: string, partial: boolean,
 *   start: number, end: number }> }}
 *
 * `start`/`end` are UTF-16 code-unit offsets into the original string
 * (compatible with String.length / String.prototype.slice).
 * `partial` tokens are byte fragments of a multi-byte character; `text` is ""
 * and the caller should display `bytes` (hex) instead.
 */
export function tokenize(text, vocab) {
  const enc = getEncoder(vocab);
  // allowedSpecial=[] and disallowedSpecial=[] => never emit/throw on special
  // tokens; literal "<|endoftext|>" in user input is tokenized as plain text.
  const ids = enc.encode(text, [], []);
  const stream = new TextDecoder("utf-8"); // streaming decoder for offset mapping

  const tokens = [];
  let charPos = 0;
  let pendingIds = [];
  let textBuf = "";

  const bytesOf = (id) => enc.textMap.get(id) ?? enc.inverseSpecialTokens?.[id] ?? null;

  const flush = (emitted, groupIds) => {
    const start = charPos;
    const end = charPos + emitted.length;
    for (const id of groupIds) {
      const bytes = bytesOf(id);
      const standalone = bytes ? nonFatal.decode(bytes) : "";
      const partial = groupIds.length > 1 || !bytes || standalone.includes("�");
      tokens.push({
        tokenId: id,
        text: partial ? "" : standalone,
        bytes: bytes ? toHex(bytes) : "",
        partial,
        start,
        end,
      });
    }
    charPos = end;
  };

  for (const id of ids) {
    pendingIds.push(id);
    const bytes = bytesOf(id);
    if (!bytes) continue; // special/unknown: zero-width, resolved on next flush
    const piece = stream.decode(bytes, { stream: true });
    if (piece.length === 0) continue; // incomplete multi-byte char — keep buffering
    const last = piece.charCodeAt(piece.length - 1);
    if (last >= 0xd800 && last <= 0xdbff) {
      // dangling high surrogate (half an astral char) — wait for the low half
      textBuf += piece;
      continue;
    }
    flush(textBuf + piece, pendingIds);
    textBuf = "";
    pendingIds = [];
  }
  if (pendingIds.length) {
    const tail = textBuf + stream.decode();
    flush(tail, pendingIds);
  }

  return { tokenCount: ids.length, tokens };
}