diff --git a/lat.md/cli.md b/lat.md/cli.md index 19753d0..d88f562 100644 --- a/lat.md/cli.md +++ b/lat.md/cli.md @@ -303,7 +303,9 @@ Core search logic in [[src/cli/search.ts#runSearch]] (returns matched sections), ### Provider Detection -Requires an LLM key resolved by [[src/config.ts#getLlmKey]] in priority order: +Resolves an embedding provider from the LLM key (or lack thereof). When no key is configured, falls back to a local embedding model so search works out of the box without any API key. + +Key resolution order (via [[src/config.ts#getLlmKey]]): 1. `LAT_LLM_KEY` env var — direct value 2. `LAT_LLM_KEY_FILE` env var — path to a file containing the key (read and trimmed) @@ -312,6 +314,7 @@ Requires an LLM key resolved by [[src/config.ts#getLlmKey]] in priority order: Provider is auto-detected from the resolved key prefix: +- *(no key)* — local embeddings via `@huggingface/transformers` (see [[cli#search#Local Embeddings]]) - `sk-...` — OpenAI (uses `text-embedding-3-small`, 1536 dims) - `vck_...` — Vercel AI Gateway (uses `openai/text-embedding-3-small`, 1536 dims) - `sk-ant-...` — Anthropic (not supported, errors with guidance) @@ -319,7 +322,21 @@ Provider is auto-detected from the resolved key prefix: Implementation: [[src/search/provider.ts]], [[src/config.ts]] -### Embeddings +### Local Embeddings + +Falls back to a local model when no API key is configured. Uses `@huggingface/transformers` (optional dep). + +Three pre-defined model sizes, selected via `LAT_LOCAL_MODEL_SIZE` env var (default `small`): + +- `small` — `Xenova/all-MiniLM-L6-v2`, 384 dims, ~45 MB download +- `medium` — `Xenova/bge-base-en-v1.5`, 768 dims, ~130 MB download +- `large` — `Xenova/bge-large-en-v1.5`, 1024 dims, ~330 MB download + +Dimensions are known statically from the model table — no need to load the model to discover them. The pipeline promise is cached so concurrent callers share a single load. Errors (missing package, invalid size) surface before any indexing work begins. + +Implementation: [[src/search/local.ts#embedLocal]] + +### API Embeddings Direct `fetch()` calls to the provider's OpenAI-compatible `/v1/embeddings` endpoint. No LangChain or other framework — keeps the dependency tree minimal. Batches up to 2048 texts per request. @@ -331,6 +348,8 @@ Uses `@libsql/client` (Turso's libsql) in local file mode — pure JS/WASM, no n Single `sections` table holds metadata, content, content hash, and the embedding vector. No separate vector table needed. +A `meta` table tracks the current embedding dimensions. On dimension mismatch (e.g. switching from API at 1536 to local at 384), the table is dropped and rebuilt in a single `db.batch()` call. + The database is stored at `lat.md/.cache/vectors.db` and should not be committed (included in `.gitignore` template). Implementation: [[src/search/db.ts]] diff --git a/lat.md/tests/mcp.md b/lat.md/tests/mcp.md index da1f748..5044e8a 100644 --- a/lat.md/tests/mcp.md +++ b/lat.md/tests/mcp.md @@ -40,5 +40,5 @@ Semantic search via `lat_search` for a login/security query returns results cont ## lat_search finds performance section Semantic search for a latency/response-times query returns results containing the Performance section. -## lat_search returns no results message -When `LAT_LLM_KEY` is not set, `lat_search` returns an error with `isError: true` explaining the missing key. +## lat_search works without an API key +When `LAT_LLM_KEY` is not set, `lat_search` handles the no-key case gracefully. If `@huggingface/transformers` is installed, returns search results via local embeddings. If not, returns a clean error with install guidance. Either way, no crash. diff --git a/lat.md/tests/search.md b/lat.md/tests/search.md index 843e755..85d480e 100644 --- a/lat.md/tests/search.md +++ b/lat.md/tests/search.md @@ -8,7 +8,19 @@ Tests in `tests/search.test.ts`. ## Provider Detection -Unit tests (always run). Verify `detectProvider` correctly identifies OpenAI (`sk-`), Vercel (`vck_`), rejects Anthropic (`sk-ant-`) with a helpful message, and rejects unknown prefixes. +Unit tests (always run). Verify `detectProvider` identifies providers by key prefix and returns local with correct dimensions when no key is given. + +Covers OpenAI (`sk-`), Vercel (`vck_`), local (no key, 384 dims), Anthropic rejection (`sk-ant-`), and unknown prefix rejection. + +## Schema Dimension Mismatch + +Verify that `ensureSchema` rebuilds the sections table when stored dimensions differ from the provider's. + +Creates a DB at 1536 dimensions with a row, re-inits at 384, asserts the table is empty and the new dimension is stored in meta. + +## Local Embedding + +Gated on `@huggingface/transformers` availability. Verifies that `embedLocal` produces normalized vectors with correct dimensions (384 for `Xenova/all-MiniLM-L6-v2`) and that semantically similar texts rank closer than unrelated ones. ## RAG Replay Tests diff --git a/package.json b/package.json index 1682300..46ad78b 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,9 @@ "typescript": "^5.7.0", "vitest": "^3.0.0" }, + "optionalDependencies": { + "@huggingface/transformers": "^3.8.1" + }, "dependencies": { "@folder/xdg": "^4.0.1", "@libsql/client": "^0.17.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1591ef1..db684cc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@folder/xdg': specifier: ^4.0.1 version: 4.0.1 + '@huggingface/transformers': + specifier: ^3.8.1 + version: 3.8.1 '@libsql/client': specifier: ^0.17.0 version: 0.17.0 @@ -81,6 +84,9 @@ importers: packages: + '@emnapi/runtime@1.9.0': + resolution: {integrity: sha512-QN75eB0IH2ywSpRpNddCRfQIhmJYBCJ1x5Lb3IscKAL8bMnVAKnRg8dCoXbHzVLLH7P38N2Z3mtulB7W0J0FKw==} + '@esbuild/aix-ppc64@0.27.3': resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==} engines: {node: '>=18'} @@ -246,6 +252,170 @@ packages: peerDependencies: hono: ^4 + '@huggingface/jinja@0.5.6': + resolution: {integrity: sha512-MyMWyLnjqo+KRJYSH7oWNbsOn5onuIvfXYPcc0WOGxU0eHUV7oAYUoQTl2BMdu7ml+ea/bu11UM+EshbeHwtIA==} + engines: {node: '>=18'} + + '@huggingface/transformers@3.8.1': + resolution: {integrity: sha512-tsTk4zVjImqdqjS8/AOZg2yNLd1z9S5v+7oUPpXaasDRwEDhB+xnglK1k5cad26lL5/ZIaeREgWWy0bs9y9pPA==} + + '@img/colour@1.1.0': + resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} + engines: {node: '>=18'} + + '@img/sharp-darwin-arm64@0.34.5': + resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [darwin] + + '@img/sharp-darwin-x64@0.34.5': + resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-darwin-arm64@1.2.4': + resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} + cpu: [arm64] + os: [darwin] + + '@img/sharp-libvips-darwin-x64@1.2.4': + resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} + cpu: [x64] + os: [darwin] + + '@img/sharp-libvips-linux-arm64@1.2.4': + resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-arm@1.2.4': + resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-ppc64@1.2.4': + resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-riscv64@1.2.4': + resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-s390x@1.2.4': + resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linux-x64@1.2.4': + resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-linux-arm64@0.34.5': + resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-arm@0.34.5': + resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-ppc64@0.34.5': + resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-riscv64@0.34.5': + resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-s390x@0.34.5': + resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@img/sharp-linux-x64@0.34.5': + resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@img/sharp-linuxmusl-arm64@0.34.5': + resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@img/sharp-linuxmusl-x64@0.34.5': + resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [linux] + libc: [musl] + + '@img/sharp-wasm32@0.34.5': + resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [wasm32] + + '@img/sharp-win32-arm64@0.34.5': + resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [arm64] + os: [win32] + + '@img/sharp-win32-ia32@0.34.5': + resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [ia32] + os: [win32] + + '@img/sharp-win32-x64@0.34.5': + resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + cpu: [x64] + os: [win32] + + '@isaacs/fs-minipass@4.0.1': + resolution: {integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==} + engines: {node: '>=18.0.0'} + '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} @@ -319,6 +489,36 @@ packages: '@neon-rs/load@0.0.4': resolution: {integrity: sha512-kTPhdZyTQxB+2wpiRcFWrDcejc4JI6tkPuS7UZCG4l6Zvc5kU/gGQ/ozvHTh1XR5tS+UlfAfGuPajjzQjCiHCw==} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.4': + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + + '@protobufjs/eventemitter@1.1.0': + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + + '@protobufjs/fetch@1.1.0': + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/inquire@1.1.0': + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.0': + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@repomix/tree-sitter-wasms@0.1.16': resolution: {integrity: sha512-CIINozBWFwjhH4DQALN/b4n1S08fHhXQOdjX2G7s4w+Urew37aLU0AHVyCjHM5Pbnh63tDYt4YyUkS6vRUV38A==} @@ -550,6 +750,10 @@ packages: resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} engines: {node: '>=18'} + boolean@3.2.0: + resolution: {integrity: sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. + brace-expansion@5.0.4: resolution: {integrity: sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==} engines: {node: 18 || 20 || >=22} @@ -585,6 +789,10 @@ packages: resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} engines: {node: '>= 16'} + chownr@3.0.0: + resolution: {integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==} + engines: {node: '>=18'} + commander@14.0.3: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} @@ -636,6 +844,14 @@ packages: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} + define-data-property@1.1.4: + resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} + engines: {node: '>= 0.4'} + + define-properties@1.2.1: + resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} + engines: {node: '>= 0.4'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -648,6 +864,13 @@ packages: resolution: {integrity: sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==} engines: {node: '>=8'} + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + detect-node@2.1.0: + resolution: {integrity: sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==} + devlop@1.1.0: resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} @@ -677,6 +900,9 @@ packages: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} + es6-error@4.1.1: + resolution: {integrity: sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==} + esbuild@0.27.3: resolution: {integrity: sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==} engines: {node: '>=18'} @@ -685,6 +911,10 @@ packages: escape-html@1.0.3: resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + escape-string-regexp@5.0.0: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} @@ -747,6 +977,9 @@ packages: resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} engines: {node: '>= 18.0.0'} + flatbuffers@25.9.23: + resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} + format@0.2.2: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} @@ -782,10 +1015,24 @@ packages: get-tsconfig@4.13.6: resolution: {integrity: sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw==} + global-agent@3.0.0: + resolution: {integrity: sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==} + engines: {node: '>=10.0'} + + globalthis@1.0.4: + resolution: {integrity: sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==} + engines: {node: '>= 0.4'} + gopd@1.2.0: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} + guid-typescript@1.0.9: + resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} + + has-property-descriptors@1.0.2: + resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==} + has-symbols@1.1.0: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} @@ -850,11 +1097,17 @@ packages: json-schema-typed@8.0.2: resolution: {integrity: sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==} + json-stringify-safe@5.0.1: + resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + libsql@0.5.22: resolution: {integrity: sha512-NscWthMQt7fpU8lqd7LXMvT9pi+KhhmTHAJWUB/Lj6MWa0MKFv0F2V4C6WKKpjCVZl0VwcDz4nOI3CyaT1DDiA==} cpu: [x64, arm64, wasm32, arm] os: [darwin, linux, win32] + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -864,6 +1117,10 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + matcher@3.0.0: + resolution: {integrity: sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==} + engines: {node: '>=10'} + math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} @@ -969,6 +1226,14 @@ packages: resolution: {integrity: sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==} engines: {node: 18 || 20 || >=22} + minipass@7.1.3: + resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} + engines: {node: '>=16 || 14 >=14.17'} + + minizlib@3.1.0: + resolution: {integrity: sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==} + engines: {node: '>= 18'} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -1007,6 +1272,10 @@ packages: resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} engines: {node: '>= 0.4'} + object-keys@1.1.1: + resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} + engines: {node: '>= 0.4'} + on-finished@2.4.1: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} engines: {node: '>= 0.8'} @@ -1014,6 +1283,19 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + onnxruntime-common@1.21.0: + resolution: {integrity: sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==} + + onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: + resolution: {integrity: sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ==} + + onnxruntime-node@1.21.0: + resolution: {integrity: sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==} + os: [win32, darwin, linux] + + onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: + resolution: {integrity: sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ==} + parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} @@ -1047,6 +1329,9 @@ packages: resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} engines: {node: '>=16.20.0'} + platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + postcss@8.5.6: resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} engines: {node: ^10 || ^12 || >=14} @@ -1059,6 +1344,10 @@ packages: promise-limit@2.7.0: resolution: {integrity: sha512-7nJ6v5lnJsXwGprnGXga4wx6d1POjvi5Qmf1ivTRxTjH4Z/9Czja/UCMLVmB9N93GeWOU93XaFaEt6jbuoagNw==} + protobufjs@7.5.4: + resolution: {integrity: sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==} + engines: {node: '>=12.0.0'} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -1091,6 +1380,10 @@ packages: resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + roarr@2.15.4: + resolution: {integrity: sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==} + engines: {node: '>=8.0'} + rollup@4.59.0: resolution: {integrity: sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -1103,10 +1396,22 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + semver-compare@1.0.0: + resolution: {integrity: sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==} + + semver@7.7.4: + resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} + engines: {node: '>=10'} + hasBin: true + send@1.2.1: resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} engines: {node: '>= 18'} + serialize-error@7.0.1: + resolution: {integrity: sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==} + engines: {node: '>=10'} + serve-static@2.2.1: resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} engines: {node: '>= 18'} @@ -1114,6 +1419,10 @@ packages: setprototypeof@1.2.0: resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + sharp@0.34.5: + resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} + engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} + shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} @@ -1145,6 +1454,9 @@ packages: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} + sprintf-js@1.1.3: + resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -1162,6 +1474,10 @@ packages: strip-literal@3.1.0: resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} + tar@7.5.11: + resolution: {integrity: sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ==} + engines: {node: '>=18'} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -1197,11 +1513,18 @@ packages: trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.21.0: resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} engines: {node: '>=18.0.0'} hasBin: true + type-fest@0.13.1: + resolution: {integrity: sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==} + engines: {node: '>=10'} + type-is@2.0.1: resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==} engines: {node: '>= 0.6'} @@ -1358,6 +1681,10 @@ packages: utf-8-validate: optional: true + yallist@5.0.0: + resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==} + engines: {node: '>=18'} + yaml@2.8.2: resolution: {integrity: sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==} engines: {node: '>= 14.6'} @@ -1376,6 +1703,11 @@ packages: snapshots: + '@emnapi/runtime@1.9.0': + dependencies: + tslib: 2.8.1 + optional: true + '@esbuild/aix-ppc64@0.27.3': optional: true @@ -1466,6 +1798,115 @@ snapshots: dependencies: hono: 4.12.7 + '@huggingface/jinja@0.5.6': {} + + '@huggingface/transformers@3.8.1': + dependencies: + '@huggingface/jinja': 0.5.6 + onnxruntime-node: 1.21.0 + onnxruntime-web: 1.22.0-dev.20250409-89f8206ba4 + sharp: 0.34.5 + + '@img/colour@1.1.0': {} + + '@img/sharp-darwin-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-arm64': 1.2.4 + optional: true + + '@img/sharp-darwin-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-darwin-x64': 1.2.4 + optional: true + + '@img/sharp-libvips-darwin-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-darwin-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-arm@1.2.4': + optional: true + + '@img/sharp-libvips-linux-ppc64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-riscv64@1.2.4': + optional: true + + '@img/sharp-libvips-linux-s390x@1.2.4': + optional: true + + '@img/sharp-libvips-linux-x64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-arm64@1.2.4': + optional: true + + '@img/sharp-libvips-linuxmusl-x64@1.2.4': + optional: true + + '@img/sharp-linux-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm64': 1.2.4 + optional: true + + '@img/sharp-linux-arm@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-arm': 1.2.4 + optional: true + + '@img/sharp-linux-ppc64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-ppc64': 1.2.4 + optional: true + + '@img/sharp-linux-riscv64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-riscv64': 1.2.4 + optional: true + + '@img/sharp-linux-s390x@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-s390x': 1.2.4 + optional: true + + '@img/sharp-linux-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linux-x64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-arm64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + optional: true + + '@img/sharp-linuxmusl-x64@0.34.5': + optionalDependencies: + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + optional: true + + '@img/sharp-wasm32@0.34.5': + dependencies: + '@emnapi/runtime': 1.9.0 + optional: true + + '@img/sharp-win32-arm64@0.34.5': + optional: true + + '@img/sharp-win32-ia32@0.34.5': + optional: true + + '@img/sharp-win32-x64@0.34.5': + optional: true + + '@isaacs/fs-minipass@4.0.1': + dependencies: + minipass: 7.1.3 + '@jridgewell/sourcemap-codec@1.5.5': {} '@libsql/client@0.17.0': @@ -1554,6 +1995,29 @@ snapshots: '@neon-rs/load@0.0.4': {} + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.4': {} + + '@protobufjs/eventemitter@1.1.0': {} + + '@protobufjs/fetch@1.1.0': + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/inquire@1.1.0': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.0': {} + '@repomix/tree-sitter-wasms@0.1.16': {} '@rollup/rollup-android-arm-eabi@4.59.0': @@ -1742,6 +2206,8 @@ snapshots: transitivePeerDependencies: - supports-color + boolean@3.2.0: {} + brace-expansion@5.0.4: dependencies: balanced-match: 4.0.4 @@ -1774,6 +2240,8 @@ snapshots: check-error@2.1.3: {} + chownr@3.0.0: {} + commander@14.0.3: {} content-disposition@1.0.1: {} @@ -1813,12 +2281,28 @@ snapshots: deep-eql@5.0.2: {} + define-data-property@1.1.4: + dependencies: + es-define-property: 1.0.1 + es-errors: 1.3.0 + gopd: 1.2.0 + + define-properties@1.2.1: + dependencies: + define-data-property: 1.1.4 + has-property-descriptors: 1.0.2 + object-keys: 1.1.1 + depd@2.0.0: {} dequal@2.0.3: {} detect-libc@2.0.2: {} + detect-libc@2.1.2: {} + + detect-node@2.1.0: {} + devlop@1.1.0: dependencies: dequal: 2.0.3 @@ -1843,6 +2327,8 @@ snapshots: dependencies: es-errors: 1.3.0 + es6-error@4.1.1: {} + esbuild@0.27.3: optionalDependencies: '@esbuild/aix-ppc64': 0.27.3 @@ -1874,6 +2360,8 @@ snapshots: escape-html@1.0.3: {} + escape-string-regexp@4.0.0: {} + escape-string-regexp@5.0.0: {} estree-walker@3.0.3: @@ -1958,6 +2446,8 @@ snapshots: transitivePeerDependencies: - supports-color + flatbuffers@25.9.23: {} + format@0.2.2: {} formdata-polyfill@4.0.10: @@ -1995,8 +2485,28 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 + global-agent@3.0.0: + dependencies: + boolean: 3.2.0 + es6-error: 4.1.1 + matcher: 3.0.0 + roarr: 2.15.4 + semver: 7.7.4 + serialize-error: 7.0.1 + + globalthis@1.0.4: + dependencies: + define-properties: 1.2.1 + gopd: 1.2.0 + gopd@1.2.0: {} + guid-typescript@1.0.9: {} + + has-property-descriptors@1.0.2: + dependencies: + es-define-property: 1.0.1 + has-symbols@1.1.0: {} hasown@2.0.2: @@ -2045,6 +2555,8 @@ snapshots: json-schema-typed@8.0.2: {} + json-stringify-safe@5.0.1: {} + libsql@0.5.22: dependencies: '@neon-rs/load': 0.0.4 @@ -2060,6 +2572,8 @@ snapshots: '@libsql/linux-x64-musl': 0.5.22 '@libsql/win32-x64-msvc': 0.5.22 + long@5.3.2: {} + longest-streak@3.1.0: {} loupe@3.2.1: {} @@ -2068,6 +2582,10 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + matcher@3.0.0: + dependencies: + escape-string-regexp: 4.0.0 + math-intrinsics@1.1.0: {} mdast-util-from-markdown@2.0.3: @@ -2273,6 +2791,12 @@ snapshots: dependencies: brace-expansion: 5.0.4 + minipass@7.1.3: {} + + minizlib@3.1.0: + dependencies: + minipass: 7.1.3 + ms@2.1.3: {} nanoid@3.3.11: {} @@ -2295,6 +2819,8 @@ snapshots: object-inspect@1.13.4: {} + object-keys@1.1.1: {} + on-finished@2.4.1: dependencies: ee-first: 1.1.1 @@ -2303,6 +2829,25 @@ snapshots: dependencies: wrappy: 1.0.2 + onnxruntime-common@1.21.0: {} + + onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: {} + + onnxruntime-node@1.21.0: + dependencies: + global-agent: 3.0.0 + onnxruntime-common: 1.21.0 + tar: 7.5.11 + + onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: + dependencies: + flatbuffers: 25.9.23 + guid-typescript: 1.0.9 + long: 5.3.2 + onnxruntime-common: 1.22.0-dev.20250409-89f8206ba4 + platform: 1.3.6 + protobufjs: 7.5.4 + parseurl@1.3.3: {} path-key@3.1.1: {} @@ -2321,6 +2866,8 @@ snapshots: pkce-challenge@5.0.1: {} + platform@1.3.6: {} + postcss@8.5.6: dependencies: nanoid: 3.3.11 @@ -2331,6 +2878,21 @@ snapshots: promise-limit@2.7.0: {} + protobufjs@7.5.4: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 25.3.0 + long: 5.3.2 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 @@ -2377,6 +2939,15 @@ snapshots: resolve-pkg-maps@1.0.0: {} + roarr@2.15.4: + dependencies: + boolean: 3.2.0 + detect-node: 2.1.0 + globalthis: 1.0.4 + json-stringify-safe: 5.0.1 + semver-compare: 1.0.0 + sprintf-js: 1.1.3 + rollup@4.59.0: dependencies: '@types/estree': 1.0.8 @@ -2420,6 +2991,10 @@ snapshots: safer-buffer@2.1.2: {} + semver-compare@1.0.0: {} + + semver@7.7.4: {} + send@1.2.1: dependencies: debug: 4.4.3 @@ -2436,6 +3011,10 @@ snapshots: transitivePeerDependencies: - supports-color + serialize-error@7.0.1: + dependencies: + type-fest: 0.13.1 + serve-static@2.2.1: dependencies: encodeurl: 2.0.0 @@ -2447,6 +3026,37 @@ snapshots: setprototypeof@1.2.0: {} + sharp@0.34.5: + dependencies: + '@img/colour': 1.1.0 + detect-libc: 2.1.2 + semver: 7.7.4 + optionalDependencies: + '@img/sharp-darwin-arm64': 0.34.5 + '@img/sharp-darwin-x64': 0.34.5 + '@img/sharp-libvips-darwin-arm64': 1.2.4 + '@img/sharp-libvips-darwin-x64': 1.2.4 + '@img/sharp-libvips-linux-arm': 1.2.4 + '@img/sharp-libvips-linux-arm64': 1.2.4 + '@img/sharp-libvips-linux-ppc64': 1.2.4 + '@img/sharp-libvips-linux-riscv64': 1.2.4 + '@img/sharp-libvips-linux-s390x': 1.2.4 + '@img/sharp-libvips-linux-x64': 1.2.4 + '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 + '@img/sharp-libvips-linuxmusl-x64': 1.2.4 + '@img/sharp-linux-arm': 0.34.5 + '@img/sharp-linux-arm64': 0.34.5 + '@img/sharp-linux-ppc64': 0.34.5 + '@img/sharp-linux-riscv64': 0.34.5 + '@img/sharp-linux-s390x': 0.34.5 + '@img/sharp-linux-x64': 0.34.5 + '@img/sharp-linuxmusl-arm64': 0.34.5 + '@img/sharp-linuxmusl-x64': 0.34.5 + '@img/sharp-wasm32': 0.34.5 + '@img/sharp-win32-arm64': 0.34.5 + '@img/sharp-win32-ia32': 0.34.5 + '@img/sharp-win32-x64': 0.34.5 + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 @@ -2485,6 +3095,8 @@ snapshots: source-map-js@1.2.1: {} + sprintf-js@1.1.3: {} + stackback@0.0.2: {} statuses@2.0.2: {} @@ -2497,6 +3109,14 @@ snapshots: dependencies: js-tokens: 9.0.1 + tar@7.5.11: + dependencies: + '@isaacs/fs-minipass': 4.0.1 + chownr: 3.0.0 + minipass: 7.1.3 + minizlib: 3.1.0 + yallist: 5.0.0 + tinybench@2.9.0: {} tinyexec@0.3.2: {} @@ -2520,6 +3140,9 @@ snapshots: trough@2.2.0: {} + tslib@2.8.1: + optional: true + tsx@4.21.0: dependencies: esbuild: 0.27.3 @@ -2527,6 +3150,8 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + type-fest@0.13.1: {} + type-is@2.0.1: dependencies: content-type: 1.0.5 @@ -2685,6 +3310,8 @@ snapshots: ws@8.19.0: {} + yallist@5.0.0: {} + yaml@2.8.2: {} zod-to-json-schema@3.25.1(zod@4.3.6): diff --git a/src/cli/search.ts b/src/cli/search.ts index 46eb1e6..111cfe6 100644 --- a/src/cli/search.ts +++ b/src/cli/search.ts @@ -24,7 +24,7 @@ export type IndexProgress = { async function withDb( latDir: string, - key: string, + key: string | undefined, progress: IndexProgress | undefined, fn: ( db: Awaited>, @@ -57,7 +57,7 @@ async function withDb( export async function runSearch( latDir: string, query: string, - key: string, + key: string | undefined, limit: number, progress?: IndexProgress, ): Promise { @@ -85,7 +85,7 @@ export async function runSearch( */ export async function runIndex( latDir: string, - key: string, + key: string | undefined, progress?: IndexProgress, ): Promise { await withDb(latDir, key, progress, async () => {}); @@ -123,42 +123,41 @@ export async function searchCommand( opts: { limit: number; reindex?: boolean }, progress?: IndexProgress, ): Promise { - const { getLlmKey, getConfigPath } = await import('../config.js'); + const { getLlmKey } = await import('../config.js'); let key: string | undefined; try { key = getLlmKey(); } catch (err) { return { output: (err as Error).message, isError: true }; } - if (!key) { - const s = ctx.styler; - return { - output: - s.red('No API key configured.') + - ' Provide a key via LAT_LLM_KEY, LAT_LLM_KEY_FILE, LAT_LLM_KEY_HELPER, or run ' + - s.cyan('lat init') + - (ctx.mode === 'cli' - ? ' to save one in ' + s.dim(getConfigPath()) - : '') + - '.', - isError: true, - }; - } - if (!query) { - await runIndex(ctx.latDir, key, progress); - return { output: '' }; - } + try { + if (!query) { + await runIndex(ctx.latDir, key, progress); + return { output: '' }; + } - const result = await runSearch(ctx.latDir, query, key, opts.limit, progress); + const result = await runSearch( + ctx.latDir, + query, + key, + opts.limit, + progress, + ); - if (result.matches.length === 0) { - return { output: 'No results found.' }; - } + if (result.matches.length === 0) { + return { output: 'No results found.' }; + } - return { - output: - formatResultList(ctx, `Search results for "${query}":`, result.matches) + - formatNavHints(ctx), - }; + return { + output: + formatResultList( + ctx, + `Search results for "${query}":`, + result.matches, + ) + formatNavHints(ctx), + }; + } catch (err) { + return { output: (err as Error).message, isError: true }; + } } diff --git a/src/search/db.ts b/src/search/db.ts index d92fb7b..d202641 100644 --- a/src/search/db.ts +++ b/src/search/db.ts @@ -17,7 +17,37 @@ export async function ensureSchema( db: Client, dimensions: number, ): Promise { + if (!Number.isInteger(dimensions) || dimensions <= 0) { + throw new Error(`Invalid embedding dimensions: ${dimensions}`); + } + await db.execute( + `CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + )`, + ); + + const metaRows = await db.execute( + "SELECT value FROM meta WHERE key = 'embedding_dimensions'", + ); + const stored = + metaRows.rows.length > 0 + ? parseInt(metaRows.rows[0].value as string, 10) + : null; + const needsRebuild = stored !== null && stored !== dimensions; + + if (needsRebuild) { + process.stderr.write( + `Embedding dimensions changed (${stored} → ${dimensions}), rebuilding index...\n`, + ); + await db.batch([ + 'DROP INDEX IF EXISTS sections_vec_idx', + 'DROP TABLE IF EXISTS sections', + ]); + } + + await db.batch([ `CREATE TABLE IF NOT EXISTS sections ( id TEXT PRIMARY KEY, file TEXT NOT NULL, @@ -27,19 +57,13 @@ export async function ensureSchema( embedding F32_BLOB(${dimensions}), updated_at INTEGER NOT NULL )`, - ); - - await db.execute( `CREATE INDEX IF NOT EXISTS sections_vec_idx ON sections (libsql_vector_idx(embedding))`, - ); - - await db.execute( - `CREATE TABLE IF NOT EXISTS meta ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - )`, - ); + { + sql: "INSERT OR REPLACE INTO meta (key, value) VALUES ('embedding_dimensions', ?)", + args: [String(dimensions)], + }, + ]); } export async function closeDb(db: Client): Promise { diff --git a/src/search/embeddings.ts b/src/search/embeddings.ts index daa2f03..194b7c5 100644 --- a/src/search/embeddings.ts +++ b/src/search/embeddings.ts @@ -1,16 +1,16 @@ -import type { EmbeddingProvider } from './provider.js'; +import type { ApiProvider, EmbeddingProvider } from './provider.js'; +import { embedLocal } from './local.js'; -const MAX_BATCH = 2048; +const API_MAX_BATCH = 2048; -export async function embed( +async function embedApi( texts: string[], - provider: EmbeddingProvider, + provider: ApiProvider, key: string, ): Promise { const results: number[][] = []; - - for (let i = 0; i < texts.length; i += MAX_BATCH) { - const batch = texts.slice(i, i + MAX_BATCH); + for (let i = 0; i < texts.length; i += API_MAX_BATCH) { + const batch = texts.slice(i, i + API_MAX_BATCH); const resp = await fetch(`${provider.apiBase}/embeddings`, { method: 'POST', headers: provider.headers(key), @@ -35,6 +35,17 @@ export async function embed( results.push(item.embedding); } } - return results; } + +export async function embed( + texts: string[], + provider: EmbeddingProvider, + key?: string, +): Promise { + if (provider.kind === 'local') { + return embedLocal(texts, provider.model); + } + if (!key) throw new Error('API embedding provider requires a key'); + return embedApi(texts, provider, key); +} diff --git a/src/search/index.ts b/src/search/index.ts index 70076cf..6fa8ff1 100644 --- a/src/search/index.ts +++ b/src/search/index.ts @@ -31,7 +31,7 @@ export async function indexSections( latDir: string, db: Client, provider: EmbeddingProvider, - key: string, + key?: string, ): Promise { const projectRoot = dirname(latDir); const allSections = await loadAllSections(latDir); diff --git a/src/search/local.ts b/src/search/local.ts new file mode 100644 index 0000000..fbc95d1 --- /dev/null +++ b/src/search/local.ts @@ -0,0 +1,57 @@ +interface Extractor { + ( + texts: string[], + opts: { pooling: string; normalize: boolean }, + ): Promise<{ tolist(): number[][] }>; +} + +const LOCAL_BATCH = 32; + +let _pipeline: Promise | null = null; +let _pipelineModel: string | null = null; + +async function requireTransformers() { + try { + return await import('@huggingface/transformers'); + } catch { + throw new Error( + 'Local embeddings require @huggingface/transformers — install it alongside lat.md.\n' + + 'Or set LAT_LLM_KEY to use an API provider instead.', + ); + } +} + +async function loadPipeline(model: string): Promise { + const { pipeline } = await requireTransformers(); + return pipeline('feature-extraction', model, { + dtype: 'fp16', + }) as unknown as Extractor; +} + +function getLocalPipeline(model: string): Promise { + if (_pipeline && _pipelineModel === model) return _pipeline; + _pipelineModel = model; + _pipeline = loadPipeline(model).catch((err) => { + _pipeline = null; + _pipelineModel = null; + throw err; + }); + return _pipeline; +} + +export async function embedLocal( + texts: string[], + model: string, +): Promise { + const extractor = await getLocalPipeline(model); + const results: number[][] = []; + for (let i = 0; i < texts.length; i += LOCAL_BATCH) { + const batch = texts.slice(i, i + LOCAL_BATCH); + const output = await extractor(batch, { + pooling: 'mean', + normalize: true, + }); + results.push(...output.tolist()); + } + return results; +} diff --git a/src/search/provider.ts b/src/search/provider.ts index 16ee2b6..2719e24 100644 --- a/src/search/provider.ts +++ b/src/search/provider.ts @@ -1,4 +1,5 @@ -export type EmbeddingProvider = { +export type ApiProvider = { + kind: 'api'; name: string; apiBase: string; model: string; @@ -6,7 +7,53 @@ export type EmbeddingProvider = { headers: (key: string) => Record; }; -const openai: EmbeddingProvider = { +export type LocalProvider = { + kind: 'local'; + name: string; + model: string; + dimensions: number; +}; + +export type EmbeddingProvider = ApiProvider | LocalProvider; + +type LocalModelSize = 'small' | 'medium' | 'large'; + +type LocalModelEntry = { + model: string; + dimensions: number; +}; + +const LOCAL_MODELS: Record = { + small: { model: 'Xenova/all-MiniLM-L6-v2', dimensions: 384 }, + medium: { model: 'Xenova/bge-base-en-v1.5', dimensions: 768 }, + large: { model: 'Xenova/bge-large-en-v1.5', dimensions: 1024 }, +}; + +function parseModelSize(): LocalModelSize { + const raw = process.env.LAT_LOCAL_MODEL_SIZE; + if (!raw) return 'small'; + const normalized = raw.toLowerCase().trim(); + if (!(normalized in LOCAL_MODELS)) { + throw new Error( + `Invalid LAT_LOCAL_MODEL_SIZE "${raw}". Valid sizes: ${Object.keys(LOCAL_MODELS).join(', ')}.`, + ); + } + return normalized as LocalModelSize; +} + +export function getLocalProvider(): LocalProvider { + const size = parseModelSize(); + const entry = LOCAL_MODELS[size]; + return { + kind: 'local', + name: 'local', + model: entry.model, + dimensions: entry.dimensions, + }; +} + +const openai: ApiProvider = { + kind: 'api', name: 'openai', apiBase: 'https://api.openai.com/v1', model: 'text-embedding-3-small', @@ -17,7 +64,8 @@ const openai: EmbeddingProvider = { }), }; -const vercel: EmbeddingProvider = { +const vercel: ApiProvider = { + kind: 'api', name: 'vercel', apiBase: 'https://ai-gateway.vercel.sh/v1', model: 'openai/text-embedding-3-small', @@ -28,10 +76,13 @@ const vercel: EmbeddingProvider = { }), }; -export function detectProvider(key: string): EmbeddingProvider { +export function detectProvider(key?: string): EmbeddingProvider { + if (!key) return getLocalProvider(); + if (key.startsWith('REPLAY_LAT_LLM_KEY::')) { const replayUrl = key.slice('REPLAY_LAT_LLM_KEY::'.length); return { + kind: 'api', name: 'replay', apiBase: replayUrl, model: 'replay', @@ -41,12 +92,12 @@ export function detectProvider(key: string): EmbeddingProvider { } if (key.startsWith('sk-ant-')) { throw new Error( - "Anthropic doesn't offer an embedding model. Set LAT_LLM_KEY to an OpenAI (sk-...) or Vercel AI Gateway (vck_...) key.", + "Anthropic doesn't offer an embedding model. Set LAT_LLM_KEY to an OpenAI (sk-...) or Vercel AI Gateway (vck_...) key, or omit it to use local embeddings.", ); } if (key.startsWith('vck_')) return vercel; if (key.startsWith('sk-')) return openai; throw new Error( - `Unrecognized LAT_LLM_KEY prefix. Supported: OpenAI (sk-...), Vercel AI Gateway (vck_...).`, + `Unrecognized LAT_LLM_KEY prefix. Supported: OpenAI (sk-...), Vercel AI Gateway (vck_...). Omit LAT_LLM_KEY to use local embeddings.`, ); } diff --git a/src/search/search.ts b/src/search/search.ts index bb474cf..3f304c5 100644 --- a/src/search/search.ts +++ b/src/search/search.ts @@ -13,7 +13,7 @@ export async function searchSections( db: Client, query: string, provider: EmbeddingProvider, - key: string, + key?: string, limit = 5, ): Promise { const [queryVec] = await embed([query], provider, key); diff --git a/tests/mcp.test.ts b/tests/mcp.test.ts index 04a95a3..f166d12 100644 --- a/tests/mcp.test.ts +++ b/tests/mcp.test.ts @@ -181,8 +181,14 @@ describe.skipIf(!canRunSearch)('mcp search (rag)', () => { expect(text).toContain('Performance'); }); - // @lat: [[tests/mcp#lat_search returns no results message]] - it('lat_search returns no results message when key is missing', async () => { + // @lat: [[tests/mcp#lat_search works without an API key]] + it('lat_search works without an API key', async () => { + let hasTransformers = false; + try { + await import('@huggingface/transformers'); + hasTransformers = true; + } catch {} + // Spin up a separate MCP server without LAT_LLM_KEY and without XDG config const transport2 = new StdioClientTransport({ command: 'node', @@ -195,11 +201,16 @@ describe.skipIf(!canRunSearch)('mcp search (rag)', () => { const result = await client2.callTool({ name: 'lat_search', - arguments: { query: 'anything' }, + arguments: { query: 'how do we run tests?' }, }); const text = (result.content as { type: string; text: string }[])[0].text; - expect(text).toContain('No API key configured'); - expect(result.isError).toBe(true); + if (hasTransformers) { + expect(result.isError).toBeFalsy(); + expect(text).toContain('Search results'); + } else { + expect(result.isError).toBe(true); + expect(text).toContain('@huggingface/transformers'); + } await client2.close(); }); diff --git a/tests/rag-replay-server.ts b/tests/rag-replay-server.ts index b079f63..fff8152 100644 --- a/tests/rag-replay-server.ts +++ b/tests/rag-replay-server.ts @@ -12,7 +12,7 @@ import { createHash } from 'node:crypto'; import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; import { join } from 'node:path'; import { createServer, type Server } from 'node:http'; -import type { EmbeddingProvider } from '../src/search/provider.js'; +import type { ApiProvider } from '../src/search/provider.js'; type Manifest = { dimensions: number; @@ -74,7 +74,7 @@ function createReplayHandler(replayDir: string) { function createCaptureHandler( replayDir: string, - realProvider: EmbeddingProvider, + realProvider: ApiProvider, realKey: string, ) { const captured = new Map(); @@ -149,7 +149,7 @@ function createCaptureHandler( export function startReplayServer( replayDir: string, - opts?: { capture: true; provider: EmbeddingProvider; key: string }, + opts?: { capture: true; provider: ApiProvider; key: string }, ): Promise { let handler: (input: string[]) => any; let flush = () => {}; diff --git a/tests/search.test.ts b/tests/search.test.ts index bc284dd..174fcbe 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; import { mkdtempSync, rmSync, cpSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; @@ -17,6 +17,13 @@ import type { Server } from 'node:http'; // @lat: [[search#Provider Detection]] describe('detectProvider', () => { + it('returns local provider when no key given', () => { + const p = detectProvider(); + expect(p.kind).toBe('local'); + expect(p.name).toBe('local'); + expect(p.dimensions).toBe(384); + }); + it('detects OpenAI key', () => { const p = detectProvider('sk-abc123'); expect(p.name).toBe('openai'); @@ -36,6 +43,109 @@ describe('detectProvider', () => { }); }); +// --- Schema tests --- + +// @lat: [[search#Schema Dimension Mismatch]] +describe('ensureSchema dimension mismatch', () => { + let tmp: string; + let db: Client; + + beforeAll(() => { + tmp = mkdtempSync(join(tmpdir(), 'lat-dim-')); + }); + + afterAll(async () => { + if (db) await closeDb(db); + rmSync(tmp, { recursive: true, force: true }); + }); + + it('rebuilds table and logs diagnostic when dimensions change', async () => { + db = openDb(tmp); + + // Create with 1536 dimensions + await ensureSchema(db, 1536); + await db.execute({ + sql: `INSERT INTO sections + (id, file, heading, content, content_hash, embedding, updated_at) + VALUES (?, ?, ?, ?, ?, vector(?), ?)`, + args: [ + 'test-id', + 'test.md', + 'Test', + 'content', + 'hash123', + JSON.stringify(new Array(1536).fill(0)), + Date.now(), + ], + }); + const before = await db.execute('SELECT COUNT(*) as n FROM sections'); + expect(before.rows[0].n).toBe(1); + + // Re-init with 384 dimensions — should drop and recreate + const spy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + + await ensureSchema(db, 384); + + const calls = spy.mock.calls.map((c) => String(c[0])); + spy.mockRestore(); + + const after = await db.execute('SELECT COUNT(*) as n FROM sections'); + expect(after.rows[0].n).toBe(0); + + // Verify new dimension is stored in meta + const meta = await db.execute( + "SELECT value FROM meta WHERE key = 'embedding_dimensions'", + ); + expect(meta.rows[0].value).toBe('384'); + + // Verify diagnostic was printed to stderr + expect(calls.some((m) => m.includes('dimensions changed'))).toBe(true); + }); +}); + +// --- Local embedding tests (requires @huggingface/transformers) --- + +let hasTransformers = false; +try { + await import('@huggingface/transformers'); + hasTransformers = true; +} catch {} + +// @lat: [[search#Local Embedding]] +describe.skipIf(!hasTransformers)('local embedding', () => { + it('produces normalized vectors with correct dimensions', async () => { + const { embedLocal } = await import('../src/search/local.js'); + const model = 'Xenova/all-MiniLM-L6-v2'; + + const [vec] = await embedLocal(['hello world'], model); + expect(vec.length).toBe(384); + + // Mean-pooled + normalized vectors should have unit length. + const norm = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0)); + expect(norm).toBeCloseTo(1.0, 2); + }); + + it('ranks semantically similar texts closer', async () => { + const { embedLocal } = await import('../src/search/local.js'); + const model = 'Xenova/all-MiniLM-L6-v2'; + + const [a, b, c] = await embedLocal( + [ + 'how to authenticate users', + 'user login and security', + 'banana split recipe', + ], + model, + ); + + const dot = (x: number[], y: number[]) => + x.reduce((s, v, i) => s + v * y[i], 0); + + // auth↔login should be more similar than auth↔banana + expect(dot(a, b)).toBeGreaterThan(dot(a, c)); + }); +}); + // --- RAG functional tests --- // // Two modes: @@ -64,6 +174,8 @@ describe.skipIf(!canRun)('search (rag)', () => { const realKey = process.env.LAT_LLM_KEY; if (!realKey) throw new Error('LAT_LLM_KEY must be set in capture mode'); const realProvider = detectProvider(realKey); + if (realProvider.kind !== 'api') + throw new Error('Capture mode requires an API provider'); const replay = await startReplayServer(replayDir, { capture: true,