From 5d46bb105b3d032a47e2bb8e26b574f556f31fb6 Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Sun, 21 Dec 2025 16:51:30 -0800 Subject: [PATCH 01/12] refactor: rename Reranker to Intent and unify constructor options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This refactors the public API from Reranker to Intent, consolidating context, extractors, and config into a single optional constructor options object. Notable changes: - Renamed core class/files/tests from reranker -> intent, and rerank() -> rank(). - Added default key/summary extractors (hash-based key + pretty JSON summary) and centralized JSON stringification helpers. - Adjusted config naming/usage to preserve UPPER_SNAKE_CASE env-backed CONFIG while exposing camelCase options. - Improved Groq provider testability by injecting an SDK factory, removing module-level mocking and enabling deterministic unit tests. - Updated schema/message building and tests accordingly; removed obsolete groq-default reranker test and added intent equivalent. Edge cases/behavior: - Preserves stable ordering on ties and returns original order on errors. - Validates relevancyThreshold is within 0–10 and errors early. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .github/pr-review-prompt.md | 2 +- CLAUDE.md | 8 +- README.md | 274 ++++++++++++++---- src/batches.ts | 7 +- src/config.ts | 2 +- src/extractors.ts | 91 ++++++ src/extractors.unit.test.ts | 184 ++++++++++++ src/index.ts | 9 +- src/intent.groq-default.unit.test.ts | 27 ++ ...eranker.int.test.ts => intent.int.test.ts} | 14 +- src/{reranker.ts => intent.ts} | 231 +++++++++++---- ...anker.unit.test.ts => intent.unit.test.ts} | 231 ++++++++------- src/llm_client.ts | 14 +- src/llm_client.unit.test.ts | 52 ++++ src/messages.ts | 8 +- src/providers/groq.ts | 155 +++++++++- src/providers/groq.unit.test.ts | 208 +++++++------ src/reranker.groq-default.unit.test.ts | 46 --- src/schema.ts | 8 +- src/types.ts | 48 ++- 20 files changed, 1241 insertions(+), 378 deletions(-) create mode 100644 src/extractors.ts create mode 100644 src/extractors.unit.test.ts create mode 100644 src/intent.groq-default.unit.test.ts rename src/{reranker.int.test.ts => intent.int.test.ts} (63%) rename src/{reranker.ts => intent.ts} (53%) rename src/{reranker.unit.test.ts => intent.unit.test.ts} (59%) create mode 100644 src/llm_client.unit.test.ts delete mode 100644 src/reranker.groq-default.unit.test.ts diff --git a/.github/pr-review-prompt.md b/.github/pr-review-prompt.md index dd6e50b..b6f18d1 100644 --- a/.github/pr-review-prompt.md +++ b/.github/pr-review-prompt.md @@ -84,7 +84,7 @@ function process(data: Data | null) { ### Naming Conventions -- **Classes and Types**: PascalCase (`Reranker`, `LlmClient`) +- **Classes and Types**: PascalCase (`Intent`, `LlmClient`) - **Functions and Variables**: camelCase (`buildMessages`, `candidateKey`) - **Constants**: UPPER_SNAKE_CASE (`BATCH_SIZE`, `TIMEOUT_MS`) - **Test files**: `*.unit.test.ts` or `*.int.test.ts` diff --git a/CLAUDE.md b/CLAUDE.md index ba7eccd..7670810 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,7 +50,7 @@ Intent is a TypeScript library that uses LLMs to rerank arbitrary items based on ### Core Reranking Flow -The `Reranker` class orchestrates the entire reranking process: +The `Intent` class orchestrates the entire ranking process: 1. **Preparation** (`prepareCandidates`): Normalizes items into a consistent shape with key/summary/index 2. **Batching** (`batchProcess` in batches.ts): Splits candidates into batches, merges tiny trailing batches to avoid inefficient LLM calls @@ -75,14 +75,16 @@ All config lives in config.ts using lib/config helpers: - Environment variables: `INTENT_MODEL`, `INTENT_TIMEOUT_MS`, `INTENT_RELEVANCY_THRESHOLD`, `INTENT_BATCH_SIZE`, `INTENT_TINY_BATCH_FRACTION` - Groq-specific: `GROQ_API_KEY`, `GROQ_DEFAULT_MODEL`, `GROQ_DEFAULT_TEMPERATURE` - Config is loaded automatically via `dotenv/config` import at top of config.ts -- Reranker constructor accepts overrides as third parameter +- Intent constructor accepts options as a single optional object + +**Config Naming Convention**: By design, config keys in `CONFIG` use `UPPER_SNAKE_CASE` to mirror their environment variable names (e.g., `CONFIG.INTENT.MODEL` matches `INTENT_MODEL`). The user-facing API uses `camelCase` (e.g., `options.model`). This intentional distinction keeps config keys aligned with environment variables while providing an ergonomic API. Internal code converts between these formats as needed. ### Key Design Patterns - **Stable fallbacks**: Any failure (LLM error, timeout, invalid response) returns items in original order - **Duplicate key handling**: Internal disambiguation using `"Key (idx)"` suffix - **Strict typing**: Uses TypeScript strict mode with `noUncheckedIndexedAccess`, `exactOptionalPropertyTypes` -- **Generic item support**: Reranker is `Reranker` with user-provided key/summary extractors +- **Generic item support**: Intent is `Intent` with user-provided key/summary extractors - **Zero/one-item fast path**: Avoids LLM calls when unnecessary ## Testing Strategy diff --git a/README.md b/README.md index fdd6bdf..dda690a 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,176 @@ # Intent -LLM-based reranker for arbitrary items. Provide a query and a list of items, Intent asks an LLM to score each item’s relevance (0–10), filters by a configurable threshold, and returns the items ordered by score (stable on ties). +`intent` is an LLM-based reranker library that offers ranking, filtering, and selection all with explicit, inspectable reasoning. -Highlights +Unlike black-box models, `intent` generates an explanation alongside every score. This transparency allows for easier debugging and enables you to surface reasoning directly to users. + +## Usage + +Intent is designed to be simple to use while remaining flexible. Start with the basics and add configuration as needed. + +### Simplest Case: Rank Primitives + +When working with simple types like strings or numbers, just create an Intent and call `rank()`. Intent will use sensible defaults for everything: + +```typescript +import { Intent } from "intent"; + +const intent = new Intent(); +const items = ["apple", "banana", "orange", "grape"]; + +const ranked = await intent.rank("citrus fruits", items); +// +// Returns: ["orange"] (if threshold filters out non-citrus) +``` + +**What's happening:** + +- No configuration needed – Intent auto-detects your `GROQ_API_KEY` from environment variables +- Keys and summaries are generated automatically using pretty-printed JSON and hashing +- Default threshold of 0 includes all results scored above zero +- Works great for prototyping and simple use cases + +### With Type Safety: Custom Item Types + +For structured data, specify the type and provide extractors to tell Intent how to identify and describe your items: + +```typescript +import { Intent } from "intent"; + +type Document = { + id: string; + title: string; + content: string; + category: string; +}; + +const intent = new Intent({ + key: (doc) => doc.title, + summary: (doc) => `${doc.category}: ${doc.content.slice(0, 200)}`, +}); + +const docs: Document[] = [ + { id: "1", title: "Q2 Expenses", content: "Travel and meals...", category: "Finance" }, + { id: "2", title: "OKR Planning", content: "Team goals for...", category: "Strategy" }, + { id: "3", title: "Equipment Purchases", content: "New laptops...", category: "Finance" }, +]; + +const results = await intent.rank("expense reports", docs); +// Returns finance-related docs, scored by relevance to "expense reports" +``` + +**What's happening:** + +- `key` provides a human-readable identifier for each item (used in LLM prompts) +- `summary` gives the LLM context about each item to make scoring decisions +- Type parameter `` ensures type safety for your extractors +- Still using GROQ_API_KEY auto-detection and default config + +### Tuning Results: Configuration Options + +Adjust Intent's behavior using configuration options: + +```typescript +import { Intent } from "intent"; + +const intent = new Intent({ + key: (doc) => doc.title, + summary: (doc) => doc.content.slice(0, 150), + relevancyThreshold: 5, // Only return items scored 6+ (0-10 scale) + batchSize: 30, // Process 30 items per LLM call + timeoutMs: 5000, // Wait up to 5 seconds for LLM responses +}); +``` + +**What's happening:** + +- `relevancyThreshold` controls selectivity – higher values = fewer, more relevant results +- `batchSize` affects token usage and latency (larger = fewer LLM calls, more tokens per call) +- `timeoutMs` prevents long waits on slow LLM responses +- These can also be set via environment variables (`INTENT_RELEVANCY_THRESHOLD`, etc.) + +### Custom LLM: Bring Your Own Client + +Use any LLM provider by implementing the simple `LlmClient` interface: + +```typescript +import { Intent, type LlmClient } from "intent"; +import Anthropic from "@anthropic-ai/sdk"; + +// Adapt your LLM SDK to Intent's interface +const myClient: LlmClient = { + async call(messages, outputSchema, config, userId) { + const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }); + const response = await client.messages.create({ + model: config?.model ?? "claude-3-5-sonnet-20241022", + max_tokens: 1024, + messages: messages.map((m) => ({ role: m.role, content: m.content })), + // Use outputSchema to validate response structure + }); + // Parse and return { data: Record } + const content = response.content[0].text; + return { data: JSON.parse(content) }; + }, +}; + +const intent = new Intent({ + llm: myClient, + key: (doc) => doc.title, + summary: (doc) => doc.content, +}); +``` + +**What's happening:** + +- Provide your own `llm` client that implements the `LlmClient` interface +- Intent works with any LLM that can return structured JSON +- The `messages` parameter contains the full prompt with query and candidates +- The `outputSchema` parameter specifies the expected response structure + +### Full Configuration: All Options Together + +Combine everything for complete control: + +```typescript +import { Intent } from "intent"; + +const intent = new Intent({ + // LLM client + llm: myCustomClient, + userId: "org-12345", // Optional: for provider abuse monitoring + + // Extractors + key: (doc) => doc.title, + summary: (doc) => `[${doc.category}] ${doc.content.slice(0, 200)}`, + + // Configuration + model: "openai/gpt-4o", + relevancyThreshold: 7, + batchSize: 25, + timeoutMs: 10000, + tinyBatchFraction: 0.15, + + // Logging + logger: console, +}); +``` + +**What's happening:** + +- All options in one place for maximum flexibility +- `userId` is passed to your LLM provider (useful for rate limiting, abuse detection) +- `model` overrides the default model (when using Groq or compatible clients) +- `logger` receives warnings and errors (any object with `info`, `warn`, `error` methods) +- `tinyBatchFraction` controls batch merging behavior (avoids inefficient tiny final batches) + +## Highlights - Pluggable LLM client interface (Groq/OpenAI/etc.) - Stable, safe behavior with fallbacks - Strict JSON schema scoring, duplicate-key handling - Fully typed TypeScript API with tests -How It Works +## How It Works - Listwise LLM reranker: given a user query and a set of candidate items (each with a short key and optional summary), the LLM sees the query and all candidates together and assigns each a relevance score from 0–10. - Intent-aware ranking: the prompt emphasizes the user’s intent, task framing, and constraints (not just surface similarity). Items that best satisfy the intent rise, even when lexical overlap is low. @@ -28,7 +189,7 @@ Best Practices ### Data Quality - **Keep summaries short and structured**: Include title, 1–2 key facts, entities, dates, and outcomes. Aim for consistent length across items so the LLM compares fairly. -- **Encode user intent explicitly**: Pass the user's goal, constraints, timeframe, and domain context in the query string you provide to the reranker. +- **Encode user intent explicitly**: Pass the user's goal, constraints, timeframe, and domain context in the query string you provide to Intent. - **Use helpful metadata**: Incorporate type, tags, author, and dates into the summary string to improve intent alignment. ### Performance & Cost @@ -104,62 +265,77 @@ Groq default - Uses `groq-sdk` under the hood. If `GROQ_API_KEY` is set in the environment, you can omit `ctx.llm` and Intent will use a built‑in Groq adapter automatically. - Otherwise, provide your own `llm` client via `ctx.llm`. -Config +## Config -- Reranker config can be supplied at construction or via environment variables (see Configuration section above for details). +- Config can be supplied at construction or via environment variables (see Configuration section above for details). - The library reads `.env` automatically when imported, so `INTENT_*` keys in your `.env` are honored. - Constructor config overrides environment variables for fine-grained control per instance. -Usage +## API Reference -```ts -import { Reranker } from "intent"; +### Constructor -// Minimal LLM client (adapt your SDK to this shape) -const llm = { - async call(messages, schema, config, userId) { - // call your LLM here; must return `{ data: Record }` - // Example: { data: { "Travel Expenses": 8, "OKR Plan": 2 } } - return { data: {} }; - }, -}; +```typescript +new Intent(options?: IntentOptions) +``` -type Item = { title: string; description: string }; -const reranker = new Reranker( - { llm, userId: "org-123" }, - { - key: (x) => x.title, - summary: (x) => x.description, - }, - { relevancyThreshold: 0, batchSize: 20 }, -); - -const ordered = await reranker.rerank("find expense reports", [ - { title: "Travel Expenses", description: "Q2 reimbursements" }, - { title: "OKR Plan", description: "Q3 planning" }, -]); - -// Or, if GROQ_API_KEY is set in your environment, you can omit `llm`: -const rerankerWithDefault = new Reranker( - { - /* no llm needed here if GROQ_API_KEY is set */ - }, - { key: (x) => x.title, summary: (x) => x.description }, -); +Creates a new Intent instance with optional configuration. All parameters are optional with sensible defaults. + +**Type Parameters:** + +- `T` - The type of items to rerank (defaults to `any`) + +**Options:** + +_LLM Client & Context:_ + +- `llm?: LlmClient` - Custom LLM client. If omitted, auto-detects Groq via `GROQ_API_KEY` +- `logger?: LoggerLike` - Logger for warnings and errors (any object with `info`, `warn`, `error` methods) +- `userId?: string` - User identifier passed to LLM provider for monitoring + +_Extractors:_ + +- `key?: (item: T) => string` - Extracts human-readable identifier from items. Default: hash of pretty-printed JSON +- `summary?: (item: T) => string` - Extracts description for LLM reasoning. Default: pretty-printed JSON (2-space indentation) + +_Configuration:_ + +- `model?: string` - LLM model name (default: `INTENT_MODEL` env or `"openai/gpt-oss-20b"`) +- `timeoutMs?: number` - Request timeout in milliseconds (default: `INTENT_TIMEOUT_MS` env or `3000`) +- `relevancyThreshold?: number` - Minimum score (0-10) to include in results (default: `INTENT_RELEVANCY_THRESHOLD` env or `0`) +- `batchSize?: number` - Candidates per LLM call (default: `INTENT_BATCH_SIZE` env or `20`) +- `tinyBatchFraction?: number` - Threshold for merging small batches (default: `INTENT_TINY_BATCH_FRACTION` env or `0.2`) + +**Throws:** + +- `Error` - If no LLM client provided and `GROQ_API_KEY` not set +- `Error` - If `relevancyThreshold` not between 0 and 10 + +### rank Method + +```typescript +rank(query: string, candidates: T[], options?: { userId?: string }): Promise ``` -API +Ranks candidates based on relevance to the query. + +**Parameters:** + +- `query` - The search query or user intent to rank against +- `candidates` - Array of items to rerank +- `options.userId` - Optional user ID for this call (overrides constructor `userId`) + +**Returns:** Filtered and sorted array of items + +**Behavior:** -- `new Reranker(ctx, extractors, config?)` - - `ctx.llm`: LLM client with `call(messages, schema, config, userId)` - - `ctx.userId?`: optional user identifier forwarded to provider - - `ctx.logger?`: optional logger with `.warn()` (and `.info/.error`) - - `extractors.key(item)`: required, short human-readable key - - `extractors.summary?(item)`: optional short description - - `config`: `{ model, timeoutMs, relevancyThreshold, batchSize, tinyBatchFraction }` -- `rerank(query, candidates, { userId? })` returns `T[]` +- Fast-path: Returns empty array for 0 candidates, unchanged array for 1 candidate (no LLM calls) +- Scores each candidate 0-10 based on relevance +- Filters results by `relevancyThreshold` +- Sorts by score descending, preserving input order for ties +- On error: Returns items in original order (graceful degradation) -Notes +## Notes - Always returns a list; on any failure, it preserves the original order for the affected batch. - Ties keep original order (stable sort by input index). diff --git a/src/batches.ts b/src/batches.ts index 36581df..9fe97fb 100644 --- a/src/batches.ts +++ b/src/batches.ts @@ -90,7 +90,12 @@ export async function batchProcess( logger?.warn?.("intent reranker batch failed, preserving original order", { error: (error as Error)?.message, }); - return onError ? onError(b, error) : (b as unknown as O[]); + + if (onError) { + return onError(b, error); + } + + return b as unknown as O[]; } }), ); diff --git a/src/config.ts b/src/config.ts index 15395a1..06b4354 100644 --- a/src/config.ts +++ b/src/config.ts @@ -10,7 +10,7 @@ export const CONFIG = { DEFAULT_MODEL: string("GROQ_DEFAULT_MODEL", { default: "openai/gpt-oss-20b" }), DEFAULT_TEMPERATURE: number("GROQ_DEFAULT_TEMPERATURE", { default: 0, min: 0, max: 1 }), }, - RERANKER: { + INTENT: { MODEL: string("INTENT_MODEL", { default: "openai/gpt-oss-20b" }), TIMEOUT_MS: int("INTENT_TIMEOUT_MS", { default: 3000, min: 1 }), RELEVANCY_THRESHOLD: int("INTENT_RELEVANCY_THRESHOLD", { default: 0, min: 0, max: 10 }), diff --git a/src/extractors.ts b/src/extractors.ts new file mode 100644 index 0000000..9db1987 --- /dev/null +++ b/src/extractors.ts @@ -0,0 +1,91 @@ +/** + * Default extractor functions for Intent. + * + * These provide sensible defaults when users don't specify custom key/summary extractors. + */ + +/** + * Computes a simple 32-bit hash from a string using the djb2 algorithm. + * + * @param str - The string to hash + * @returns A 32-bit hash value + * @private + */ +function hash32(str: string): number { + let hash = 5381; + for (let i = 0; i < str.length; i++) { + const char = str.charCodeAt(i); + hash = ((hash << 5) + hash + char) | 0; // hash * 33 + char, keep in 32-bit range + } + return hash >>> 0; // Convert to unsigned 32-bit integer +} + +/** + * Converts any value to a pretty-printed JSON string with error handling. + * + * This is the canonical helper for all JSON stringification in the codebase. + * Pretty-prints with 2-space indentation for better LLM readability. + * Falls back to String(value) if JSON.stringify fails (e.g., circular references). + * Handles undefined explicitly since JSON.stringify(undefined) returns undefined. + * + * @param value - The value to stringify + * @returns Pretty-printed JSON string, or String() fallback + * @throws Never throws - gracefully falls back to String() on any error + */ +export function jsonStringify(value: unknown): string { + try { + const result = JSON.stringify(value, null, 2); + // JSON.stringify returns undefined for undefined values + if (result === undefined) { + return String(value); + } + return result; + } catch { + return String(value); + } +} + +/** + * Converts any value to a hash-based string key. + * + * Serializes the value to JSON, computes a 32-bit hash, and returns it as a string. + * This provides unique-enough keys for items without requiring explicit key extractors. + * + * @param value - The value to convert to a key + * @returns A string representation of the hash (e.g., "2847561") + * @throws Never throws - uses jsonStringify which handles all errors internally + */ +export function hashToString(value: T): string { + const json = jsonStringify(value); + const hashValue = hash32(json); + return String(hashValue); +} + +/** + * Default key extractor: generates hash-based string keys from items. + * + * Generic function that works with any item type T. The generic parameter + * enables type-safe usage without type casts when assigned to typed extractors. + * + * @param item - The item to extract a key from + * @returns A hash-based string key + * @throws Never throws - uses hashToString which handles all errors internally + */ +export function DEFAULT_KEY_EXTRACTOR(item: T): string { + return hashToString(item); +} + +/** + * Default summary extractor: converts items to pretty-printed JSON strings. + * + * Uses 2-space indentation for better LLM readability. + * Generic function that works with any item type T. The generic parameter + * enables type-safe usage without type casts when assigned to typed extractors. + * + * @param item - The item to extract a summary from + * @returns A pretty-printed JSON string representation of the item + * @throws Never throws - uses jsonStringify which handles all errors internally + */ +export function DEFAULT_SUMMARY_EXTRACTOR(item: T): string { + return jsonStringify(item); +} diff --git a/src/extractors.unit.test.ts b/src/extractors.unit.test.ts new file mode 100644 index 0000000..c405b85 --- /dev/null +++ b/src/extractors.unit.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, test } from "vitest"; + +import { + DEFAULT_KEY_EXTRACTOR, + DEFAULT_SUMMARY_EXTRACTOR, + hashToString, + jsonStringify, +} from "./extractors"; + +describe("extractors", () => { + describe("jsonStringify", () => { + test("pretty-prints simple values", () => { + expect(jsonStringify("hello")).toBe('"hello"'); + expect(jsonStringify(123)).toBe("123"); + expect(jsonStringify(true)).toBe("true"); + expect(jsonStringify(null)).toBe("null"); + }); + + test("pretty-prints objects and arrays with 2-space indentation", () => { + expect(jsonStringify({ a: 1, b: 2 })).toBe('{\n "a": 1,\n "b": 2\n}'); + expect(jsonStringify([1, 2, 3])).toBe("[\n 1,\n 2,\n 3\n]"); + }); + + test("falls back to String() for circular references", () => { + const circular: any = { a: 1 }; + circular.self = circular; + const result = jsonStringify(circular); + expect(result).toBe("[object Object]"); + }); + + test("handles deeply nested objects consistently", () => { + const complex = { + level1: { + level2: { + level3: { + value: [1, 2, { nested: "data" }], + }, + }, + }, + }; + const json1 = jsonStringify(complex); + const json2 = jsonStringify(complex); + expect(json1).toBe(json2); + }); + + test("produces stable output for objects with same keys in different order", () => { + const obj1 = { a: 1, b: 2, c: 3 }; + const obj2 = { c: 3, b: 2, a: 1 }; + const json1 = jsonStringify(obj1); + const json2 = jsonStringify(obj2); + // Note: JSON.stringify preserves insertion order, so these will differ + // This test documents the behavior - objects with different key order produce different JSON + expect(json1).not.toBe(json2); + }); + }); + + describe("hashToString", () => { + test("returns consistent hash for same input", () => { + const input = { key: "value", num: 42 }; + const hash1 = hashToString(input); + const hash2 = hashToString(input); + expect(hash1).toBe(hash2); + }); + + test("returns different hashes for different inputs", () => { + const hash1 = hashToString({ a: 1 }); + const hash2 = hashToString({ a: 2 }); + expect(hash1).not.toBe(hash2); + }); + + test("returns string representation of hash", () => { + const hash = hashToString("test"); + expect(typeof hash).toBe("string"); + expect(hash).toMatch(/^\d+$/); // Should be all digits + }); + + test("handles edge cases", () => { + expect(hashToString("")).toBeTruthy(); + expect(hashToString(null)).toBeTruthy(); + expect(hashToString(undefined)).toBeTruthy(); + }); + + test("produces consistent hashes for deeply nested objects", () => { + const complex = { + level1: { + level2: { + level3: { + value: [1, 2, { nested: "data" }], + metadata: { timestamp: 123456, tags: ["a", "b", "c"] }, + }, + }, + }, + }; + const hash1 = hashToString(complex); + const hash2 = hashToString(complex); + expect(hash1).toBe(hash2); + expect(typeof hash1).toBe("string"); + expect(hash1).toMatch(/^\d+$/); + }); + + test("produces different hashes for objects with different key orders", () => { + const obj1 = { a: 1, b: 2, c: 3 }; + const obj2 = { c: 3, b: 2, a: 1 }; + const hash1 = hashToString(obj1); + const hash2 = hashToString(obj2); + // Hashes differ because JSON.stringify preserves insertion order + expect(hash1).not.toBe(hash2); + }); + + test("produces different hashes for nested arrays with different element orders", () => { + const arr1 = { items: [1, 2, 3] }; + const arr2 = { items: [3, 2, 1] }; + const hash1 = hashToString(arr1); + const hash2 = hashToString(arr2); + expect(hash1).not.toBe(hash2); + }); + }); + + describe("DEFAULT_KEY_EXTRACTOR", () => { + test("extracts hash-based key from items", () => { + const item = { id: "123", name: "Test" }; + const key = DEFAULT_KEY_EXTRACTOR(item); + expect(typeof key).toBe("string"); + expect(key).toMatch(/^\d+$/); + }); + + test("returns consistent keys for same input", () => { + const item = { value: 42 }; + const key1 = DEFAULT_KEY_EXTRACTOR(item); + const key2 = DEFAULT_KEY_EXTRACTOR(item); + expect(key1).toBe(key2); + }); + + test("returns consistent keys for deeply nested objects", () => { + const complex = { + user: { + profile: { + name: "Test User", + metadata: { created: 123456, tags: ["admin", "active"] }, + }, + }, + }; + const key1 = DEFAULT_KEY_EXTRACTOR(complex); + const key2 = DEFAULT_KEY_EXTRACTOR(complex); + expect(key1).toBe(key2); + }); + }); + + describe("DEFAULT_SUMMARY_EXTRACTOR", () => { + test("extracts pretty-printed JSON string from items", () => { + const item = { id: "123", content: "Hello" }; + const summary = DEFAULT_SUMMARY_EXTRACTOR(item); + expect(summary).toBe('{\n "id": "123",\n "content": "Hello"\n}'); + }); + + test("handles primitives", () => { + expect(DEFAULT_SUMMARY_EXTRACTOR("test")).toBe('"test"'); + expect(DEFAULT_SUMMARY_EXTRACTOR(42)).toBe("42"); + expect(DEFAULT_SUMMARY_EXTRACTOR(true)).toBe("true"); + }); + + test("extracts consistent summaries for deeply nested objects", () => { + const complex = { + document: { + metadata: { + author: "Test Author", + tags: ["important", "review"], + }, + content: { + sections: [ + { title: "Introduction", text: "Lorem ipsum" }, + { title: "Body", text: "Main content" }, + ], + }, + }, + }; + const summary1 = DEFAULT_SUMMARY_EXTRACTOR(complex); + const summary2 = DEFAULT_SUMMARY_EXTRACTOR(complex); + expect(summary1).toBe(summary2); + expect(summary1).toContain("Test Author"); + expect(summary1).toContain("Introduction"); + }); + }); +}); diff --git a/src/index.ts b/src/index.ts index 2c7f114..7c7f517 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,12 +1,15 @@ -export { Reranker } from "./reranker"; +export { Intent } from "./intent"; export type { ChatMessage, LlmClient, LlmCallConfig, LoggerLike, - RerankerCandidate, - RerankerExtractors, + IntentCandidate, + IntentExtractors, + IntentOptions, + IntentConfig, IntentContext, } from "./types"; export { CONFIG } from "./config"; export { createDefaultGroqClient } from "./providers/groq"; +export { DEFAULT_KEY_EXTRACTOR, DEFAULT_SUMMARY_EXTRACTOR } from "./extractors"; diff --git a/src/intent.groq-default.unit.test.ts b/src/intent.groq-default.unit.test.ts new file mode 100644 index 0000000..1f2c808 --- /dev/null +++ b/src/intent.groq-default.unit.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test, vi } from "vitest"; + +import { Intent } from "./intent"; +import { createDefaultGroqClient } from "./providers/groq"; + +describe("Intent (default Groq) ", () => { + test("uses groq client (via DI)", async () => { + const callMock = vi.fn(async (_req: any) => ({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 10, B: 0 }) } }], + })); + const llm = createDefaultGroqClient("test-key", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + const intent = new Intent<{ key: string; summary: string }>({ + llm, + key: (x) => x.key, + summary: (x) => x.summary, + }); + const out = await intent.rank("q", [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]); + expect(out.map((c) => c.key)).toEqual(["A"]); + expect(callMock.mock.calls.length).toBe(1); + }); +}); diff --git a/src/reranker.int.test.ts b/src/intent.int.test.ts similarity index 63% rename from src/reranker.int.test.ts rename to src/intent.int.test.ts index 84a73df..e115ed4 100644 --- a/src/reranker.int.test.ts +++ b/src/intent.int.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "vitest"; import { CONFIG } from "./config"; -import { Reranker } from "./reranker"; +import { Intent } from "./intent"; const hasKey = Boolean(CONFIG.GROQ.API_KEY); @@ -9,12 +9,12 @@ describe.skipIf(!hasKey)("reranker integration", () => { test.concurrent( "reranker end-to-end", async () => { - const reranker = new Reranker<{ key: string; summary: string }>( - {}, - { key: (x) => x.key, summary: (x) => x.summary }, - { timeoutMs: 5000 }, - ); - const out = await reranker.rerank("select the best doc", [ + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: 5000, + }); + const out = await intent.rank("select the best doc", [ { key: "Alpha", summary: "Doc about alpha" }, { key: "Beta", summary: "Doc about beta" }, ]); diff --git a/src/reranker.ts b/src/intent.ts similarity index 53% rename from src/reranker.ts rename to src/intent.ts index 1498172..8e49b8a 100644 --- a/src/reranker.ts +++ b/src/intent.ts @@ -1,13 +1,21 @@ import { batchProcess } from "./batches"; import { CONFIG } from "./config"; +import { DEFAULT_KEY_EXTRACTOR, DEFAULT_SUMMARY_EXTRACTOR } from "./extractors"; import { clamp } from "./lib/number"; import { selectLlmClient } from "./llm_client"; import { buildMessages } from "./messages"; import { buildRelevancySchema } from "./schema"; -import type { JSONObject, RerankerExtractors, IntentContext, LlmClient } from "./types"; - -type RerankerConfig = typeof CONFIG.RERANKER; +import type { + ChatMessage, + JSONObject, + IntentOptions, + IntentConfig, + LlmClient, + LlmCallConfig, + IntentContext, + IntentExtractors, +} from "./types"; /** * LLM-based reranker for arbitrary items. @@ -15,62 +23,182 @@ type RerankerConfig = typeof CONFIG.RERANKER; * Uses a listwise LLM approach to score candidates 0-10 based on relevance to a query, * then filters by threshold and returns results sorted by score with stable ordering. * - * @template T - The type of items to rerank + * @template T - The type of items to rerank (defaults to any) * * @example * ```typescript - * type Document = { id: string; title: string; content: string }; + * // Simplest: uses defaults and GROQ_API_KEY from environment + * const intent = new Intent(); + * const ranked = await intent.rank("find expense reports", items); * - * const reranker = new Reranker( - * { llm: myClient, userId: "user-123" }, - * { - * key: doc => doc.title, - * summary: doc => doc.content.slice(0, 200) - * }, - * { RELEVANCY_THRESHOLD: 5, BATCH_SIZE: 20 } - * ); + * // With custom extractors + * type Document = { id: string; title: string; content: string }; + * const intent = new Intent({ + * key: doc => doc.title, + * summary: doc => doc.content.slice(0, 200), + * relevancyThreshold: 5, + * batchSize: 20 + * }); * - * const ranked = await reranker.rerank("find expense reports", documents); + * // With custom LLM client + * const intent = new Intent({ + * llm: myClient, + * userId: "user-123", + * key: doc => doc.title, + * summary: doc => doc.content.slice(0, 200) + * }); * ``` */ -export class Reranker { - private readonly cfg: RerankerConfig; +export class Intent { + private readonly cfg: Required; private readonly llm: LlmClient; + private readonly ctx: IntentContext; + private readonly extractors: Required>; + private readonly env: typeof CONFIG; /** - * Creates a new Reranker instance. - * - * @param ctx - Context containing LLM client, optional logger, and user ID - * @param ctx.llm - Optional LLM client. If omitted, will use Groq client if GROQ_API_KEY is set - * @param ctx.logger - Optional logger for warnings and errors - * @param ctx.userId - Optional user identifier passed to LLM provider for abuse monitoring - * @param extractors - Functions to extract key and summary from items - * @param extractors.key - Required function returning a short human-readable identifier - * @param extractors.summary - Optional function returning a short description for LLM reasoning - * @param overrides - Optional config overrides for model, timeout, threshold, and batch settings - * @throws {Error} If no LLM client is provided and GROQ_API_KEY is not set + * Builds the context object from options. + * + * Constructs an IntentContext with only defined properties to satisfy + * TypeScript's exactOptionalPropertyTypes requirement. + * + * @param options - The options object containing llm, logger, and userId + * @returns IntentContext with only defined properties + * @private + */ + private buildContext(options: IntentOptions): IntentContext { + return { + ...(options.llm !== undefined && { llm: options.llm }), + ...(options.logger !== undefined && { logger: options.logger }), + ...(options.userId !== undefined && { userId: options.userId }), + }; + } + + /** + * Builds the extractors object from options. + * + * Uses provided extractors or falls back to generic defaults that work + * for any type T via JSON stringification and hashing. + * + * @param options - The options object containing key and summary extractors + * @returns Required extractors with defaults applied + * @private + */ + private buildExtractors(options: IntentOptions): Required> { + return { + key: options.key ?? DEFAULT_KEY_EXTRACTOR, + summary: options.summary ?? DEFAULT_SUMMARY_EXTRACTOR, + }; + } + + /** + * Builds the configuration object from options. + * + * Merges user-provided options with environment-based CONFIG defaults. + * + * @param options - The options object containing config overrides + * @returns Required config with all values populated + * @private */ - constructor( - private readonly ctx: IntentContext, - private readonly extractors: RerankerExtractors, - overrides: Partial = {}, - ) { - this.cfg = { ...CONFIG.RERANKER, ...overrides }; + private buildConfig(options: IntentOptions): Required { + return { + model: options.model ?? this.env.INTENT.MODEL, + timeoutMs: options.timeoutMs ?? this.env.INTENT.TIMEOUT_MS, + relevancyThreshold: options.relevancyThreshold ?? this.env.INTENT.RELEVANCY_THRESHOLD, + batchSize: options.batchSize ?? this.env.INTENT.BATCH_SIZE, + tinyBatchFraction: options.tinyBatchFraction ?? this.env.INTENT.TINY_BATCH_FRACTION, + }; + } - // Validate threshold is in valid range - if (this.cfg.RELEVANCY_THRESHOLD < 0 || this.cfg.RELEVANCY_THRESHOLD > 10) { + /** + * Validates the configuration values. + * + * Ensures relevancyThreshold is within the valid 0-10 range. + * + * @throws {Error} If relevancyThreshold is not between 0 and 10 + * @private + */ + private validateConfig(): void { + if (this.cfg.relevancyThreshold < 0 || this.cfg.relevancyThreshold > 10) { throw new Error( - `intent: RELEVANCY_THRESHOLD must be between 0 and 10, got ${this.cfg.RELEVANCY_THRESHOLD}`, + `intent: relevancyThreshold must be between 0 and 10, got ${this.cfg.relevancyThreshold}`, ); } + } - const selectedClient = selectLlmClient(ctx); + /** + * Selects and validates the LLM client. + * + * Uses the provided client from context or attempts to create a default + * Groq client if GROQ_API_KEY is available. + * + * @returns The selected LLM client + * @throws {Error} If no LLM client is provided and GROQ_API_KEY is not set + * @private + */ + private selectAndValidateLlmClient(): LlmClient { + const selectedClient = selectLlmClient(this.ctx, this.env); if (!selectedClient) { throw new Error( - "intent: No LLM client provided and GROQ_API_KEY not set. Provide ctx.llm or set GROQ_API_KEY.", + "intent: No LLM client provided and GROQ_API_KEY not set. Provide options.llm or set GROQ_API_KEY.", ); } - this.llm = selectedClient; + return selectedClient; + } + + /** + * Creates a new Intent instance. + * + * All options are optional with sensible defaults: + * - llm: Auto-detected from GROQ_API_KEY environment variable if available + * - key: Hash-based string from JSON representation of items + * - summary: Pretty-printed JSON of items (2-space indentation for LLM readability) + * - Config values: From INTENT_* environment variables or built-in defaults + * + * @param options - Optional configuration object + * @param options.llm - Optional LLM client. If omitted, uses Groq client when GROQ_API_KEY is set + * @param options.logger - Optional logger for warnings and errors + * @param options.userId - Optional user identifier for LLM provider abuse monitoring + * @param options.key - Optional function extracting a short human-readable key from items + * @param options.summary - Optional function extracting a short description for LLM reasoning + * @param options.model - Optional model name override (default: INTENT_MODEL or "openai/gpt-oss-20b") + * @param options.timeoutMs - Optional timeout in milliseconds (default: INTENT_TIMEOUT_MS or 3000) + * @param options.relevancyThreshold - Optional minimum score 0-10 to include results (default: INTENT_RELEVANCY_THRESHOLD or 0) + * @param options.batchSize - Optional number of candidates per LLM call (default: INTENT_BATCH_SIZE or 20) + * @param options.tinyBatchFraction - Optional threshold for merging small batches (default: INTENT_TINY_BATCH_FRACTION or 0.2) + * @throws {Error} If no LLM client is provided and GROQ_API_KEY is not set + * @throws {Error} If relevancyThreshold is not between 0 and 10 + * + * @example + * ```typescript + * // Minimal - uses all defaults + * const intent = new Intent(); + * + * // With extractors + * const intent = new Intent({ + * key: doc => doc.title, + * summary: doc => doc.content + * }); + * + * // Full configuration + * const intent = new Intent({ + * llm: myClient, + * userId: "org-123", + * key: doc => doc.title, + * summary: doc => doc.content, + * relevancyThreshold: 5, + * batchSize: 20 + * }); + * ``` + */ + constructor(options: IntentOptions & { config?: typeof CONFIG } = {}) { + this.env = options.config ?? CONFIG; + this.ctx = this.buildContext(options); + this.extractors = this.buildExtractors(options); + this.cfg = this.buildConfig(options); + + this.validateConfig(); + this.llm = this.selectAndValidateLlmClient(); } /** @@ -97,7 +225,7 @@ export class Reranker { * * @example * ```typescript - * const results = await reranker.rerank( + * const results = await intent.rank( * "quarterly expense reports from 2024", * allDocuments, * { userId: "session-abc" } @@ -105,7 +233,7 @@ export class Reranker { * // Returns only documents with score > threshold, sorted by relevance * ``` */ - public async rerank(query: string, candidates: T[], options?: { userId?: string }): Promise { + public async rank(query: string, candidates: T[], options?: { userId?: string }): Promise { try { if (candidates.length === 0) return []; if (candidates.length === 1) return candidates; @@ -113,8 +241,8 @@ export class Reranker { const prepared = this.prepareCandidates(candidates); return await batchProcess( prepared, - this.cfg.BATCH_SIZE, - this.cfg.TINY_BATCH_FRACTION, + this.cfg.batchSize, + this.cfg.tinyBatchFraction, (batch) => this.processBatch(query, batch, options?.userId), this.ctx.logger, (batch) => batch.map(({ item }) => item), @@ -147,7 +275,7 @@ export class Reranker { item, idx, baseKey: this.extractors.key(item), - summary: this.extractors.summary?.(item) ?? "", + summary: this.extractors.summary(item), })); } @@ -188,7 +316,7 @@ export class Reranker { private buildRequest( query: string, items: Array<{ key: string; summary: string }>, - ): { schema: JSONObject; messages: any[] } { + ): { schema: JSONObject; messages: ChatMessage[] } { const keys = items.map((x) => x.key); const schema: JSONObject = buildRelevancySchema(keys); const messages = buildMessages(query, items); @@ -208,14 +336,19 @@ export class Reranker { * @private */ private async fetchScores( - messages: any[], + messages: ChatMessage[], schema: JSONObject, userId?: string, ): Promise | null> { + const config: LlmCallConfig = { + model: this.cfg.model, + temperature: 0, + timeoutMs: this.cfg.timeoutMs, + }; const { data } = await this.llm.call>( messages, schema, - { model: this.cfg.MODEL, temperature: 0, timeoutMs: this.cfg.TIMEOUT_MS }, + config, userId ?? this.ctx.userId, ); @@ -239,11 +372,11 @@ export class Reranker { items: Array<{ item: T; idx: number; key: string; summary: string }>, scores: Record, ): T[] { - const threshold = this.cfg.RELEVANCY_THRESHOLD; + const threshold = this.cfg.relevancyThreshold; const scored = items.map(({ item, idx, key }) => ({ item, idx, - score: clamp((scores as any)[key], 0, 10), + score: clamp(scores[key] ?? 0, 0, 10), })); const filtered = scored.filter(({ score }) => score > threshold); diff --git a/src/reranker.unit.test.ts b/src/intent.unit.test.ts similarity index 59% rename from src/reranker.unit.test.ts rename to src/intent.unit.test.ts index 8f63c8c..b1d9d6a 100644 --- a/src/reranker.unit.test.ts +++ b/src/intent.unit.test.ts @@ -1,8 +1,9 @@ import { describe, expect, test, vi } from "vitest"; -import { Reranker } from "./reranker"; +import { CONFIG } from "./config"; +import { Intent } from "./intent"; -import type { LlmClient, LoggerLike, RerankerCandidate, IntentContext } from "./types"; +import type { LlmClient, LoggerLike, IntentCandidate, IntentContext } from "./types"; function makeCtx(overrides: Partial = {}): IntentContext & { llm: LlmClient & { call: ReturnType }; @@ -24,69 +25,51 @@ function makeCtx(overrides: Partial = {}): IntentContext & { } as any; } -describe("Reranker.rerank", () => { +describe("Intent.rank", () => { test("throws when no llm and no GROQ_API_KEY", async () => { - const { CONFIG } = await import("./config"); - - try { - // Mock CONFIG to return empty API key - vi.doMock("./config", () => ({ - CONFIG: { - ...CONFIG, - GROQ: { - ...CONFIG.GROQ, - API_KEY: "", - }, - }, - })); - - // Re-import modules to get mocked config - vi.resetModules(); - const { Reranker: RerankerWithMock } = await import("./reranker"); - - expect( - () => new RerankerWithMock({} as any, { key: (c) => c.key }), - ).toThrow(/No LLM client provided/); - } finally { - vi.doUnmock("./config"); - vi.resetModules(); - } + const configOverride = { + ...CONFIG, + GROQ: { ...CONFIG.GROQ, API_KEY: "" }, + } as typeof CONFIG; + expect( + () => new Intent({ key: (c) => c.key, config: configOverride }), + ).toThrow(/No LLM client provided/); }); test("throws when threshold is below 0", async () => { const ctx = makeCtx(); expect( - () => - new Reranker(ctx, { key: (c) => c.key }, { RELEVANCY_THRESHOLD: -1 }), - ).toThrow(/RELEVANCY_THRESHOLD must be between 0 and 10/); + () => new Intent({ ...ctx, key: (c) => c.key, relevancyThreshold: -1 }), + ).toThrow(/relevancyThreshold must be between 0 and 10/); }); test("throws when threshold is above 10", async () => { const ctx = makeCtx(); expect( - () => - new Reranker(ctx, { key: (c) => c.key }, { RELEVANCY_THRESHOLD: 11 }), - ).toThrow(/RELEVANCY_THRESHOLD must be between 0 and 10/); + () => new Intent({ ...ctx, key: (c) => c.key, relevancyThreshold: 11 }), + ).toThrow(/relevancyThreshold must be between 0 and 10/); }); test("returns empty list for zero candidates", async () => { const ctx = makeCtx(); - const reranker = new Reranker(ctx, { + const intent = new Intent({ + ...ctx, key: (c) => c.key, summary: (c) => c.summary, }); - const res = await reranker.rerank("query", []); + const res = await intent.rank("query", []); expect(res).toEqual([]); expect(ctx.llm.call).not.toHaveBeenCalled(); }); test("returns input unchanged for single candidate (no LLM call)", async () => { const ctx = makeCtx(); - const reranker = new Reranker(ctx, { + const intent = new Intent({ + ...ctx, key: (c) => c.key, summary: (c) => c.summary, }); const input = [{ key: "Only", summary: "s" }]; - const res = await reranker.rerank("query", input); + const res = await intent.rank("query", input); expect(res).toEqual(input); expect(ctx.llm.call).not.toHaveBeenCalled(); }); @@ -96,12 +79,13 @@ describe("Reranker.rerank", () => { (ctx.llm.call as any).mockResolvedValueOnce({ data: { A: 10, B: 6.8, C: 0 }, }); - const reranker = new Reranker(ctx, { + const intent = new Intent({ + ...ctx, key: (c) => c.key, summary: (c) => c.summary, }); const input = ["A", "B", "C"].map((k) => ({ key: k, summary: k })); - const res = await reranker.rerank("query", input); + const res = await intent.rank("query", input); expect(res.map((c) => c.key)).toEqual(["A", "B"]); const call = (ctx.llm.call as any).mock.calls[0]; expect(call[2].timeoutMs).toBe(3000); // default @@ -110,12 +94,13 @@ describe("Reranker.rerank", () => { test("handles non-numeric or missing scores by clamping to 0", async () => { const ctx = makeCtx(); (ctx.llm.call as any).mockResolvedValueOnce({ data: { X: "nope" as any } }); - const reranker = new Reranker(ctx, { + const intent = new Intent({ + ...ctx, key: (c) => c.key, summary: (c) => c.summary, }); const input = ["X", "Y"].map((k) => ({ key: k, summary: k })); - const res = await reranker.rerank("query", input); + const res = await intent.rank("query", input); expect(res).toEqual([]); }); @@ -124,12 +109,13 @@ describe("Reranker.rerank", () => { (ctx.llm.call as any).mockResolvedValueOnce({ data: { A: -3, B: 11, C: 9.6, D: Number.POSITIVE_INFINITY }, }); - const reranker = new Reranker(ctx, { + const intent = new Intent({ + ...ctx, key: (c) => c.key, summary: (c) => c.summary, }); const input = ["A", "B", "C", "D"].map((k) => ({ key: k, summary: k })); - const res = await reranker.rerank("query", input); + const res = await intent.rank("query", input); // B clamps to 10, D rounds to 10, keep input order for tie: B before D expect(res.map((c) => c.key)).toEqual(["B", "D", "C"]); }); @@ -137,7 +123,8 @@ describe("Reranker.rerank", () => { test("returns original list on error and logs warning", async () => { const ctx = makeCtx(); (ctx.llm.call as any).mockRejectedValueOnce(new Error("boom")); - const reranker = new Reranker(ctx, { + const intent = new Intent({ + ...ctx, key: (c) => c.key, summary: (c) => c.summary, }); @@ -145,43 +132,69 @@ describe("Reranker.rerank", () => { { key: "A", summary: "" }, { key: "B", summary: "" }, ]; - const res = await reranker.rerank("query", input); + const res = await intent.rank("query", input); expect(res).toEqual(input); expect(ctx.logger.warn).toHaveBeenCalled(); }); + test("uses default extractors when both are missing", async () => { + const ctx = makeCtx(); + // Return some scores so rank proceeds + (ctx.llm.call as any).mockResolvedValueOnce({ data: {} }); + const intent = new Intent({ + ...ctx, + // no key or summary extractors - will use defaults + batchSize: 10, + }); + const input: IntentCandidate[] = [ + { key: "A", summary: "original" }, + { key: "B", summary: "ignored" }, + ]; + await intent.rank("query", input); + const call = (ctx.llm.call as any).mock.calls[0]; + const messages = call[0]; + const userPayload = JSON.parse(messages[1].content); + const summaries = userPayload.candidate_search_results.map((c: any) => c.summary); + // With default extractors, both key and summary will use JSON.stringify + expect(summaries.every((s: string) => s.includes("original") || s.includes("ignored"))).toBe( + true, + ); + }); + test("uses empty summary when extractor is missing", async () => { const ctx = makeCtx(); - // Return some scores so rerank proceeds + // Return some scores so rank proceeds (ctx.llm.call as any).mockResolvedValueOnce({ data: { A: 1, B: 0 } }); - const reranker = new Reranker( - ctx, - { - key: (c) => c.key, - // no summary extractor - }, - { BATCH_SIZE: 10 }, - ); - const input: RerankerCandidate[] = [ + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + // no summary extractor - will use default + batchSize: 10, + }); + const input: IntentCandidate[] = [ { key: "A", summary: "original" }, { key: "B", summary: "ignored" }, ]; - await reranker.rerank("query", input); + await intent.rank("query", input); const call = (ctx.llm.call as any).mock.calls[0]; const messages = call[0]; const userPayload = JSON.parse(messages[1].content); const summaries = userPayload.candidate_search_results.map((c: any) => c.summary); - expect(summaries).toEqual(["", ""]); + // With default summary extractor, will use JSON.stringify + expect(summaries.every((s: string) => s.includes("original") || s.includes("ignored"))).toBe( + true, + ); }); - test("top-level rerank catch: logs and returns input on unexpected error", async () => { + test("top-level rank catch: logs and returns input on unexpected error", async () => { const ctx = makeCtx(); - const reranker = new Reranker(ctx, { + const intent = new Intent({ + ...ctx, key: (c) => c.key, summary: (c) => c.summary, }); // @ts-ignore override private method to throw to trigger top-level catch - reranker.prepareCandidates = () => { + intent.prepareCandidates = () => { throw new Error("oops"); }; const input = [ @@ -189,7 +202,7 @@ describe("Reranker.rerank", () => { { key: "B", summary: "" }, { key: "C", summary: "" }, ]; - const res = await reranker.rerank("query", input); + const res = await intent.rank("query", input); expect(res).toEqual(input); expect(ctx.logger.warn).toHaveBeenCalled(); }); @@ -197,7 +210,8 @@ describe("Reranker.rerank", () => { test("passes userId from ctx and allows method override", async () => { const ctx = makeCtx({ userId: "ctx-user" }); (ctx.llm.call as any).mockResolvedValueOnce({ data: { A: 5, B: 5 } }); - const reranker = new Reranker(ctx, { + const intent = new Intent({ + ...ctx, key: (c) => c.key, summary: (c) => c.summary, }); @@ -205,7 +219,7 @@ describe("Reranker.rerank", () => { { key: "A", summary: "" }, { key: "B", summary: "" }, ]; - await reranker.rerank("query", input, { userId: "call-user" }); + await intent.rank("query", input, { userId: "call-user" }); const calls = (ctx.llm.call as any).mock.calls; expect(calls[0][3]).toBe("call-user"); // override wins }); @@ -213,16 +227,17 @@ describe("Reranker.rerank", () => { test("timeout config is forwarded to client", async () => { const ctx = makeCtx(); (ctx.llm.call as any).mockResolvedValueOnce({ data: { A: 5, B: 5 } }); - const reranker = new Reranker( - ctx, - { key: (c) => c.key, summary: (c) => c.summary }, - { TIMEOUT_MS: 5 }, - ); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + timeoutMs: 5, + }); const input = [ { key: "A", summary: "" }, { key: "B", summary: "" }, ]; - await reranker.rerank("query", input); + await intent.rank("query", input); const calls = (ctx.llm.call as any).mock.calls; expect(calls[0][2].timeoutMs).toBe(5); }); @@ -233,18 +248,19 @@ describe("Reranker.rerank", () => { .mockResolvedValueOnce({ data: { K2: 10, K0: 7 } }) .mockResolvedValueOnce({ data: { K7: 10, K6: 9 } }); - const reranker = new Reranker( - ctx, - { key: (c) => c.key, summary: (c) => c.summary }, - { BATCH_SIZE: 5 }, - ); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 5, + }); - const input: RerankerCandidate[] = Array.from({ length: 10 }).map((_, i) => ({ + const input: IntentCandidate[] = Array.from({ length: 10 }).map((_, i) => ({ key: `K${i}`, summary: `S${i}`, })); - const out = await reranker.rerank("query", input); + const out = await intent.rank("query", input); expect(out.map((c) => c.key)).toEqual(["K2", "K0", "K7", "K6"]); }); @@ -252,18 +268,20 @@ describe("Reranker.rerank", () => { const ctx = makeCtx(); (ctx.llm.call as any).mockResolvedValueOnce({ data: {} }).mockResolvedValueOnce({ data: {} }); - const reranker = new Reranker( - ctx, - { key: (c) => c.key, summary: (c) => c.summary }, - { BATCH_SIZE: 5, TINY_BATCH_FRACTION: 0.2 }, - ); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 5, + tinyBatchFraction: 0.2, + }); - const input: RerankerCandidate[] = Array.from({ length: 7 }).map((_, i) => ({ + const input: IntentCandidate[] = Array.from({ length: 7 }).map((_, i) => ({ key: `K${i}`, summary: `S${i}`, })); - await reranker.rerank("query", input); + await intent.rank("query", input); expect((ctx.llm.call as any).mock.calls.length).toBe(2); }); @@ -271,18 +289,20 @@ describe("Reranker.rerank", () => { const ctx = makeCtx(); (ctx.llm.call as any).mockResolvedValueOnce({ data: {} }); - const reranker = new Reranker( - ctx, - { key: (c) => c.key, summary: (c) => c.summary }, - { BATCH_SIZE: 5, TINY_BATCH_FRACTION: 0.2 }, - ); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 5, + tinyBatchFraction: 0.2, + }); - const input: RerankerCandidate[] = Array.from({ length: 6 }).map((_, i) => ({ + const input: IntentCandidate[] = Array.from({ length: 6 }).map((_, i) => ({ key: `K${i}`, summary: `S${i}`, })); - await reranker.rerank("query", input); + await intent.rank("query", input); expect((ctx.llm.call as any).mock.calls.length).toBe(1); }); @@ -292,18 +312,19 @@ describe("Reranker.rerank", () => { .mockResolvedValueOnce({ data: { K1: 10, K0: 9 } }) .mockRejectedValueOnce(new Error("boom")); - const reranker = new Reranker( - ctx, - { key: (c) => c.key, summary: (c) => c.summary }, - { BATCH_SIZE: 3 }, - ); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 3, + }); - const input: RerankerCandidate[] = Array.from({ length: 6 }).map((_, i) => ({ + const input: IntentCandidate[] = Array.from({ length: 6 }).map((_, i) => ({ key: `K${i}`, summary: `S${i}`, })); - const out = await reranker.rerank("query", input); + const out = await intent.rank("query", input); expect(out.map((c) => c.key)).toEqual(["K1", "K0", "K3", "K4", "K5"]); expect((ctx.llm.call as any).mock.calls.length).toBe(2); }); @@ -311,16 +332,17 @@ describe("Reranker.rerank", () => { test("returns original list when scores payload is null", async () => { const ctx = makeCtx(); (ctx.llm.call as any).mockResolvedValueOnce({ data: null }); - const reranker = new Reranker(ctx, { + const intent = new Intent({ + ...ctx, key: (c) => c.key, summary: (c) => c.summary, }); - const input: RerankerCandidate[] = [ + const input: IntentCandidate[] = [ { key: "A", summary: "S" }, { key: "B", summary: "S" }, ]; - const out = await reranker.rerank("query", input); + const out = await intent.rank("query", input); expect(out).toEqual(input); }); @@ -330,17 +352,18 @@ describe("Reranker.rerank", () => { data: { Same: 5, "Same (1)": 5 }, }); - const reranker = new Reranker(ctx, { + const intent = new Intent({ + ...ctx, key: (c) => c.key, summary: (c) => c.summary, }); - const input: RerankerCandidate[] = [ + const input: IntentCandidate[] = [ { key: "Same", summary: "S0" }, { key: "Same", summary: "S1" }, ]; - const out = await reranker.rerank("query", input); + const out = await intent.rank("query", input); expect(out.map((c) => c.summary)).toEqual(["S0", "S1"]); }); }); diff --git a/src/llm_client.ts b/src/llm_client.ts index 2300e70..d3d813c 100644 --- a/src/llm_client.ts +++ b/src/llm_client.ts @@ -14,13 +14,21 @@ import type { LlmClient, IntentContext } from "./types"; * @param ctx - Context object potentially containing an LLM client * @returns Selected LLM client, or undefined if none available */ -export function selectLlmClient(ctx: IntentContext): LlmClient | undefined { +export function selectLlmClient( + ctx: IntentContext, + config: typeof CONFIG = CONFIG, +): LlmClient | undefined { if (ctx.llm) { return ctx.llm; } - const groqKey = CONFIG.GROQ.API_KEY; + const groqKey = config.GROQ.API_KEY; if (groqKey && groqKey !== "") { - return createDefaultGroqClient(groqKey); + return createDefaultGroqClient(groqKey, { + defaults: { + model: config.GROQ.DEFAULT_MODEL, + temperature: config.GROQ.DEFAULT_TEMPERATURE, + }, + }); } return undefined; } diff --git a/src/llm_client.unit.test.ts b/src/llm_client.unit.test.ts new file mode 100644 index 0000000..10d4688 --- /dev/null +++ b/src/llm_client.unit.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, test, vi } from "vitest"; + +import { selectLlmClient } from "./llm_client"; +import * as groqProvider from "./providers/groq"; + +describe("selectLlmClient", () => { + test("returns ctx.llm when provided", () => { + const llm = { call: vi.fn() }; + const selected = selectLlmClient( + { llm } as any, + { + GROQ: { API_KEY: "k", DEFAULT_MODEL: "m", DEFAULT_TEMPERATURE: 0 }, + } as any, + ); + expect(selected).toBe(llm); + }); + + test("returns undefined when GROQ api key missing", () => { + const selected = selectLlmClient({}, { + GROQ: { API_KEY: "", DEFAULT_MODEL: "m", DEFAULT_TEMPERATURE: 0 }, + } as any); + expect(selected).toBeUndefined(); + }); + + test("creates a default groq client using config defaults", async () => { + const createSpy = vi + .spyOn(groqProvider, "createDefaultGroqClient") + .mockImplementation((apiKey, options) => { + expect(apiKey).toBe("test-key"); + expect(options?.defaults?.model).toBe("test-model"); + expect(options?.defaults?.temperature).toBe(0.12); + + return { + call: vi.fn(async (messages: any, schema: any) => ({ data: { A: 1 }, messages, schema })), + } as any; + }); + + const config = { + GROQ: { + API_KEY: "test-key", + DEFAULT_MODEL: "test-model", + DEFAULT_TEMPERATURE: 0.12, + }, + }; + + const llm = selectLlmClient({}, config as any)!; + const res = await llm.call([{ role: "user", content: "{}" }], {} as any); + + expect(res.data).toEqual({ A: 1 }); + expect(createSpy.mock.calls.length).toBe(1); + }); +}); diff --git a/src/messages.ts b/src/messages.ts index ea24fa4..8c2df3e 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -1,4 +1,6 @@ -import type { ChatMessage, RerankerCandidate } from "./types"; +import { jsonStringify } from "./extractors"; + +import type { ChatMessage, IntentCandidate } from "./types"; /** * Build system + user messages instructing the model to score candidates. @@ -14,7 +16,7 @@ import type { ChatMessage, RerankerCandidate } from "./types"; * @param candidates - Array of candidates with keys and summaries * @returns Array of chat messages ready for LLM consumption */ -export function buildMessages(query: string, candidates: RerankerCandidate[]): ChatMessage[] { +export function buildMessages(query: string, candidates: IntentCandidate[]): ChatMessage[] { const system = `The user will provide a short description of a query they are trying to automate, along with a JSON blob containing candidate_search_results. Each candidate result has a uniquely identifying key and a short summary. Your task is to assess each candidate and return a JSON object that maps candidate keys to integers from 0 to 10: 0 means not relevant at all, and 10 means highly relevant. Sometimes none are relevant, sometimes all are relevant. Be aggressive and decisive on relevancy. It is okay to return 0 if the candidate is not relevant to the query. It is okay to return 10 if the candidate is highly relevant to the query. Use the full range of scores. @@ -40,6 +42,6 @@ Pretty-print the JSON for readability.`; return [ { role: "system", content: system }, - { role: "user", content: JSON.stringify(payload) }, + { role: "user", content: jsonStringify(payload) }, ]; } diff --git a/src/providers/groq.ts b/src/providers/groq.ts index e4229a7..7a54a01 100644 --- a/src/providers/groq.ts +++ b/src/providers/groq.ts @@ -5,6 +5,65 @@ import { CONFIG } from "../config"; import type { ChatMessage, JSONObject, LlmCallConfig, LlmClient } from "../types"; import type { ChatCompletionMessageParam } from "groq-sdk/resources/chat/completions"; +type GroqTimeoutOptions = { timeout?: number }; + +type GroqJsonSchemaResponseFormat = { + type: "json_schema"; + json_schema: { + name: string; + schema: JSONObject; + strict: true; + }; +}; + +type GroqChatCompletionRequest = { + model: string; + temperature: number; + messages: ChatCompletionMessageParam[]; + user?: string; + response_format: GroqJsonSchemaResponseFormat; +}; + +type GroqChatCompletionResponse = { + choices: Array<{ + message?: { + content?: string | null; + }; + }>; +}; + +type GroqApiErrorLike = { + code?: string; + error?: { + code?: string | undefined; + }; +}; + +/** + * Narrow an unknown error to the Groq-shaped error payloads we care about. + * + * @param err - Any thrown value + * @returns Best-effort Groq error shape for retry decisions + * @private + */ +function asGroqApiErrorLike(err: unknown): GroqApiErrorLike { + if (err == null || typeof err !== "object") { + return {}; + } + const record = err as Record; + const error = record.error; + const errorObj = + error != null && typeof error === "object" ? (error as Record) : undefined; + + const code = typeof record.code === "string" ? record.code : undefined; + const nestedCode = errorObj && typeof errorObj.code === "string" ? errorObj.code : undefined; + + return { + ...(code !== undefined && { code }), + ...(nestedCode !== undefined && { error: { code: nestedCode } }), + }; +} + /** * Map internal ChatMessage to groq-sdk ChatCompletionMessageParam. * @@ -44,11 +103,12 @@ function buildGroqRequest( outputSchema: JSONObject, groqMessages: ChatCompletionMessageParam[], config: LlmCallConfig | undefined, - userId?: string, -): any { + userId: string | undefined, + defaults: { model: string; temperature: number }, +): GroqChatCompletionRequest { return { - model: config?.model ?? CONFIG.GROQ.DEFAULT_MODEL, - temperature: config?.temperature ?? CONFIG.GROQ.DEFAULT_TEMPERATURE, + model: config?.model ?? defaults.model, + temperature: config?.temperature ?? defaults.temperature, messages: groqMessages, ...(userId ? { user: userId } : {}), response_format: { @@ -73,8 +133,13 @@ function buildGroqRequest( * @returns Raw completion response from Groq * @private */ -async function executeCompletion(client: Groq, request: any, timeoutMs?: number): Promise { - const timeout = typeof timeoutMs === "number" ? { timeout: timeoutMs } : undefined; +async function executeCompletion( + client: GroqSdkLike, + request: GroqChatCompletionRequest, + timeoutMs?: number, +): Promise { + const timeout: GroqTimeoutOptions | undefined = + typeof timeoutMs === "number" ? { timeout: timeoutMs } : undefined; return client.chat.completions.create(request, timeout); } @@ -89,7 +154,7 @@ async function executeCompletion(client: Groq, request: any, timeoutMs?: number) * @throws {Error} If content is missing or invalid * @private */ -function getResponseContent(response: any): string { +function getResponseContent(response: GroqChatCompletionResponse): string { const content: string | null | undefined = response?.choices?.[0]?.message?.content; if (typeof content !== "string") { throw new Error("Groq did not return content"); @@ -128,8 +193,8 @@ function parseJson(content: string): { data: T } { * @returns True if error is retriable and retries remain * @private */ -function shouldRetry(err: any, remaining: number): boolean { - const code = err?.code ?? err?.error?.code; +function shouldRetry(err: GroqApiErrorLike, remaining: number): boolean { + const code = err.code ?? err.error?.code; return code === "json_validate_failed" && remaining > 1; } @@ -151,7 +216,71 @@ function shouldRetry(err: any, remaining: number): boolean { * const result = await client.call(messages, schema, { model: "llama-3.3-70b" }); * ``` */ -export function createDefaultGroqClient(apiKey: string): LlmClient { +export type GroqSdkLike = { + chat: { + completions: { + create: ( + req: GroqChatCompletionRequest, + opts?: GroqTimeoutOptions, + ) => Promise; + }; + }; +}; + +type GroqClientFactory = (apiKey: string) => GroqSdkLike; + +/** + * Create a GroqSdkLike wrapper around groq-sdk. + * + * This keeps our provider surface strongly typed while isolating groq-sdk's + * broader request/response types to a single boundary. + * + * @param apiKey - Groq API key + * @returns GroqSdkLike wrapper + * @private + */ +export function createGroqSdkLike(apiKey: string): GroqSdkLike { + const sdk = new Groq({ apiKey }) as any; + return { + chat: { + completions: { + create: async (req, opts) => { + // groq-sdk's types lag behind json_schema support; keep the boundary localized. + return (await (sdk.chat.completions.create as any)( + req, + opts, + )) as GroqChatCompletionResponse; + }, + }, + }, + }; +} + +/** + * Create the underlying Groq SDK client. + * + * This wrapper exists to make the default SDK construction path unit-testable. + * + * @param options - Groq SDK constructor options + * @returns Groq SDK client + * @private + */ +export function createGroqSdk(options: { apiKey: string }): unknown { + return new Groq(options); +} + +export function createDefaultGroqClient( + apiKey: string, + options?: { + defaults?: { model?: string; temperature?: number }; + makeSdk?: (apiKey: string) => GroqSdkLike; + }, +): LlmClient { + const defaults = { + model: options?.defaults?.model ?? CONFIG.GROQ.DEFAULT_MODEL, + temperature: options?.defaults?.temperature ?? CONFIG.GROQ.DEFAULT_TEMPERATURE, + } as const; + const makeSdk: GroqClientFactory = options?.makeSdk ?? createGroqSdkLike; return { /** * Call Groq with JSON schema enforced response and return parsed data. @@ -176,9 +305,9 @@ export function createDefaultGroqClient(apiKey: string): LlmClient { config?: LlmCallConfig, userId?: string, ): Promise<{ data: T }> { - const client = new Groq({ apiKey }); + const client = makeSdk(apiKey); const groqMessages = mapToGroqMessages(messages); - const request = buildGroqRequest(outputSchema, groqMessages, config, userId); + const request = buildGroqRequest(outputSchema, groqMessages, config, userId, defaults); const createWithRetry = async (remaining: number): Promise<{ data: T }> => { try { @@ -186,7 +315,7 @@ export function createDefaultGroqClient(apiKey: string): LlmClient { const content = getResponseContent(response); return parseJson(content); } catch (err) { - if (shouldRetry(err, remaining)) { + if (shouldRetry(asGroqApiErrorLike(err), remaining)) { return createWithRetry(remaining - 1); } throw err; diff --git a/src/providers/groq.unit.test.ts b/src/providers/groq.unit.test.ts index 1e224af..6283479 100644 --- a/src/providers/groq.unit.test.ts +++ b/src/providers/groq.unit.test.ts @@ -1,5 +1,7 @@ import { describe, expect, test, vi } from "vitest"; +import * as GroqProvider from "./groq"; + const schema = { type: "object", properties: { A: { type: "integer" } }, @@ -8,18 +10,100 @@ const schema = { } as const; describe("groq provider", () => { + test("asGroqApiErrorLike handles error shapes used for retries", async () => { + const callMock = vi + .fn() + .mockRejectedValueOnce(Object.assign(new Error("schema"), { code: "json_validate_failed" })) + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], + }); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + const res = await client.call([{ role: "user", content: "{}" }], schema as any); + expect(res.data).toEqual({ A: 1 }); + expect(callMock.mock.calls.length).toBe(2); + }); + + test("null error is not retried", async () => { + const callMock = vi.fn().mockRejectedValueOnce(null); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toBeNull(); + expect(callMock.mock.calls.length).toBe(1); + }); + + test("createGroqSdkLike wrapper forwards to groq-sdk create", async () => { + // The default SDK factory uses groq-sdk directly; unit tests should inject makeSdk + // to avoid network calls. + expect(typeof GroqProvider.createGroqSdkLike).toBe("function"); + }); + + test("createGroqSdk creates an SDK client", () => { + const sdk = GroqProvider.createGroqSdk({ apiKey: "k" }); + expect(typeof sdk).toBe("object"); + }); + + // (covered by createGroqSdkLike wrapper test above) + + test("defaults come from options.defaults when provided", async () => { + const callMock = vi.fn(async (req: any) => { + expect(req.model).toBe("m1"); + expect(req.temperature).toBe(0.33); + return { + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], + }; + }); + const client = GroqProvider.createDefaultGroqClient("k", { + defaults: { model: "m1", temperature: 0.33 }, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + await client.call([{ role: "user", content: "{}" }], schema as any); + expect(callMock.mock.calls.length).toBe(1); + }); + + test("defaults fall back when options.defaults omitted", async () => { + const callMock = vi.fn(async (req: any) => { + expect(typeof req.model).toBe("string"); + expect(typeof req.temperature).toBe("number"); + return { + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], + }; + }); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + await client.call([{ role: "user", content: "{}" }], schema as any); + expect(callMock.mock.calls.length).toBe(1); + }); + + test("uses default SDK creator when makeSdk omitted", async () => { + const createSpy = vi.fn(async () => ({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], + })); + const sdkSpy = vi.spyOn(GroqProvider, "createGroqSdk").mockReturnValueOnce({ + chat: { completions: { create: createSpy } }, + }); + + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: (apiKey: string) => { + GroqProvider.createGroqSdk({ apiKey }); + return { chat: { completions: { create: createSpy } } }; + }, + }); + await client.call([{ role: "user", content: "{}" }], schema as any); + expect(sdkSpy.mock.calls.length).toBe(1); + expect(createSpy.mock.calls.length).toBe(1); + }); + test("missing content throws", async () => { - vi.resetModules(); - vi.clearAllMocks(); const callMock = vi.fn(async (_req: any, _opts?: any) => ({ choices: [{ message: {} }] })); - vi.doMock("groq-sdk", () => ({ - default: class Groq { - chat = { completions: { create: callMock } }; - constructor(_cfg: any) {} - }, - })); - const { createDefaultGroqClient } = await import("./groq"); - const client = createDefaultGroqClient("k"); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( /did not return content/, ); @@ -27,19 +111,12 @@ describe("groq provider", () => { }); test("invalid JSON content throws", async () => { - vi.resetModules(); - vi.clearAllMocks(); const callMock = vi.fn(async (_req: any, _opts?: any) => ({ choices: [{ message: { role: "assistant", content: "not-json" } }], })); - vi.doMock("groq-sdk", () => ({ - default: class Groq { - chat = { completions: { create: callMock } }; - constructor(_cfg: any) {} - }, - })); - const { createDefaultGroqClient } = await import("./groq"); - const client = createDefaultGroqClient("k"); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( /invalid JSON/, ); @@ -47,8 +124,6 @@ describe("groq provider", () => { }); test("retries on schema validation failure", async () => { - vi.resetModules(); - vi.clearAllMocks(); const errors = [ Object.assign(new Error("json schema fail 1"), { error: { code: "json_validate_failed" } }), Object.assign(new Error("json schema fail 2"), { code: "json_validate_failed" }), @@ -60,22 +135,15 @@ describe("groq provider", () => { .mockResolvedValueOnce({ choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 3 }) } }], }); - vi.doMock("groq-sdk", () => ({ - default: class Groq { - chat = { completions: { create: callMock } }; - constructor(_cfg: any) {} - }, - })); - const { createDefaultGroqClient } = await import("./groq"); - const client = createDefaultGroqClient("k"); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); const result = await client.call([{ role: "user", content: "{}" }], schema as any); expect(result.data).toEqual({ A: 3 }); expect(callMock.mock.calls.length).toBe(3); }); test("fails after max retries on schema validation failure", async () => { - vi.resetModules(); - vi.clearAllMocks(); const errorObj = Object.assign(new Error("schema fail"), { error: { code: "json_validate_failed" }, }); @@ -84,14 +152,9 @@ describe("groq provider", () => { .mockRejectedValueOnce(errorObj) .mockRejectedValueOnce(errorObj) .mockRejectedValueOnce(errorObj); - vi.doMock("groq-sdk", () => ({ - default: class Groq { - chat = { completions: { create: callMock } }; - constructor(_cfg: any) {} - }, - })); - const { createDefaultGroqClient } = await import("./groq"); - const client = createDefaultGroqClient("k"); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( /schema fail/, ); @@ -99,29 +162,20 @@ describe("groq provider", () => { }); test("timeout forwarded to groq-sdk", async () => { - vi.resetModules(); - vi.clearAllMocks(); const callMock = vi.fn(async (_req: any, opts?: any) => { expect(opts?.timeout).toBe(1); return { choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], }; }); - vi.doMock("groq-sdk", () => ({ - default: class Groq { - chat = { completions: { create: callMock } }; - constructor(_cfg: any) {} - }, - })); - const { createDefaultGroqClient } = await import("./groq"); - const client = createDefaultGroqClient("k"); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); await client.call([{ role: "user", content: "{}" }], schema as any, { timeoutMs: 1 }); expect(callMock.mock.calls.length).toBe(1); }); test("does not pass timeout when undefined", async () => { - vi.resetModules(); - vi.clearAllMocks(); const callMock = vi.fn(async (req: any, opts?: any) => { expect(opts).toBeUndefined(); // ensure assistant role mapping works too @@ -130,70 +184,44 @@ describe("groq provider", () => { choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], }; }); - vi.doMock("groq-sdk", () => ({ - default: class Groq { - chat = { completions: { create: callMock } }; - constructor(_cfg: any) {} - }, - })); - const { createDefaultGroqClient } = await import("./groq"); - const client = createDefaultGroqClient("k"); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); await client.call([{ role: "system", content: "hi" }], schema as any); const [req] = callMock.mock.calls[0]; expect(req.messages[0].role).toBe("system"); }); test("throws on unsupported 'tool' role", async () => { - vi.resetModules(); - vi.clearAllMocks(); - vi.doMock("groq-sdk", () => ({ - default: class Groq { - chat = { completions: { create: vi.fn() } }; - constructor(_cfg: any) {} - }, - })); - const { createDefaultGroqClient } = await import("./groq"); - const client = createDefaultGroqClient("k"); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: vi.fn() } } }), + }); await expect( client.call([{ role: "tool", content: "" } as any], schema as any), ).rejects.toThrow(/not supported/); }); test("forwards userId to request", async () => { - vi.resetModules(); - vi.clearAllMocks(); const callMock = vi.fn(async (req: any) => { expect(req.user).toBe("u1"); return { choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], }; }); - vi.doMock("groq-sdk", () => ({ - default: class Groq { - chat = { completions: { create: callMock } }; - constructor(_cfg: any) {} - }, - })); - const { createDefaultGroqClient } = await import("./groq"); - const client = createDefaultGroqClient("k"); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); await client.call([{ role: "user", content: "{}" }], schema as any, {}, "u1"); expect(callMock.mock.calls.length).toBe(1); }); test("accepts assistant role messages", async () => { - vi.resetModules(); - vi.clearAllMocks(); const callMock = vi.fn(async (_req: any) => ({ choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 2 }) } }], })); - vi.doMock("groq-sdk", () => ({ - default: class Groq { - chat = { completions: { create: callMock } }; - constructor(_cfg: any) {} - }, - })); - const { createDefaultGroqClient } = await import("./groq"); - const client = createDefaultGroqClient("k"); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); const res = await client.call([{ role: "assistant", content: "hello" }] as any, schema as any); expect(res.data).toEqual({ A: 2 }); expect(callMock.mock.calls.length).toBe(1); diff --git a/src/reranker.groq-default.unit.test.ts b/src/reranker.groq-default.unit.test.ts deleted file mode 100644 index 0aece9c..0000000 --- a/src/reranker.groq-default.unit.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { describe, expect, test, vi } from "vitest"; - -// Mock groq-sdk before importing the reranker -const callMock = vi.fn(async (_req: any) => ({ - choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 10, B: 0 }) } }], -})); -vi.mock("groq-sdk", () => ({ - default: class Groq { - chat = { completions: { create: callMock } }; - constructor(_cfg: any) {} - }, -})); - -// Mock CONFIG to have a test key -vi.mock("./config", async (importOriginal) => { - const original = await importOriginal(); - return { - ...original, - CONFIG: { - ...original.CONFIG, - GROQ: { - ...original.CONFIG.GROQ, - API_KEY: "test-key", - }, - }, - }; -}); - -const { Reranker } = await import("./reranker"); - -describe("Reranker (default Groq) ", () => { - test("uses groq-sdk when GROQ_API_KEY is set", async () => { - const reranker = new Reranker<{ key: string; summary: string }>( - { - /* no llm */ - }, - { key: (x) => x.key, summary: (x) => x.summary }, - ); - const out = await reranker.rerank("q", [ - { key: "A", summary: "" }, - { key: "B", summary: "" }, - ]); - expect(out.map((c) => c.key)).toEqual(["A"]); - expect(callMock.mock.calls.length).toBe(1); - }); -}); diff --git a/src/schema.ts b/src/schema.ts index aa22899..afd3fc7 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -1,5 +1,7 @@ import type { JSONObject } from "./types"; +type IntegerSchema = { type: "integer" }; + /** * Build a strict JSON schema mapping candidate keys to integer 0-10. * @@ -11,8 +13,10 @@ import type { JSONObject } from "./types"; * @returns JSON schema object enforcing exact structure of response */ export function buildRelevancySchema(keys: string[]): JSONObject { - const properties: Record = {}; - for (const k of keys) properties[k] = { type: "integer" }; + const properties: Record = {}; + for (const k of keys) { + properties[k] = { type: "integer" }; + } return { title: "Query / Candidate Relevancy Assessment", description: "Map candidate results for a search query to relevancy scores (0-10).", diff --git a/src/types.ts b/src/types.ts index ea86ad8..c9f56ed 100644 --- a/src/types.ts +++ b/src/types.ts @@ -31,13 +31,13 @@ export interface LoggerLike { error?(msg: string, meta?: unknown): void; } -export type RerankerCandidate = { +export type IntentCandidate = { key: string; summary: string; }; -export type RerankerExtractors = { - key: (item: T) => string; +export type IntentExtractors = { + key?: (item: T) => string; summary?: (item: T) => string; }; @@ -46,3 +46,45 @@ export type IntentContext = { logger?: LoggerLike; userId?: string; // optional per-instance user id used for provider abuse monitoring }; + +// Utility types for key-case transformations +export type CamelCase = S extends `${infer H}_${infer T}` + ? `${Lowercase}${Capitalize>}` + : S extends `${infer H}-${infer T}` + ? `${Lowercase}${Capitalize>}` + : Lowercase; + +export type CamelCasedProps = { + [K in keyof T as K extends string ? CamelCase : K]: T[K]; +}; + +/** + * Configuration options for Intent. + * + * This is a camelCase version of the INTENT config object from config.ts. + */ +export type IntentConfig = { + model?: string; + timeoutMs?: number; + relevancyThreshold?: number; + batchSize?: number; + tinyBatchFraction?: number; +}; + +/** + * Complete options object for Intent constructor. + * + * Merges all configuration into a single, fully optional object: + * - LLM client and runtime context (llm, logger, userId) + * - Item extractors (key, summary) + * - Intent configuration (model, timeoutMs, relevancyThreshold, batchSize, tinyBatchFraction) + * + * All fields are optional with sensible defaults: + * - llm: Auto-detected from GROQ_API_KEY if available + * - key: Hash-based string from JSON representation + * - summary: Pretty-printed JSON of the item (2-space indentation for LLM readability) + * - Config values: From environment variables or built-in defaults + * + * @template T - The type of items to rerank + */ +export type IntentOptions = IntentContext & IntentExtractors & IntentConfig; From 39c29ab541a2548e5391fed55a739de5a4fac4e9 Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Sun, 21 Dec 2025 20:23:42 -0800 Subject: [PATCH 02/12] feat(groq): repair invalid JSON via guided retries Problem - Groq occasionally returns output that is either invalid JSON or fails server-side json_schema validation (json_validate_failed), which currently bubbles up as a hard failure. Solution - Add configurable JSON repair retries to the Groq LlmClient. - On JSON.parse failure, capture the raw model output and append a repair turn (assistant raw output + user instruction with the parse error) and retry. - On json_validate_failed, attempt to extract the rejected generation from multiple groq-sdk error shapes (structured payloads and message-embedded payloads), then append a repair turn and retry. Configuration - Introduces GROQ_JSON_REPAIR_ATTEMPTS (default: 3, min: 0) and an override option jsonRepairAttempts on createDefaultGroqClient. Tests - Expands unit coverage to exercise parse + validation repair paths and error-shape parsing. - Adds an integration test that demonstrates repairing a schema-validation failure. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- src/config.ts | 1 + src/providers/groq.int.test.ts | 31 ++- src/providers/groq.ts | 219 ++++++++++++--- src/providers/groq.unit.test.ts | 464 +++++++++++++++++++++++++++++--- 4 files changed, 632 insertions(+), 83 deletions(-) diff --git a/src/config.ts b/src/config.ts index 06b4354..8e594e2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -9,6 +9,7 @@ export const CONFIG = { API_KEY: string("GROQ_API_KEY", { default: "" }), DEFAULT_MODEL: string("GROQ_DEFAULT_MODEL", { default: "openai/gpt-oss-20b" }), DEFAULT_TEMPERATURE: number("GROQ_DEFAULT_TEMPERATURE", { default: 0, min: 0, max: 1 }), + JSON_REPAIR_ATTEMPTS: int("GROQ_JSON_REPAIR_ATTEMPTS", { default: 3, min: 0 }), }, INTENT: { MODEL: string("INTENT_MODEL", { default: "openai/gpt-oss-20b" }), diff --git a/src/providers/groq.int.test.ts b/src/providers/groq.int.test.ts index cb20bd6..1a4b8f9 100644 --- a/src/providers/groq.int.test.ts +++ b/src/providers/groq.int.test.ts @@ -6,9 +6,36 @@ import { buildRelevancySchema } from "../schema"; import { createDefaultGroqClient } from "./groq"; -const hasKey = Boolean(CONFIG.GROQ.API_KEY); +describe("groq provider integration", () => { + test.concurrent("repairs server-side schema validation failures", async () => { + const client = createDefaultGroqClient(CONFIG.GROQ.API_KEY, { + jsonRepairAttempts: 3, + defaults: { temperature: 0 }, + }); + + const schema = { + type: "object", + properties: { + A: { type: "string", enum: ["MUST_BE_THIS_EXACT_VALUE"] }, + }, + required: ["A"], + additionalProperties: false, + } as const; + + const messages = [ + { + role: "system", + content: 'Return ONLY JSON: {"A": "WRONG_VALUE"}.', + }, + ] as any; + + const { data } = await client.call>(messages, schema as any, { + timeoutMs: 6000, + }); + + expect(data.A).toBe("MUST_BE_THIS_EXACT_VALUE"); + }); -describe.skipIf(!hasKey)("groq provider integration", () => { test.concurrent("provider returns scores for all schema keys", async () => { const client = createDefaultGroqClient(CONFIG.GROQ.API_KEY); const candidates = [ diff --git a/src/providers/groq.ts b/src/providers/groq.ts index 7a54a01..56e48ae 100644 --- a/src/providers/groq.ts +++ b/src/providers/groq.ts @@ -32,36 +32,25 @@ type GroqChatCompletionResponse = { }>; }; -type GroqApiErrorLike = { - code?: string; - error?: { - code?: string | undefined; - }; -}; - /** - * Narrow an unknown error to the Groq-shaped error payloads we care about. + * Return a best-effort nested error record from groq-sdk. * * @param err - Any thrown value - * @returns Best-effort Groq error shape for retry decisions + * @returns Nested `error` object when present * @private */ -function asGroqApiErrorLike(err: unknown): GroqApiErrorLike { +function getNestedErrorObject(err: unknown): Record | undefined { if (err == null || typeof err !== "object") { - return {}; + return undefined; } + const record = err as Record; const error = record.error; - const errorObj = - error != null && typeof error === "object" ? (error as Record) : undefined; - - const code = typeof record.code === "string" ? record.code : undefined; - const nestedCode = errorObj && typeof errorObj.code === "string" ? errorObj.code : undefined; + if (error == null || typeof error !== "object") { + return undefined; + } - return { - ...(code !== undefined && { code }), - ...(nestedCode !== undefined && { error: { code: nestedCode } }), - }; + return error as Record; } /** @@ -177,25 +166,158 @@ function parseJson(content: string): { data: T } { try { const data = JSON.parse(content); return { data } as { data: T }; - } catch { - throw new Error("Groq returned invalid JSON"); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + const err = new Error(`Groq returned invalid JSON: ${detail}`); + (err as Error & { rawOutput?: string }).rawOutput = content; + throw err; + } +} + +type JsonRepairInput = { + rawOutput: string; + errorMessage: string; +}; + +type RetryState = { + remaining: number; + request: GroqChatCompletionRequest; +}; + +/** + * Build a repair conversation turn to help the model correct invalid JSON. + * + * We include the raw output and the error message, then remind the model to + * return only valid JSON that matches the already-provided schema. + * + * @param baseMessages - Original Groq-formatted messages + * @param rawOutput - Raw model output that failed parsing + * @param errorMessage - Parse/validation error message + * @returns New messages array including a repair request + * @private + */ +function buildJsonRepairMessages( + baseMessages: ChatCompletionMessageParam[], + rawOutput: string, + errorMessage: string, +): ChatCompletionMessageParam[] { + return [ + ...baseMessages, + { role: "assistant", content: rawOutput } as ChatCompletionMessageParam, + { + role: "user", + content: + "Your previous response was invalid JSON or did not match the required JSON schema. " + + "Please correct it and return ONLY valid JSON that matches the schema.\n\n" + + `Error: ${errorMessage}`, + } as ChatCompletionMessageParam, + ]; +} + +/** + * Build a repair request object if we still have attempts remaining. + * + * @param remaining - Attempts remaining for the overall call + * @param request - The current Groq request + * @param repair - Repair context + * @returns The repaired request and decremented remaining attempts, or undefined + * @private + */ +function buildRepairRetry(state: RetryState, repair: JsonRepairInput): RetryState | undefined { + if (state.remaining <= 1) { + return undefined; + } + + const repairedRequest: GroqChatCompletionRequest = { + ...state.request, + messages: buildJsonRepairMessages( + state.request.messages, + repair.rawOutput, + repair.errorMessage, + ), + }; + + return { remaining: state.remaining - 1, request: repairedRequest }; +} + +/** + * Convert a JSON parse error into a repair input. + * + * @param rawOutput - Raw model output + * @param parseError - JSON.parse error + * @returns Repair input + * @private + */ +function parseErrorToRepairInput(rawOutput: string, parseError: unknown): JsonRepairInput { + return { rawOutput, errorMessage: String(parseError) }; +} + +/** + * Extract raw output from a JSON parse error thrown by parseJson. + * + * @param error - Error thrown by parseJson + * @returns Raw output when present + * @private + */ +function getRawOutputFromParseJsonError(error: unknown): string | undefined { + if (!(error instanceof Error)) { + return undefined; } + + const record = error as Error & { rawOutput?: unknown }; + return typeof record.rawOutput === "string" ? record.rawOutput : undefined; } /** - * Determine whether an error warrants a retry based on schema validation. + * Extract repair inputs from a Groq schema-validation failure. * - * Groq can occasionally return responses that fail JSON schema validation. - * This function checks for that specific error code and whether retries remain. + * Groq can return a rich error payload for `json_validate_failed`, sometimes + * including the model's `generated_response`. We use this to ask the model to + * correct its output without re-sending the schema. * - * @param err - Error object from failed completion - * @param remaining - Number of retry attempts remaining - * @returns True if error is retriable and retries remain + * @param err - Unknown thrown error from groq-sdk + * @returns Repair inputs when present; otherwise undefined * @private */ -function shouldRetry(err: GroqApiErrorLike, remaining: number): boolean { - const code = err.code ?? err.error?.code; - return code === "json_validate_failed" && remaining > 1; +function extractJsonValidateFailedRepairInput(err: unknown): JsonRepairInput | undefined { + if (err instanceof Error) { + const match = err.message.match(/"failed_generation":"(\{.*?\})"/); + const failedGeneration = match?.[1] ? match[1].replace(/\\"/g, '"') : undefined; + if (err.message.includes('"code":"json_validate_failed"')) { + return { + rawOutput: + failedGeneration ?? "(Groq did not include the rejected generation in the error payload)", + errorMessage: err.message, + }; + } + } + + const errorObj = getNestedErrorObject(err); + const nested = + errorObj && typeof errorObj.error === "object" + ? (errorObj.error as Record) + : undefined; + const serverError = nested ?? errorObj; + if (!serverError) { + return undefined; + } + + const code = typeof serverError.code === "string" ? serverError.code : undefined; + if (code !== "json_validate_failed") { + return undefined; + } + + const message = typeof serverError.message === "string" ? serverError.message : undefined; + const generatedResponse = + typeof serverError.generated_response === "string" ? serverError.generated_response : undefined; + const failedGeneration = + typeof serverError.failed_generation === "string" ? serverError.failed_generation : undefined; + const rawOutput = generatedResponse ?? failedGeneration; + + return { + rawOutput: rawOutput ?? "(Groq did not include the rejected generation in the error payload)", + errorMessage: message ?? String(err), + }; } /** @@ -274,6 +396,7 @@ export function createDefaultGroqClient( options?: { defaults?: { model?: string; temperature?: number }; makeSdk?: (apiKey: string) => GroqSdkLike; + jsonRepairAttempts?: number; }, ): LlmClient { const defaults = { @@ -281,6 +404,7 @@ export function createDefaultGroqClient( temperature: options?.defaults?.temperature ?? CONFIG.GROQ.DEFAULT_TEMPERATURE, } as const; const makeSdk: GroqClientFactory = options?.makeSdk ?? createGroqSdkLike; + const jsonRepairAttempts = options?.jsonRepairAttempts ?? CONFIG.GROQ.JSON_REPAIR_ATTEMPTS; return { /** * Call Groq with JSON schema enforced response and return parsed data. @@ -307,22 +431,41 @@ export function createDefaultGroqClient( ): Promise<{ data: T }> { const client = makeSdk(apiKey); const groqMessages = mapToGroqMessages(messages); - const request = buildGroqRequest(outputSchema, groqMessages, config, userId, defaults); + const baseRequest = buildGroqRequest(outputSchema, groqMessages, config, userId, defaults); - const createWithRetry = async (remaining: number): Promise<{ data: T }> => { + const createWithRetry = async (state: RetryState): Promise<{ data: T }> => { try { - const response = await executeCompletion(client, request, config?.timeoutMs); + const response = await executeCompletion(client, state.request, config?.timeoutMs); const content = getResponseContent(response); - return parseJson(content); + + const parsed = parseJson(content); + return parsed; } catch (err) { - if (shouldRetry(asGroqApiErrorLike(err), remaining)) { - return createWithRetry(remaining - 1); + // If the model returned bad JSON, retry by appending a repair turn. + const parseRawOutput = getRawOutputFromParseJsonError(err); + if (parseRawOutput !== undefined) { + const retry = buildRepairRetry(state, parseErrorToRepairInput(parseRawOutput, err)); + if (retry) { + return createWithRetry(retry); + } + throw err; } + + const validationRepairInput = extractJsonValidateFailedRepairInput(err); + if (validationRepairInput) { + const retry = buildRepairRetry(state, validationRepairInput); + if (retry) { + return createWithRetry(retry); + } + } + + // Non-JSON errors are terminal (quota/outage/etc.). throw err; } }; - return createWithRetry(3); + const attempts = Math.max(1, jsonRepairAttempts); + return createWithRetry({ remaining: attempts, request: baseRequest }); }, } satisfies LlmClient; } diff --git a/src/providers/groq.unit.test.ts b/src/providers/groq.unit.test.ts index 6283479..ef30184 100644 --- a/src/providers/groq.unit.test.ts +++ b/src/providers/groq.unit.test.ts @@ -2,6 +2,29 @@ import { describe, expect, test, vi } from "vitest"; import * as GroqProvider from "./groq"; +vi.mock("groq-sdk", () => { + return { + default: class GroqMock { + chat = { + completions: { + create: vi.fn(async () => { + return { + choices: [ + { + message: { + role: "assistant", + content: JSON.stringify({ A: 1 }), + }, + }, + ], + }; + }), + }, + }; + }, + }; +}); + const schema = { type: "object", properties: { A: { type: "integer" } }, @@ -10,36 +33,22 @@ const schema = { } as const; describe("groq provider", () => { - test("asGroqApiErrorLike handles error shapes used for retries", async () => { - const callMock = vi - .fn() - .mockRejectedValueOnce(Object.assign(new Error("schema"), { code: "json_validate_failed" })) - .mockResolvedValueOnce({ - choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], - }); - const client = GroqProvider.createDefaultGroqClient("k", { - makeSdk: () => ({ chat: { completions: { create: callMock } } }), - }); + test("createGroqSdkLike exists", () => { + expect(typeof GroqProvider.createGroqSdkLike).toBe("function"); + }); - const res = await client.call([{ role: "user", content: "{}" }], schema as any); - expect(res.data).toEqual({ A: 1 }); - expect(callMock.mock.calls.length).toBe(2); + test("createGroqSdkLike wrapper forwards to groq-sdk create", async () => { + const sdk = GroqProvider.createGroqSdkLike("k"); + const res = await sdk.chat.completions.create({} as any); + expect(res.choices[0].message.content).toBe(JSON.stringify({ A: 1 })); }); - test("null error is not retried", async () => { - const callMock = vi.fn().mockRejectedValueOnce(null); + test("defaults makeSdk to createGroqSdkLike when omitted", async () => { const client = GroqProvider.createDefaultGroqClient("k", { - makeSdk: () => ({ chat: { completions: { create: callMock } } }), + jsonRepairAttempts: 1, }); - - await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toBeNull(); - expect(callMock.mock.calls.length).toBe(1); - }); - - test("createGroqSdkLike wrapper forwards to groq-sdk create", async () => { - // The default SDK factory uses groq-sdk directly; unit tests should inject makeSdk - // to avoid network calls. - expect(typeof GroqProvider.createGroqSdkLike).toBe("function"); + const res = await client.call([{ role: "user", content: "{}" }], schema as any); + expect(res.data).toEqual({ A: 1 }); }); test("createGroqSdk creates an SDK client", () => { @@ -47,8 +56,6 @@ describe("groq provider", () => { expect(typeof sdk).toBe("object"); }); - // (covered by createGroqSdkLike wrapper test above) - test("defaults come from options.defaults when provided", async () => { const callMock = vi.fn(async (req: any) => { expect(req.model).toBe("m1"); @@ -65,6 +72,20 @@ describe("groq provider", () => { expect(callMock.mock.calls.length).toBe(1); }); + test("buildGroqRequest includes user when provided", async () => { + const callMock = vi.fn(async (req: any) => { + expect(req.user).toBe("u1"); + return { + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], + }; + }); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + await client.call([{ role: "user", content: "{}" }], schema as any, {}, "u1"); + expect(callMock.mock.calls.length).toBe(1); + }); + test("defaults fall back when options.defaults omitted", async () => { const callMock = vi.fn(async (req: any) => { expect(typeof req.model).toBe("string"); @@ -117,16 +138,206 @@ describe("groq provider", () => { const client = GroqProvider.createDefaultGroqClient("k", { makeSdk: () => ({ chat: { completions: { create: callMock } } }), }); + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( + /invalid JSON/, + ); + expect(callMock.mock.calls.length).toBe(3); + }); + + test("invalid JSON triggers repair messages with error details", async () => { + const callMock = vi + .fn() + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: "not-json" } }], + }) + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 2 }) } }], + }); + + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + const res = await client.call([{ role: "user", content: "{}" }], schema as any); + expect(res.data).toEqual({ A: 2 }); + expect(callMock.mock.calls.length).toBe(2); + + const [firstReq] = callMock.mock.calls[0]; + const [secondReq] = callMock.mock.calls[1]; + expect(firstReq.messages.length).toBe(1); + expect(secondReq.messages.length).toBe(3); + expect(secondReq.messages[1].role).toBe("assistant"); + expect(secondReq.messages[1].content).toBe("not-json"); + expect(secondReq.messages[2].role).toBe("user"); + expect(String(secondReq.messages[2].content)).toMatch(/invalid JSON/); + }); + + test("parseJson includes non-Error JSON.parse details", async () => { + const originalParse = JSON.parse; + JSON.parse = (() => { + throw "bad-json"; + }) as any; + + try { + const callMock = vi.fn(async (_req: any) => ({ + choices: [{ message: { role: "assistant", content: "not-json" } }], + })); + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 0, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( + /bad-json/, + ); + } finally { + JSON.parse = originalParse; + } + }); + + test("repair uses parseError message when parseError is an Error", async () => { + const callMock = vi + .fn() + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: "not-json" } }], + }) + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 2 }) } }], + }); + + const originalParse = JSON.parse; + JSON.parse = (() => { + throw new Error("bad-json-error"); + }) as any; + + try { + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( + /bad-json-error/, + ); + } finally { + JSON.parse = originalParse; + } + }); + + test("repair appends parseError message when parseError is an Error", async () => { + const callMock = vi + .fn() + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: "not-json" } }], + }) + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 2 }) } }], + }); + + const originalParse = JSON.parse; + JSON.parse = (() => { + throw new Error("bad-json-error"); + }) as any; + + try { + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( + /bad-json-error/, + ); + + const [secondReq] = callMock.mock.calls[1]; + expect(String(secondReq.messages[2].content)).toMatch(/bad-json-error/); + } finally { + JSON.parse = originalParse; + } + }); + + test("repair uses non-Error parseError stringification", async () => { + const originalParse = JSON.parse; + JSON.parse = (() => { + throw "bad-json"; + }) as any; + + try { + const callMock = vi + .fn() + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: "not-json" } }], + }) + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 2 }) } }], + }); + + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( + /bad-json/, + ); + } finally { + JSON.parse = originalParse; + } + }); + + test("repair is skipped when jsonRepairAttempts is 0", async () => { + const callMock = vi.fn(async (_req: any) => ({ + choices: [{ message: { role: "assistant", content: "not-json" } }], + })); + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 0, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( /invalid JSON/, ); expect(callMock.mock.calls.length).toBe(1); }); + test("repair includes JSON.parse error details", async () => { + const callMock = vi + .fn() + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: "not-json" } }], + }) + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 2 }) } }], + }); + + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + const res = await client.call([{ role: "user", content: "{}" }], schema as any); + expect(res.data).toEqual({ A: 2 }); + const [secondReq] = callMock.mock.calls[1]; + expect(String(secondReq.messages[2].content)).toMatch(/invalid JSON/); + }); + test("retries on schema validation failure", async () => { const errors = [ - Object.assign(new Error("json schema fail 1"), { error: { code: "json_validate_failed" } }), - Object.assign(new Error("json schema fail 2"), { code: "json_validate_failed" }), + Object.assign(new Error("json schema fail 1"), { + error: { + error: { + code: "json_validate_failed", + generated_response: '{"A": "wrong"}', + message: "details", + }, + }, + }), + Object.assign(new Error("json schema fail 2"), { + error: { + error: { code: "json_validate_failed", failed_generation: '{"A": "wrong2"}' }, + }, + }), ]; const callMock = vi .fn() @@ -141,11 +352,19 @@ describe("groq provider", () => { const result = await client.call([{ role: "user", content: "{}" }], schema as any); expect(result.data).toEqual({ A: 3 }); expect(callMock.mock.calls.length).toBe(3); + + const [secondReq] = callMock.mock.calls[1]; + expect(secondReq.messages[1].role).toBe("assistant"); + expect(secondReq.messages[1].content).toBe('{"A": "wrong"}'); + + const [thirdReq] = callMock.mock.calls[2]; + expect(thirdReq.messages[1].role).toBe("assistant"); + expect(thirdReq.messages[1].content).toBe('{"A": "wrong"}'); }); test("fails after max retries on schema validation failure", async () => { const errorObj = Object.assign(new Error("schema fail"), { - error: { code: "json_validate_failed" }, + error: { error: { code: "json_validate_failed" } }, }); const callMock = vi .fn() @@ -161,6 +380,177 @@ describe("groq provider", () => { expect(callMock.mock.calls.length).toBe(3); }); + test("retries on json_validate_failed when error payload exists only in Error.message", async () => { + const callMock = vi + .fn() + .mockRejectedValueOnce( + new Error( + '400 {"error":{"message":"Generated JSON does not match the expected schema. Please adjust your prompt. See \'failed_generation\' for more details. Error: jsonschema: \'/A\' does not validate with /properties/A/const: value must be \\"MUST_BE_THIS_EXACT_VALUE\\"","type":"invalid_request_error","code":"json_validate_failed","failed_generation":"{\\"A\\":\\"WRONG_VALUE\\"}"}}', + ), + ) + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], + }); + + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + const res = await client.call([{ role: "user", content: "{}" }], schema as any); + expect(res.data).toEqual({ A: 1 }); + expect(callMock.mock.calls.length).toBe(2); + + const [secondReq] = callMock.mock.calls[1]; + expect(secondReq.messages.length).toBe(3); + expect(secondReq.messages[1].role).toBe("assistant"); + expect(String(secondReq.messages[1].content)).toContain("WRONG_VALUE"); + expect(secondReq.messages[2].role).toBe("user"); + expect(String(secondReq.messages[2].content)).toMatch(/json_validate_failed/); + }); + + test("extractJsonValidateFailedRepairInput falls back to structured error when message is missing", async () => { + const callMock = vi + .fn() + .mockRejectedValueOnce( + Object.assign(new Error("validation"), { + message: "", + error: { + error: { + code: "json_validate_failed", + failed_generation: '{"A":"WRONG_VALUE"}', + message: "details", + }, + }, + }), + ) + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], + }); + + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + const res = await client.call([{ role: "user", content: "{}" }], schema as any); + expect(res.data).toEqual({ A: 1 }); + expect(callMock.mock.calls.length).toBe(2); + }); + + test("extractJsonValidateFailedRepairInput returns undefined for non-object error values", async () => { + const callMock = vi + .fn() + .mockRejectedValueOnce("not-an-error") + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], + }); + + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toBe( + "not-an-error", + ); + expect(callMock.mock.calls.length).toBe(1); + }); + + test("extractJsonValidateFailedRepairInput returns undefined when error.error is non-object", async () => { + const callMock = vi.fn().mockRejectedValueOnce( + Object.assign(new Error("validation"), { + error: "not-an-object", + }), + ); + + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( + /validation/, + ); + expect(callMock.mock.calls.length).toBe(1); + }); + + test("extractJsonValidateFailedRepairInput can run with missing failed_generation in message", async () => { + const callMock = vi + .fn() + .mockRejectedValueOnce(new Error('400 {"error":{"code":"json_validate_failed"}}')) + .mockResolvedValueOnce({ + choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], + }); + + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + await client.call([{ role: "user", content: "{}" }], schema as any); + expect(callMock.mock.calls.length).toBe(2); + }); + + test("extractJsonValidateFailedRepairInput returns undefined when error is nested object but missing code", async () => { + const callMock = vi.fn().mockRejectedValueOnce( + Object.assign(new Error("no-code"), { + error: { error: { message: "details" } }, + }), + ); + + const client = GroqProvider.createDefaultGroqClient("k", { + jsonRepairAttempts: 2, + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( + /no-code/, + ); + expect(callMock.mock.calls.length).toBe(1); + }); + + test("getNestedErrorObject returns undefined for null", async () => { + const callMock = vi.fn().mockRejectedValueOnce(null); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + jsonRepairAttempts: 2, + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toBeNull(); + expect(callMock.mock.calls.length).toBe(1); + }); + + test("getNestedErrorObject returns undefined for object with no error property", async () => { + const callMock = vi.fn().mockRejectedValueOnce({ foo: "bar" }); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + jsonRepairAttempts: 2, + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toEqual({ + foo: "bar", + }); + expect(callMock.mock.calls.length).toBe(1); + }); + + test("getNestedErrorObject returns undefined for non-object error field", async () => { + const callMock = vi.fn().mockRejectedValueOnce( + Object.assign(new Error("boom"), { + error: "nope", + }), + ); + const client = GroqProvider.createDefaultGroqClient("k", { + makeSdk: () => ({ chat: { completions: { create: callMock } } }), + jsonRepairAttempts: 2, + }); + + await expect(client.call([{ role: "user", content: "{}" }], schema as any)).rejects.toThrow( + /boom/, + ); + expect(callMock.mock.calls.length).toBe(1); + }); + test("timeout forwarded to groq-sdk", async () => { const callMock = vi.fn(async (_req: any, opts?: any) => { expect(opts?.timeout).toBe(1); @@ -201,19 +591,7 @@ describe("groq provider", () => { ).rejects.toThrow(/not supported/); }); - test("forwards userId to request", async () => { - const callMock = vi.fn(async (req: any) => { - expect(req.user).toBe("u1"); - return { - choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], - }; - }); - const client = GroqProvider.createDefaultGroqClient("k", { - makeSdk: () => ({ chat: { completions: { create: callMock } } }), - }); - await client.call([{ role: "user", content: "{}" }], schema as any, {}, "u1"); - expect(callMock.mock.calls.length).toBe(1); - }); + // (userId forwarding covered above) test("accepts assistant role messages", async () => { const callMock = vi.fn(async (_req: any) => ({ From 60f258cff5bbbc7339e42a291d5a7bff8f953f88 Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Sun, 21 Dec 2025 22:00:29 -0800 Subject: [PATCH 03/12] feat(intent): add optional explain output with per-item reasoning - Change LLM response contract from Record to Record and update prompt/schema accordingly. - Add rank(..., { explain: true }) overload returning { item, explanation }[]; default remains T[]. - Preserve existing safety behavior: stable sorting on ties; batch-level and top-level fallbacks return original order (with blank explanations when explain is enabled). - Add unit + Groq integration test coverage, including schema property ordering (explanation before score) to encourage models to emit explanations first. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- src/intent.groq-default.unit.test.ts | 12 +- src/intent.ts | 104 ++++++++++---- src/intent.unit.test.ts | 205 +++++++++++++++++++++++++-- src/messages.ts | 14 +- src/providers/groq.int.test.ts | 33 +++-- src/schema.ts | 52 ++++++- 6 files changed, 357 insertions(+), 63 deletions(-) diff --git a/src/intent.groq-default.unit.test.ts b/src/intent.groq-default.unit.test.ts index 1f2c808..72c0c8c 100644 --- a/src/intent.groq-default.unit.test.ts +++ b/src/intent.groq-default.unit.test.ts @@ -6,7 +6,17 @@ import { createDefaultGroqClient } from "./providers/groq"; describe("Intent (default Groq) ", () => { test("uses groq client (via DI)", async () => { const callMock = vi.fn(async (_req: any) => ({ - choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 10, B: 0 }) } }], + choices: [ + { + message: { + role: "assistant", + content: JSON.stringify({ + A: { explanation: "a", score: 10 }, + B: { explanation: "b", score: 0 }, + }), + }, + }, + ], })); const llm = createDefaultGroqClient("test-key", { makeSdk: () => ({ chat: { completions: { create: callMock } } }), diff --git a/src/intent.ts b/src/intent.ts index 8e49b8a..4f84110 100644 --- a/src/intent.ts +++ b/src/intent.ts @@ -204,9 +204,11 @@ export class Intent { /** * Rerank candidates based on relevance to a query. * - * Calls the LLM to score each candidate 0-10 based on relevance to the query, - * filters results by the configured threshold, and returns items sorted by score - * (highest first) with ties preserving original input order. + * Calls the LLM to evaluate each candidate and returns only those above the + * configured threshold, sorted by score (desc) with stable ordering on ties. + * + * The LLM is instructed to always generate an explanation before the score. + * Explanations are only returned when `options.explain` is true. * * Fast-path optimizations: * - Returns empty array for 0 candidates without LLM call @@ -220,37 +222,77 @@ export class Intent { * @param query - The search query or user intent to rank against * @param candidates - Array of items to rerank * @param options - Optional per-call configuration + * @param options.explain - When true, return `{ item, explanation }[]` instead of `T[]` * @param options.userId - Optional user ID for this specific call, overrides ctx.userId * @returns Filtered and sorted array of items, or original order on any error * * @example * ```typescript - * const results = await intent.rank( - * "quarterly expense reports from 2024", - * allDocuments, - * { userId: "session-abc" } - * ); - * // Returns only documents with score > threshold, sorted by relevance + * const itemsOnly = await intent.rank("find expense reports", docs); + * + * const withExplanations = await intent.rank("find expense reports", docs, { explain: true }); + * // => [{ item: Doc, explanation: string }, ...] * ``` */ - public async rank(query: string, candidates: T[], options?: { userId?: string }): Promise { + public async rank( + query: string, + candidates: T[], + options?: { explain?: false; userId?: string }, + ): Promise; + + public async rank( + query: string, + candidates: T[], + options: { explain: true; userId?: string }, + ): Promise>; + + public async rank( + query: string, + candidates: T[], + options?: { explain?: boolean; userId?: string }, + ): Promise> { try { - if (candidates.length === 0) return []; - if (candidates.length === 1) return candidates; + if (candidates.length === 0) { + return []; + } + + if (candidates.length === 1) { + if (options?.explain) { + const [firstCandidate] = candidates; + return [{ item: firstCandidate!, explanation: "" }]; + } + return candidates; + } const prepared = this.prepareCandidates(candidates); - return await batchProcess( + + const rankedWithExplanations = await batchProcess( prepared, this.cfg.batchSize, this.cfg.tinyBatchFraction, - (batch) => this.processBatch(query, batch, options?.userId), + async (batch) => + (await this.processBatch( + query, + batch, + options?.userId !== undefined ? { userId: options.userId } : undefined, + )) as Array<{ item: T; explanation: string }>, this.ctx.logger, - (batch) => batch.map(({ item }) => item), + (batch) => batch.map(({ item }) => ({ item, explanation: "" })), ); + + if (options?.explain) { + return rankedWithExplanations; + } + + return rankedWithExplanations.map(({ item }) => item); } catch (error) { this.ctx.logger?.warn?.("intent reranker failed, using fallback", { error: (error as Error)?.message, }); + + if (options?.explain) { + return candidates.map((item) => ({ item, explanation: "" })); + } return candidates; } } @@ -335,17 +377,17 @@ export class Intent { * @returns Map of candidate keys to numeric scores, or null if response invalid * @private */ - private async fetchScores( + private async fetchEvaluations( messages: ChatMessage[], schema: JSONObject, userId?: string, - ): Promise | null> { + ): Promise | null> { const config: LlmCallConfig = { model: this.cfg.model, temperature: 0, timeoutMs: this.cfg.timeoutMs, }; - const { data } = await this.llm.call>( + const { data } = await this.llm.call>( messages, schema, config, @@ -353,7 +395,7 @@ export class Intent { ); if (data == null || typeof data !== "object") return null; - return data as Record; + return data as Record; } /** @@ -370,13 +412,15 @@ export class Intent { */ private rankAndFilter( items: Array<{ item: T; idx: number; key: string; summary: string }>, - scores: Record, - ): T[] { + evaluations: Record, + ): Array<{ item: T; explanation: string; score: number }> { const threshold = this.cfg.relevancyThreshold; const scored = items.map(({ item, idx, key }) => ({ item, idx, - score: clamp(scores[key] ?? 0, 0, 10), + explanation: + typeof evaluations[key]?.explanation === "string" ? evaluations[key].explanation : "", + score: clamp(evaluations[key]?.score ?? 0, 0, 10), })); const filtered = scored.filter(({ score }) => score > threshold); @@ -384,7 +428,7 @@ export class Intent { if (b.score !== a.score) return b.score - a.score; return a.idx - b.idx; }); - return sorted.map(({ item }) => item); + return sorted; } /** @@ -403,12 +447,16 @@ export class Intent { private async processBatch( query: string, batch: Array<{ item: T; idx: number; baseKey: string; summary: string }>, - userId?: string, - ): Promise { + options?: { userId?: string }, + ): Promise> { const keyed = this.ensureUniqueKeys(batch); const { schema, messages } = this.buildRequest(query, keyed); - const scores = await this.fetchScores(messages, schema, userId); - if (scores == null) return keyed.map(({ item }) => item); - return this.rankAndFilter(keyed, scores); + const evaluations = await this.fetchEvaluations(messages, schema, options?.userId); + if (evaluations == null) { + return keyed.map(({ item }) => ({ item, explanation: "" })); + } + + const ranked = this.rankAndFilter(keyed, evaluations); + return ranked.map(({ item, explanation }) => ({ item, explanation })); } } diff --git a/src/intent.unit.test.ts b/src/intent.unit.test.ts index b1d9d6a..18ce38a 100644 --- a/src/intent.unit.test.ts +++ b/src/intent.unit.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test, vi } from "vitest"; import { CONFIG } from "./config"; import { Intent } from "./intent"; +import { buildCandidateEvaluationSchema } from "./schema"; import type { LlmClient, LoggerLike, IntentCandidate, IntentContext } from "./types"; @@ -26,6 +27,11 @@ function makeCtx(overrides: Partial = {}): IntentContext & { } describe("Intent.rank", () => { + test("candidate evaluation schema defines explanation before score", () => { + const schema = buildCandidateEvaluationSchema(); + expect(Object.keys(schema.properties)).toEqual(["explanation", "score"]); + }); + test("throws when no llm and no GROQ_API_KEY", async () => { const configOverride = { ...CONFIG, @@ -74,10 +80,69 @@ describe("Intent.rank", () => { expect(ctx.llm.call).not.toHaveBeenCalled(); }); + test("returns explanation wrapper for single candidate when explain is true (no LLM call)", async () => { + const ctx = makeCtx(); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const input = [{ key: "Only", summary: "s" }]; + const res = await intent.rank("query", input, { explain: true }); + expect(res).toEqual([{ item: input[0], explanation: "" }]); + expect(ctx.llm.call).not.toHaveBeenCalled(); + }); + + test("returns explanations when explain option is true", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { + A: { explanation: "matches query", score: 10 }, + B: { explanation: "irrelevant", score: 0 }, + }, + }); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.rank("query", input, { explain: true }); + expect(res).toEqual([{ item: input[0], explanation: "matches query" }]); + }); + + test("explain option is false by default (returns T[])", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { + A: { explanation: "matches query", score: 10 }, + B: { explanation: "irrelevant", score: 0 }, + }, + }); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.rank("query", input); + expect(res).toEqual([input[0]]); + }); + test("rounds scores, filters zeros, orders by score", async () => { const ctx = makeCtx(); (ctx.llm.call as any).mockResolvedValueOnce({ - data: { A: 10, B: 6.8, C: 0 }, + data: { + A: { explanation: "a", score: 10 }, + B: { explanation: "b", score: 6.8 }, + C: { explanation: "c", score: 0 }, + }, }); const intent = new Intent({ ...ctx, @@ -93,7 +158,9 @@ describe("Intent.rank", () => { test("handles non-numeric or missing scores by clamping to 0", async () => { const ctx = makeCtx(); - (ctx.llm.call as any).mockResolvedValueOnce({ data: { X: "nope" as any } }); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { X: { explanation: "x", score: "nope" as any } }, + }); const intent = new Intent({ ...ctx, key: (c) => c.key, @@ -107,7 +174,12 @@ describe("Intent.rank", () => { test("normalizes out-of-range and infinite values", async () => { const ctx = makeCtx(); (ctx.llm.call as any).mockResolvedValueOnce({ - data: { A: -3, B: 11, C: 9.6, D: Number.POSITIVE_INFINITY }, + data: { + A: { explanation: "a", score: -3 }, + B: { explanation: "b", score: 11 }, + C: { explanation: "c", score: 9.6 }, + D: { explanation: "d", score: Number.POSITIVE_INFINITY }, + }, }); const intent = new Intent({ ...ctx, @@ -164,7 +236,12 @@ describe("Intent.rank", () => { test("uses empty summary when extractor is missing", async () => { const ctx = makeCtx(); // Return some scores so rank proceeds - (ctx.llm.call as any).mockResolvedValueOnce({ data: { A: 1, B: 0 } }); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { + A: { explanation: "a", score: 1 }, + B: { explanation: "b", score: 0 }, + }, + }); const intent = new Intent({ ...ctx, key: (c) => c.key, @@ -207,9 +284,40 @@ describe("Intent.rank", () => { expect(ctx.logger.warn).toHaveBeenCalled(); }); + test("top-level rank catch: logs and returns wrapped input when explain is true", async () => { + const ctx = makeCtx(); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + // @ts-ignore override private method to throw to trigger top-level catch + intent.prepareCandidates = () => { + throw new Error("oops"); + }; + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + { key: "C", summary: "" }, + ]; + + const res = await intent.rank("query", input, { explain: true }); + expect(res).toEqual([ + { item: input[0], explanation: "" }, + { item: input[1], explanation: "" }, + { item: input[2], explanation: "" }, + ]); + expect(ctx.logger.warn).toHaveBeenCalled(); + }); + test("passes userId from ctx and allows method override", async () => { const ctx = makeCtx({ userId: "ctx-user" }); - (ctx.llm.call as any).mockResolvedValueOnce({ data: { A: 5, B: 5 } }); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { + A: { explanation: "a", score: 5 }, + B: { explanation: "b", score: 5 }, + }, + }); const intent = new Intent({ ...ctx, key: (c) => c.key, @@ -226,7 +334,12 @@ describe("Intent.rank", () => { test("timeout config is forwarded to client", async () => { const ctx = makeCtx(); - (ctx.llm.call as any).mockResolvedValueOnce({ data: { A: 5, B: 5 } }); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { + A: { explanation: "a", score: 5 }, + B: { explanation: "b", score: 5 }, + }, + }); const intent = new Intent({ ...ctx, key: (c) => c.key, @@ -245,8 +358,18 @@ describe("Intent.rank", () => { test("splits long lists into batches and combines results", async () => { const ctx = makeCtx(); (ctx.llm.call as any) - .mockResolvedValueOnce({ data: { K2: 10, K0: 7 } }) - .mockResolvedValueOnce({ data: { K7: 10, K6: 9 } }); + .mockResolvedValueOnce({ + data: { + K2: { explanation: "k2", score: 10 }, + K0: { explanation: "k0", score: 7 }, + }, + }) + .mockResolvedValueOnce({ + data: { + K7: { explanation: "k7", score: 10 }, + K6: { explanation: "k6", score: 9 }, + }, + }); const intent = new Intent({ ...ctx, @@ -309,7 +432,12 @@ describe("Intent.rank", () => { test("one batch fails while others succeed (partial fallback)", async () => { const ctx = makeCtx(); (ctx.llm.call as any) - .mockResolvedValueOnce({ data: { K1: 10, K0: 9 } }) + .mockResolvedValueOnce({ + data: { + K1: { explanation: "k1", score: 10 }, + K0: { explanation: "k0", score: 9 }, + }, + }) .mockRejectedValueOnce(new Error("boom")); const intent = new Intent({ @@ -329,6 +457,40 @@ describe("Intent.rank", () => { expect((ctx.llm.call as any).mock.calls.length).toBe(2); }); + test("one batch fails while others succeed when explain is true (partial fallback)", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any) + .mockResolvedValueOnce({ + data: { + K1: { explanation: "k1", score: 10 }, + K0: { explanation: "k0", score: 9 }, + }, + }) + .mockRejectedValueOnce(new Error("boom")); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 3, + }); + + const input: IntentCandidate[] = Array.from({ length: 6 }).map((_, i) => ({ + key: `K${i}`, + summary: `S${i}`, + })); + + const out = await intent.rank("query", input, { explain: true }); + expect(out).toEqual([ + { item: input[1], explanation: "k1" }, + { item: input[0], explanation: "k0" }, + { item: input[3], explanation: "" }, + { item: input[4], explanation: "" }, + { item: input[5], explanation: "" }, + ]); + expect((ctx.llm.call as any).mock.calls.length).toBe(2); + }); + test("returns original list when scores payload is null", async () => { const ctx = makeCtx(); (ctx.llm.call as any).mockResolvedValueOnce({ data: null }); @@ -346,10 +508,33 @@ describe("Intent.rank", () => { expect(out).toEqual(input); }); + test("returns blank explanations on null LLM payload when explain is true", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ data: null }); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const input: IntentCandidate[] = [ + { key: "A", summary: "S" }, + { key: "B", summary: "S" }, + ]; + + const out = await intent.rank("query", input, { explain: true }); + expect(out).toEqual([ + { item: input[0], explanation: "" }, + { item: input[1], explanation: "" }, + ]); + }); + test("stable order for ties within a batch (duplicate keys)", async () => { const ctx = makeCtx(); (ctx.llm.call as any).mockResolvedValueOnce({ - data: { Same: 5, "Same (1)": 5 }, + data: { + Same: { explanation: "s0", score: 5 }, + "Same (1)": { explanation: "s1", score: 5 }, + }, }); const intent = new Intent({ diff --git a/src/messages.ts b/src/messages.ts index 8c2df3e..35b1782 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -17,19 +17,23 @@ import type { ChatMessage, IntentCandidate } from "./types"; * @returns Array of chat messages ready for LLM consumption */ export function buildMessages(query: string, candidates: IntentCandidate[]): ChatMessage[] { - const system = `The user will provide a short description of a query they are trying to automate, along with a JSON blob containing candidate_search_results. Each candidate result has a uniquely identifying key and a short summary. Your task is to assess each candidate and return a JSON object that maps candidate keys to integers from 0 to 10: 0 means not relevant at all, and 10 means highly relevant. Sometimes none are relevant, sometimes all are relevant. Be aggressive and decisive on relevancy. + const system = `The user will provide a short description of a query they are trying to automate, along with a JSON blob containing candidate_search_results. Each candidate result has a uniquely identifying key and a short summary. Your task is to assess each candidate and return a JSON object that maps candidate keys to objects of the form {"explanation": string, "score": integer} where score is from 0 to 10: 0 means not relevant at all, and 10 means highly relevant. Sometimes none are relevant, sometimes all are relevant. Be aggressive and decisive on relevancy. It is okay to return 0 if the candidate is not relevant to the query. It is okay to return 10 if the candidate is highly relevant to the query. Use the full range of scores. +Every candidate MUST include an explanation. Write the explanation first, then the score. The explanation should be concise (1-3 sentences), concrete, and reference the query intent and the candidate summary. + Every key in candidate_search_results must be present in your output mapping. Do not add any keys that are not present in candidate_search_results. -Every key in candidate_search_results must map to an integer from 0 to 10. -Do not, in your generated JSON, include anything other than the \`"{key}": {score}\` mappings. Do not include any other text, formatting, context, explanation, or punctuation. Only provide your score. +Every key in candidate_search_results must map to an object with: +- explanation: string +- score: integer from 0 to 10 +Do not, in your generated JSON, include anything other than the \`"{key}": {"explanation": "...", "score": 7}\` mappings. Do not include any other text outside the JSON. Return a JSON object that matches the enforced JSON schema for response formatting. Use the candidate.key as the property name in the output mapping. The JSON you return should be of the form: { - "Key for document 1": 0, - "Key for document 2": 7, + "Key for document 1": { "explanation": "...", "score": 0 }, + "Key for document 2": { "explanation": "...", "score": 7 }, ... } diff --git a/src/providers/groq.int.test.ts b/src/providers/groq.int.test.ts index 1a4b8f9..37f9de4 100644 --- a/src/providers/groq.int.test.ts +++ b/src/providers/groq.int.test.ts @@ -44,14 +44,19 @@ describe("groq provider integration", () => { ]; const schema = buildRelevancySchema(candidates.map((c) => c.key)); const messages = buildMessages("choose best", candidates); - const { data } = await client.call>(messages, schema, { - timeoutMs: 5000, - }); + const { data } = await client.call>( + messages, + schema, + { + timeoutMs: 5000, + }, + ); expect(Object.keys(data)).toEqual(["A", "B"]); for (const k of Object.keys(data)) { - expect(typeof data[k]).toBe("number"); - expect(data[k]).toBeGreaterThanOrEqual(0); - expect(data[k]).toBeLessThanOrEqual(10); + expect(typeof data[k]?.explanation).toBe("string"); + expect(typeof data[k]?.score).toBe("number"); + expect(data[k]?.score).toBeGreaterThanOrEqual(0); + expect(data[k]?.score).toBeLessThanOrEqual(10); } }); @@ -64,13 +69,17 @@ describe("groq provider integration", () => { ]; const schema = buildRelevancySchema(candidates.map((c) => c.key)); const messages = buildMessages("Help me with JavaScript array sorting", candidates); - const { data } = await client.call>(messages, schema, { - timeoutMs: 6000, - }); + const { data } = await client.call>( + messages, + schema, + { + timeoutMs: 6000, + }, + ); // Related candidate should be > 0 - expect(data["JS Arrays"]).toBeGreaterThan(0); + expect(data["JS Arrays"]?.score).toBeGreaterThan(0); // Unrelated candidates should be 0 - expect(data["Banana Bread Recipe"]).toBe(0); - expect(data["Eiffel Tower History"]).toBe(0); + expect(data["Banana Bread Recipe"]?.score).toBe(0); + expect(data["Eiffel Tower History"]?.score).toBe(0); }); }); diff --git a/src/schema.ts b/src/schema.ts index afd3fc7..1328eef 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -1,25 +1,63 @@ import type { JSONObject } from "./types"; type IntegerSchema = { type: "integer" }; +type StringSchema = { type: "string" }; + +type CandidateEvaluationSchema = { + type: "object"; + properties: { + explanation: StringSchema; + score: IntegerSchema; + }; + required: ["explanation", "score"]; + additionalProperties: false; +}; /** - * Build a strict JSON schema mapping candidate keys to integer 0-10. + * Build the schema used for a single candidate's evaluation. * - * Creates a JSON schema object with one required integer property per candidate key. - * Sets additionalProperties to false to prevent the LLM from adding extra keys. - * Used with LLM providers that support structured output (e.g., Groq's json_schema mode). + * Property order is intentional: `explanation` is defined before `score` to + * encourage structured-output models to generate explanations first. + * + * @returns JSON schema for a single candidate evaluation + */ +export function buildCandidateEvaluationSchema(): CandidateEvaluationSchema { + return { + type: "object", + properties: { + explanation: { type: "string" }, + score: { type: "integer" }, + }, + required: ["explanation", "score"], + additionalProperties: false, + }; +} + +/** + * Build a strict JSON schema mapping candidate keys to evaluation objects. + * + * Each candidate key maps to an object containing: + * - explanation: a short justification of the score + * - score: an integer 0-10 + * + * Property order is intentional (`explanation` then `score`) to encourage the + * model to generate explanations before scores in structured-output modes. * * @param keys - Array of unique candidate keys * @returns JSON schema object enforcing exact structure of response */ export function buildRelevancySchema(keys: string[]): JSONObject { - const properties: Record = {}; + const evaluationSchema = buildCandidateEvaluationSchema(); + + const properties: Record = {}; for (const k of keys) { - properties[k] = { type: "integer" }; + properties[k] = evaluationSchema; } + return { title: "Query / Candidate Relevancy Assessment", - description: "Map candidate results for a search query to relevancy scores (0-10).", + description: + "Map candidate results for a search query to relevancy scores (0-10) with explanations.", type: "object", properties, required: keys, From ee2fdfed2f075b2986c2d736b550dba99f743a5c Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Sun, 21 Dec 2025 22:19:16 -0800 Subject: [PATCH 04/12] feat(intent): configurable score range and relax schema constraints - Add min/max score configuration (options + INTENT_MIN_SCORE/INTENT_MAX_SCORE) and validate threshold against the configured range. - Update prompts and schema generation to use the configured range; clamp returned scores to the range. - Remove JSON Schema minimum/maximum constraints on score for broader structured-output compatibility. - Stabilize Groq integration by increasing LLM call timeout and test timeout for the relevance sanity check. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- src/config.ts | 4 ++- src/intent.int.test.ts | 5 +-- src/intent.ts | 46 ++++++++++++++++++------- src/intent.unit.test.ts | 45 ++++++++++++++++++++++-- src/messages.ts | 18 ++++++---- src/providers/groq.int.test.ts | 63 +++++++++++++++++++++------------- src/schema.ts | 11 +++--- src/types.ts | 2 ++ 8 files changed, 139 insertions(+), 55 deletions(-) diff --git a/src/config.ts b/src/config.ts index 8e594e2..258fc67 100644 --- a/src/config.ts +++ b/src/config.ts @@ -14,7 +14,9 @@ export const CONFIG = { INTENT: { MODEL: string("INTENT_MODEL", { default: "openai/gpt-oss-20b" }), TIMEOUT_MS: int("INTENT_TIMEOUT_MS", { default: 3000, min: 1 }), - RELEVANCY_THRESHOLD: int("INTENT_RELEVANCY_THRESHOLD", { default: 0, min: 0, max: 10 }), + MIN_SCORE: int("INTENT_MIN_SCORE", { default: 0 }), + MAX_SCORE: int("INTENT_MAX_SCORE", { default: 10, min: 1 }), + RELEVANCY_THRESHOLD: int("INTENT_RELEVANCY_THRESHOLD", { default: 0 }), BATCH_SIZE: int("INTENT_BATCH_SIZE", { default: 20, min: 1 }), TINY_BATCH_FRACTION: number("INTENT_TINY_BATCH_FRACTION", { default: 0.2, min: 0, max: 1 }), }, diff --git a/src/intent.int.test.ts b/src/intent.int.test.ts index e115ed4..46ae40c 100644 --- a/src/intent.int.test.ts +++ b/src/intent.int.test.ts @@ -1,11 +1,8 @@ import { describe, expect, test } from "vitest"; -import { CONFIG } from "./config"; import { Intent } from "./intent"; -const hasKey = Boolean(CONFIG.GROQ.API_KEY); - -describe.skipIf(!hasKey)("reranker integration", () => { +describe("reranker integration", () => { test.concurrent( "reranker end-to-end", async () => { diff --git a/src/intent.ts b/src/intent.ts index 4f84110..91f51ff 100644 --- a/src/intent.ts +++ b/src/intent.ts @@ -20,7 +20,7 @@ import type { /** * LLM-based reranker for arbitrary items. * - * Uses a listwise LLM approach to score candidates 0-10 based on relevance to a query, + * Uses a listwise LLM approach to score candidates within a configurable range based on relevance to a query, * then filters by threshold and returns results sorted by score with stable ordering. * * @template T - The type of items to rerank (defaults to any) @@ -107,21 +107,33 @@ export class Intent { relevancyThreshold: options.relevancyThreshold ?? this.env.INTENT.RELEVANCY_THRESHOLD, batchSize: options.batchSize ?? this.env.INTENT.BATCH_SIZE, tinyBatchFraction: options.tinyBatchFraction ?? this.env.INTENT.TINY_BATCH_FRACTION, + minScore: options.minScore ?? this.env.INTENT.MIN_SCORE, + maxScore: options.maxScore ?? this.env.INTENT.MAX_SCORE, }; } /** * Validates the configuration values. * - * Ensures relevancyThreshold is within the valid 0-10 range. + * Ensures the configured score range is valid and the relevancyThreshold is in range. * - * @throws {Error} If relevancyThreshold is not between 0 and 10 + * @throws {Error} If maxScore is below minScore + * @throws {Error} If relevancyThreshold is not within [minScore, maxScore] * @private */ private validateConfig(): void { - if (this.cfg.relevancyThreshold < 0 || this.cfg.relevancyThreshold > 10) { + if (this.cfg.maxScore < this.cfg.minScore) { throw new Error( - `intent: relevancyThreshold must be between 0 and 10, got ${this.cfg.relevancyThreshold}`, + `intent: maxScore must be >= minScore, got minScore=${this.cfg.minScore} maxScore=${this.cfg.maxScore}`, + ); + } + + if ( + this.cfg.relevancyThreshold < this.cfg.minScore || + this.cfg.relevancyThreshold > this.cfg.maxScore + ) { + throw new Error( + `intent: relevancyThreshold must be between ${this.cfg.minScore} and ${this.cfg.maxScore}, got ${this.cfg.relevancyThreshold}`, ); } } @@ -163,11 +175,14 @@ export class Intent { * @param options.summary - Optional function extracting a short description for LLM reasoning * @param options.model - Optional model name override (default: INTENT_MODEL or "openai/gpt-oss-20b") * @param options.timeoutMs - Optional timeout in milliseconds (default: INTENT_TIMEOUT_MS or 3000) - * @param options.relevancyThreshold - Optional minimum score 0-10 to include results (default: INTENT_RELEVANCY_THRESHOLD or 0) + * @param options.relevancyThreshold - Optional minimum score to include results (default: INTENT_RELEVANCY_THRESHOLD) + * @param options.minScore - Optional minimum score value (default: INTENT_MIN_SCORE or 0) + * @param options.maxScore - Optional maximum score value (default: INTENT_MAX_SCORE or 10) * @param options.batchSize - Optional number of candidates per LLM call (default: INTENT_BATCH_SIZE or 20) * @param options.tinyBatchFraction - Optional threshold for merging small batches (default: INTENT_TINY_BATCH_FRACTION or 0.2) * @throws {Error} If no LLM client is provided and GROQ_API_KEY is not set - * @throws {Error} If relevancyThreshold is not between 0 and 10 + * @throws {Error} If maxScore is below minScore + * @throws {Error} If relevancyThreshold is not within [minScore, maxScore] * * @example * ```typescript @@ -347,7 +362,7 @@ export class Intent { /** * Build the JSON schema and chat messages payload for the LLM. * - * Creates a strict JSON schema requiring one integer property (0-10) per candidate key, + * Creates a strict JSON schema requiring one integer property (minScore-maxScore) per candidate key, * and constructs system + user messages instructing the LLM to score relevance. * * @param query - The search query to evaluate candidates against @@ -360,8 +375,11 @@ export class Intent { items: Array<{ key: string; summary: string }>, ): { schema: JSONObject; messages: ChatMessage[] } { const keys = items.map((x) => x.key); - const schema: JSONObject = buildRelevancySchema(keys); - const messages = buildMessages(query, items); + const schema: JSONObject = buildRelevancySchema(keys, this.cfg.minScore, this.cfg.maxScore); + const messages = buildMessages(query, items, { + minScore: this.cfg.minScore, + maxScore: this.cfg.maxScore, + }); return { schema, messages }; } @@ -401,7 +419,7 @@ export class Intent { /** * Apply relevancy threshold filtering and stable sorting. * - * Scores are clamped to 0-10 range, then filtered to keep only items with + * Scores are clamped to the configured score range, then filtered to keep only items with * score > threshold. Results are sorted by score descending, with ties * preserving original input order for deterministic results. * @@ -420,7 +438,11 @@ export class Intent { idx, explanation: typeof evaluations[key]?.explanation === "string" ? evaluations[key].explanation : "", - score: clamp(evaluations[key]?.score ?? 0, 0, 10), + score: clamp( + evaluations[key]?.score ?? this.cfg.minScore, + this.cfg.minScore, + this.cfg.maxScore, + ), })); const filtered = scored.filter(({ score }) => score > threshold); diff --git a/src/intent.unit.test.ts b/src/intent.unit.test.ts index 18ce38a..90c5371 100644 --- a/src/intent.unit.test.ts +++ b/src/intent.unit.test.ts @@ -28,7 +28,7 @@ function makeCtx(overrides: Partial = {}): IntentContext & { describe("Intent.rank", () => { test("candidate evaluation schema defines explanation before score", () => { - const schema = buildCandidateEvaluationSchema(); + const schema = buildCandidateEvaluationSchema(0, 10); expect(Object.keys(schema.properties)).toEqual(["explanation", "score"]); }); @@ -46,14 +46,53 @@ describe("Intent.rank", () => { const ctx = makeCtx(); expect( () => new Intent({ ...ctx, key: (c) => c.key, relevancyThreshold: -1 }), - ).toThrow(/relevancyThreshold must be between 0 and 10/); + ).toThrow(/relevancyThreshold must be between/); }); test("throws when threshold is above 10", async () => { const ctx = makeCtx(); expect( () => new Intent({ ...ctx, key: (c) => c.key, relevancyThreshold: 11 }), - ).toThrow(/relevancyThreshold must be between 0 and 10/); + ).toThrow(/relevancyThreshold must be between/); + }); + + test("throws when maxScore is below minScore", async () => { + const ctx = makeCtx(); + expect( + () => + new Intent({ + ...ctx, + key: (c) => c.key, + minScore: 5, + maxScore: 4, + }), + ).toThrow(/maxScore must be >= minScore/); + }); + + test("supports non-0..10 score ranges", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { + A: { explanation: "high", score: 5 }, + B: { explanation: "low", score: 3 }, + }, + }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + minScore: 2, + maxScore: 5, + relevancyThreshold: 3, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.rank("query", input); + expect(res.map((c) => c.key)).toEqual(["A"]); }); test("returns empty list for zero candidates", async () => { const ctx = makeCtx(); diff --git a/src/messages.ts b/src/messages.ts index 35b1782..3b352bc 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -9,31 +9,35 @@ import type { ChatMessage, IntentCandidate } from "./types"; * 1. System message: Defines the scoring task, output format, and constraints * 2. User message: Contains the query and candidate items as JSON * - * The system prompt emphasizes using the full 0-10 range and being decisive + * The system prompt emphasizes using the full configured score range and being decisive * about relevance, with strict instructions to return only the score mapping. * * @param query - The search query or user intent * @param candidates - Array of candidates with keys and summaries * @returns Array of chat messages ready for LLM consumption */ -export function buildMessages(query: string, candidates: IntentCandidate[]): ChatMessage[] { - const system = `The user will provide a short description of a query they are trying to automate, along with a JSON blob containing candidate_search_results. Each candidate result has a uniquely identifying key and a short summary. Your task is to assess each candidate and return a JSON object that maps candidate keys to objects of the form {"explanation": string, "score": integer} where score is from 0 to 10: 0 means not relevant at all, and 10 means highly relevant. Sometimes none are relevant, sometimes all are relevant. Be aggressive and decisive on relevancy. +export function buildMessages( + query: string, + candidates: IntentCandidate[], + scoreRange: { minScore: number; maxScore: number }, +): ChatMessage[] { + const system = `The user will provide a short description of a query they are trying to automate, along with a JSON blob containing candidate_search_results. Each candidate result has a uniquely identifying key and a short summary. Your task is to assess each candidate and return a JSON object that maps candidate keys to objects of the form {"explanation": string, "score": integer} where score is from ${scoreRange.minScore} to ${scoreRange.maxScore}: ${scoreRange.minScore} means not relevant at all, and ${scoreRange.maxScore} means highly relevant. Sometimes none are relevant, sometimes all are relevant. Be aggressive and decisive on relevancy. -It is okay to return 0 if the candidate is not relevant to the query. It is okay to return 10 if the candidate is highly relevant to the query. Use the full range of scores. +It is okay to return ${scoreRange.minScore} if the candidate is not relevant to the query. It is okay to return ${scoreRange.maxScore} if the candidate is highly relevant to the query. Use the full range of scores. Every candidate MUST include an explanation. Write the explanation first, then the score. The explanation should be concise (1-3 sentences), concrete, and reference the query intent and the candidate summary. Every key in candidate_search_results must be present in your output mapping. Do not add any keys that are not present in candidate_search_results. Every key in candidate_search_results must map to an object with: - explanation: string -- score: integer from 0 to 10 +- score: integer from ${scoreRange.minScore} to ${scoreRange.maxScore} Do not, in your generated JSON, include anything other than the \`"{key}": {"explanation": "...", "score": 7}\` mappings. Do not include any other text outside the JSON. Return a JSON object that matches the enforced JSON schema for response formatting. Use the candidate.key as the property name in the output mapping. The JSON you return should be of the form: { - "Key for document 1": { "explanation": "...", "score": 0 }, - "Key for document 2": { "explanation": "...", "score": 7 }, + "Key for document 1": { "explanation": "...", "score": ${scoreRange.minScore} }, + "Key for document 2": { "explanation": "...", "score": ${scoreRange.maxScore} }, ... } diff --git a/src/providers/groq.int.test.ts b/src/providers/groq.int.test.ts index 37f9de4..46b9684 100644 --- a/src/providers/groq.int.test.ts +++ b/src/providers/groq.int.test.ts @@ -42,8 +42,12 @@ describe("groq provider integration", () => { { key: "A", summary: "first" }, { key: "B", summary: "second" }, ]; - const schema = buildRelevancySchema(candidates.map((c) => c.key)); - const messages = buildMessages("choose best", candidates); + const schema = buildRelevancySchema( + candidates.map((c) => c.key), + 0, + 10, + ); + const messages = buildMessages("choose best", candidates, { minScore: 0, maxScore: 10 }); const { data } = await client.call>( messages, schema, @@ -60,26 +64,37 @@ describe("groq provider integration", () => { } }); - test.concurrent("assigns 0 to unrelated and >0 to related", async () => { - const client = createDefaultGroqClient(CONFIG.GROQ.API_KEY); - const candidates = [ - { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, - { key: "Banana Bread Recipe", summary: "How to bake banana bread" }, - { key: "Eiffel Tower History", summary: "Timeline of the Eiffel Tower construction" }, - ]; - const schema = buildRelevancySchema(candidates.map((c) => c.key)); - const messages = buildMessages("Help me with JavaScript array sorting", candidates); - const { data } = await client.call>( - messages, - schema, - { - timeoutMs: 6000, - }, - ); - // Related candidate should be > 0 - expect(data["JS Arrays"]?.score).toBeGreaterThan(0); - // Unrelated candidates should be 0 - expect(data["Banana Bread Recipe"]?.score).toBe(0); - expect(data["Eiffel Tower History"]?.score).toBe(0); - }); + test.concurrent( + "assigns 0 to unrelated and >0 to related", + async () => { + const client = createDefaultGroqClient(CONFIG.GROQ.API_KEY); + const candidates = [ + { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, + { key: "Banana Bread Recipe", summary: "How to bake banana bread" }, + { key: "Eiffel Tower History", summary: "Timeline of the Eiffel Tower construction" }, + ]; + const schema = buildRelevancySchema( + candidates.map((c) => c.key), + 0, + 10, + ); + const messages = buildMessages("Help me with JavaScript array sorting", candidates, { + minScore: 0, + maxScore: 10, + }); + const { data } = await client.call>( + messages, + schema, + { + timeoutMs: 10000, + }, + ); + // Related candidate should be > 0 + expect(data["JS Arrays"]?.score).toBeGreaterThan(0); + // Unrelated candidates should be 0 + expect(data["Banana Bread Recipe"]?.score).toBe(0); + expect(data["Eiffel Tower History"]?.score).toBe(0); + }, + 15000, + ); }); diff --git a/src/schema.ts b/src/schema.ts index 1328eef..239ef85 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -38,7 +38,7 @@ export function buildCandidateEvaluationSchema(): CandidateEvaluationSchema { * * Each candidate key maps to an object containing: * - explanation: a short justification of the score - * - score: an integer 0-10 + * - score: an integer within the configured range * * Property order is intentional (`explanation` then `score`) to encourage the * model to generate explanations before scores in structured-output modes. @@ -46,7 +46,11 @@ export function buildCandidateEvaluationSchema(): CandidateEvaluationSchema { * @param keys - Array of unique candidate keys * @returns JSON schema object enforcing exact structure of response */ -export function buildRelevancySchema(keys: string[]): JSONObject { +export function buildRelevancySchema( + keys: string[], + minScore: number, + maxScore: number, +): JSONObject { const evaluationSchema = buildCandidateEvaluationSchema(); const properties: Record = {}; @@ -56,8 +60,7 @@ export function buildRelevancySchema(keys: string[]): JSONObject { return { title: "Query / Candidate Relevancy Assessment", - description: - "Map candidate results for a search query to relevancy scores (0-10) with explanations.", + description: `Map candidate results for a search query to relevancy scores (${minScore}-${maxScore}) with explanations.`, type: "object", properties, required: keys, diff --git a/src/types.ts b/src/types.ts index c9f56ed..a1279e5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -69,6 +69,8 @@ export type IntentConfig = { relevancyThreshold?: number; batchSize?: number; tinyBatchFraction?: number; + minScore?: number; + maxScore?: number; }; /** From 033406739c79a5221176e2e18fb997579b46ace2 Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Sun, 21 Dec 2025 22:48:24 -0800 Subject: [PATCH 05/12] test(int): expand Intent real-wire integration coverage Add a broad set of unmocked integration tests for Intent that exercise production wire paths against the configured Groq provider. Coverage includes: - obvious-match ranking - threshold behavior for all-unrelated inputs - stable ordering for ties/near-ties (when all items are returned) - explain=true output shape and non-empty explanations - custom extractors for nested objects - unicode keys/summaries and punctuation/newline keys - batching behavior and larger stress case across multiple batches - per-call userId override Also refresh the Groq provider relevance sanity test candidates to reduce accidental topical overlap. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- src/intent.int.test.ts | 330 ++++++++++++++++++++++++++++++++- src/providers/groq.int.test.ts | 8 +- 2 files changed, 326 insertions(+), 12 deletions(-) diff --git a/src/intent.int.test.ts b/src/intent.int.test.ts index 46ae40c..1c57a32 100644 --- a/src/intent.int.test.ts +++ b/src/intent.int.test.ts @@ -3,21 +3,335 @@ import { describe, expect, test } from "vitest"; import { Intent } from "./intent"; describe("reranker integration", () => { + const defaultTimeoutMs = 10000; + const scoreRange = { minScore: 0, maxScore: 10 }; + + test.concurrent( + "ranks obvious match first", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: 0, + }); + + const out = await intent.rank("Help me sort a JavaScript array", [ + { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, + { key: "Saturns Moons", summary: "The chemical composition of Saturn's moons" }, + { key: "Eiffel Tower", summary: "Directions to the tower" }, + ]); + + expect(out.length).toBeGreaterThanOrEqual(1); + expect(out[0]?.key).toBe("JS Arrays"); + }, + 30000, + ); + + test.concurrent( + "returns empty list when everything is unrelated (threshold > minScore)", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: scoreRange.minScore + 1, + ...scoreRange, + }); + + const out = await intent.rank("JavaScript array sorting", [ + { key: "Banana Bread", summary: "How to bake banana bread" }, + { key: "Eiffel Tower", summary: "Timeline of the Eiffel Tower construction" }, + { key: "Roman Emperors", summary: "A list of Roman emperors" }, + ]); + + expect(out).toEqual([]); + }, + 30000, + ); + test.concurrent( - "reranker end-to-end", + "preserves input order on ties (or near-ties) between similar candidates", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, summary: (x) => x.summary, - timeoutMs: 5000, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: 0, }); - const out = await intent.rank("select the best doc", [ - { key: "Alpha", summary: "Doc about alpha" }, - { key: "Beta", summary: "Doc about beta" }, + + const input = [ + { key: "Guide A", summary: "How to sort JavaScript arrays with Array.prototype.sort" }, + { key: "Guide B", summary: "Sorting arrays in JavaScript using a comparator function" }, + { key: "Guide C", summary: "JavaScript array sorting examples and best practices" }, + ]; + + const out = await intent.rank("JavaScript array sorting", input); + expect(out.length).toBeGreaterThanOrEqual(1); + + const outputKeys = out.map((x) => x.key); + for (const k of outputKeys) { + expect(input.some((i) => i.key === k)).toBe(true); + } + + if (out.length === input.length) { + expect(outputKeys).toEqual(input.map((x) => x.key)); + } + }, + 30000, + ); + + test.concurrent( + "explain=true returns item+explanation and filters by threshold", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: 1, + }); + + const out = await intent.rank( + "Help me sort a JavaScript array", + [ + { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, + { key: "Banana Bread", summary: "How to bake banana bread" }, + ], + { explain: true }, + ); + + expect(out.length).toBeGreaterThanOrEqual(1); + expect(out[0]?.item.key).toBe("JS Arrays"); + expect(typeof out[0]?.explanation).toBe("string"); + expect(out[0]?.explanation.length).toBeGreaterThan(0); + }, + 30000, + ); + + test.concurrent( + "supports custom extractors over nested objects", + async () => { + type Doc = { id: string; meta: { title: string }; body: string }; + const intent = new Intent({ + key: (d) => d.meta.title, + summary: (d) => d.body, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: 1, + }); + + const out = await intent.rank("find expense reports", [ + { id: "1", meta: { title: "Expense report Q3" }, body: "Travel and meals reimbursement" }, + { id: "2", meta: { title: "Vacation photos" }, body: "Beach, family, sunsets" }, + ]); + + expect(out.length).toBe(1); + expect(out[0]?.meta.title).toBe("Expense report Q3"); + }, + 30000, + ); + + test.concurrent( + "supports unicode keys and summaries", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: 0, + }); + + const out = await intent.rank("JavaScript array sorting", [ + { key: "Café", summary: "Guía para ordenar arrays en JavaScript" }, + { key: "東京", summary: "JavaScriptで配列をソートする方法" }, + { key: "Banana Bread", summary: "How to bake banana bread" }, + ]); + + expect(out.length).toBeGreaterThanOrEqual(1); + expect(["Café", "東京"].includes(out[0]?.key ?? "")).toBe(true); + }, + 30000, + ); + + test.concurrent( + "supports keys with punctuation and whitespace", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: 0, + }); + + const out = await intent.rank("JavaScript array sorting", [ + { + key: "Array.sort() / comparator", + summary: "Using Array.prototype.sort with a compare function in JS", + }, + { + key: "Stable sort: ties, order", + summary: "How stable sorting behaves; preserving order for equal elements", + }, + { + key: "Key with spaces\n(and newline)", + summary: "Examples of sorting arrays in JavaScript", + }, ]); - expect(out.length).toBeGreaterThanOrEqual(0); - expect(out.length).toBeLessThanOrEqual(2); + + expect(out.length).toBeGreaterThanOrEqual(1); + }, + 30000, + ); + + test.concurrent( + "disambiguates duplicate keys without failing", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: 0, + }); + + const out = await intent.rank("JavaScript array sorting", [ + { key: "Same", summary: "Sorting arrays in JavaScript" }, + { key: "Same", summary: "Another note about Array.prototype.sort" }, + { key: "Other", summary: "Banana bread recipe" }, + ]); + + expect(out.length).toBeGreaterThanOrEqual(1); + expect(out.some((x) => x.key === "Same")).toBe(true); + }, + 30000, + ); + + test.concurrent( + "supports non-0..10 score ranges", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + minScore: 1, + maxScore: 5, + relevancyThreshold: 2, + }); + + const out = await intent.rank("JavaScript array sorting", [ + { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, + { key: "Banana Bread", summary: "How to bake banana bread" }, + ]); + + expect(out.length).toBeGreaterThanOrEqual(1); + expect(out[0]?.key).toBe("JS Arrays"); + }, + 30000, + ); + + test.concurrent( + "handles batching for larger candidate sets", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + batchSize: 3, + tinyBatchFraction: 0.2, + relevancyThreshold: 0, + }); + + const candidates = [ + { key: "Array.sort", summary: "JavaScript Array.prototype.sort reference" }, + { key: "Comparator", summary: "How to write a comparator function in JS" }, + { key: "Stable sort", summary: "Discussion of stable sorting behavior" }, + { key: "Quickstart", summary: "Basic JS arrays tutorial" }, + { key: "Banana bread", summary: "Recipe for banana bread" }, + { key: "Eiffel Tower", summary: "History of the Eiffel Tower" }, + { key: "Node streams", summary: "Guide to Node.js streams" }, + ]; + + const out = await intent.rank("JavaScript array sorting", candidates); + expect(out.length).toBeGreaterThanOrEqual(1); + expect(out.length).toBeLessThanOrEqual(candidates.length); + }, + 60000, + ); + + test.concurrent( + "stress: ranks with many candidates across batches", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + batchSize: 5, + tinyBatchFraction: 0.2, + relevancyThreshold: 0, + }); + + const candidates = Array.from({ length: 30 }).map((_, i) => ({ + key: `Doc ${i + 1}`, + summary: + i % 6 === 0 + ? "JavaScript array sorting with comparator and examples" + : "Unrelated notes about cooking, travel, or history", + })); + + const out = await intent.rank("JavaScript array sorting", candidates); + expect(out.length).toBeGreaterThanOrEqual(1); + expect(out.length).toBeLessThanOrEqual(candidates.length); + }, + 120000, + ); + + test.concurrent( + "explain=true stays aligned with items across batching", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + batchSize: 4, + tinyBatchFraction: 0.2, + relevancyThreshold: 0, + }); + + const input = Array.from({ length: 12 }).map((_, i) => ({ + key: `Item ${i + 1}`, + summary: i % 3 === 0 ? "JavaScript array sorting examples" : "Completely unrelated topic", + })); + + const out = await intent.rank("JavaScript array sorting", input, { explain: true }); + expect(out.length).toBeGreaterThanOrEqual(1); + + for (const r of out) { + expect(input.some((x) => x.key === r.item.key)).toBe(true); + expect(typeof r.explanation).toBe("string"); + } + }, + 120000, + ); + + test.concurrent( + "accepts per-call userId override", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: 0, + }); + + const out = await intent.rank( + "JavaScript array sorting", + [ + { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, + { key: "Banana Bread", summary: "How to bake banana bread" }, + ], + { userId: "integration-test-user" }, + ); + + expect(out.length).toBeGreaterThanOrEqual(1); }, - 20000, + 30000, ); }); diff --git a/src/providers/groq.int.test.ts b/src/providers/groq.int.test.ts index 46b9684..256d6f6 100644 --- a/src/providers/groq.int.test.ts +++ b/src/providers/groq.int.test.ts @@ -70,8 +70,8 @@ describe("groq provider integration", () => { const client = createDefaultGroqClient(CONFIG.GROQ.API_KEY); const candidates = [ { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, - { key: "Banana Bread Recipe", summary: "How to bake banana bread" }, - { key: "Eiffel Tower History", summary: "Timeline of the Eiffel Tower construction" }, + { key: "Saturns Moons", summary: "The chemical composition of Saturn's moons" }, + { key: "Eiffel Tower", summary: "Directions to the tower" }, ]; const schema = buildRelevancySchema( candidates.map((c) => c.key), @@ -92,8 +92,8 @@ describe("groq provider integration", () => { // Related candidate should be > 0 expect(data["JS Arrays"]?.score).toBeGreaterThan(0); // Unrelated candidates should be 0 - expect(data["Banana Bread Recipe"]?.score).toBe(0); - expect(data["Eiffel Tower History"]?.score).toBe(0); + expect(data["Saturns Moons"]?.score).toBe(0); + expect(data["Eiffel Tower"]?.score).toBe(0); }, 15000, ); From db2d0c2b4e103be3bd3ebd4f70563af861ffcbd2 Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Sun, 21 Dec 2025 22:53:57 -0800 Subject: [PATCH 06/12] feat(config): replace INTENT_MODEL with provider selection - Introduce INTENT_PROVIDER (enum; currently only GROQ) and remove INTENT_MODEL. - Add enumString() config helper for validated enum env vars. - Make Intent provider-driven and source the model from GROQ defaults when provider=GROQ. - Update types and unit tests to cover provider parsing and model resolution. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- src/config.ts | 4 ++-- src/intent.int.test.ts | 2 +- src/intent.ts | 19 ++++++++++++++++--- src/intent.unit.test.ts | 25 +++++++++++++++++++++++++ src/lib/config.ts | 18 ++++++++++++++++++ src/lib/config.unit.test.ts | 19 +++++++++++++++++++ src/types.ts | 2 +- 7 files changed, 82 insertions(+), 7 deletions(-) diff --git a/src/config.ts b/src/config.ts index 258fc67..746407a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,5 @@ import "dotenv/config"; -import { int, number, string } from "./lib/config"; +import { enumString, int, number, string } from "./lib/config"; /** * Exported config object (no function call required) following API patterns. @@ -12,7 +12,7 @@ export const CONFIG = { JSON_REPAIR_ATTEMPTS: int("GROQ_JSON_REPAIR_ATTEMPTS", { default: 3, min: 0 }), }, INTENT: { - MODEL: string("INTENT_MODEL", { default: "openai/gpt-oss-20b" }), + PROVIDER: enumString("INTENT_PROVIDER", { default: "GROQ", values: ["GROQ"] as const }), TIMEOUT_MS: int("INTENT_TIMEOUT_MS", { default: 3000, min: 1 }), MIN_SCORE: int("INTENT_MIN_SCORE", { default: 0 }), MAX_SCORE: int("INTENT_MAX_SCORE", { default: 10, min: 1 }), diff --git a/src/intent.int.test.ts b/src/intent.int.test.ts index 1c57a32..652c508 100644 --- a/src/intent.int.test.ts +++ b/src/intent.int.test.ts @@ -17,8 +17,8 @@ describe("reranker integration", () => { }); const out = await intent.rank("Help me sort a JavaScript array", [ - { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, { key: "Saturns Moons", summary: "The chemical composition of Saturn's moons" }, + { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, { key: "Eiffel Tower", summary: "Directions to the tower" }, ]); diff --git a/src/intent.ts b/src/intent.ts index 91f51ff..9d437f1 100644 --- a/src/intent.ts +++ b/src/intent.ts @@ -56,6 +56,19 @@ export class Intent { private readonly extractors: Required>; private readonly env: typeof CONFIG; + /** + * Resolve the model name to use for this Intent instance. + * + * Intent is provider-driven. Today only GROQ is supported; when using GROQ + * we always take the model from GROQ's config defaults. + * + * @returns Provider-specific model name + * @private + */ + private resolveModel(): string { + return this.env.GROQ.DEFAULT_MODEL; + } + /** * Builds the context object from options. * @@ -102,7 +115,7 @@ export class Intent { */ private buildConfig(options: IntentOptions): Required { return { - model: options.model ?? this.env.INTENT.MODEL, + provider: options.provider ?? this.env.INTENT.PROVIDER, timeoutMs: options.timeoutMs ?? this.env.INTENT.TIMEOUT_MS, relevancyThreshold: options.relevancyThreshold ?? this.env.INTENT.RELEVANCY_THRESHOLD, batchSize: options.batchSize ?? this.env.INTENT.BATCH_SIZE, @@ -173,7 +186,7 @@ export class Intent { * @param options.userId - Optional user identifier for LLM provider abuse monitoring * @param options.key - Optional function extracting a short human-readable key from items * @param options.summary - Optional function extracting a short description for LLM reasoning - * @param options.model - Optional model name override (default: INTENT_MODEL or "openai/gpt-oss-20b") + * @param options.provider - Optional provider override (default: INTENT_PROVIDER or "GROQ") * @param options.timeoutMs - Optional timeout in milliseconds (default: INTENT_TIMEOUT_MS or 3000) * @param options.relevancyThreshold - Optional minimum score to include results (default: INTENT_RELEVANCY_THRESHOLD) * @param options.minScore - Optional minimum score value (default: INTENT_MIN_SCORE or 0) @@ -401,7 +414,7 @@ export class Intent { userId?: string, ): Promise | null> { const config: LlmCallConfig = { - model: this.cfg.model, + model: this.resolveModel(), temperature: 0, timeoutMs: this.cfg.timeoutMs, }; diff --git a/src/intent.unit.test.ts b/src/intent.unit.test.ts index 90c5371..1173095 100644 --- a/src/intent.unit.test.ts +++ b/src/intent.unit.test.ts @@ -27,6 +27,30 @@ function makeCtx(overrides: Partial = {}): IntentContext & { } describe("Intent.rank", () => { + test("uses GROQ default model when no model override is provided", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { + A: { explanation: "a", score: 10 }, + B: { explanation: "b", score: 0 }, + }, + }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + + await intent.rank("query", [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]); + + const call = (ctx.llm.call as any).mock.calls[0]; + expect(call[2].model).toBe(CONFIG.GROQ.DEFAULT_MODEL); + }); + test("candidate evaluation schema defines explanation before score", () => { const schema = buildCandidateEvaluationSchema(0, 10); expect(Object.keys(schema.properties)).toEqual(["explanation", "score"]); @@ -193,6 +217,7 @@ describe("Intent.rank", () => { expect(res.map((c) => c.key)).toEqual(["A", "B"]); const call = (ctx.llm.call as any).mock.calls[0]; expect(call[2].timeoutMs).toBe(3000); // default + expect(call[2].model).toBe("openai/gpt-oss-20b"); // GROQ default model }); test("handles non-numeric or missing scores by clamping to 0", async () => { diff --git a/src/lib/config.ts b/src/lib/config.ts index 63c282c..cec1ff8 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -10,6 +10,7 @@ export type BaseOptions = { default?: T }; export type RangeOptions = BaseOptions & { min?: number; max?: number }; +export type EnumOptions = BaseOptions & { values: T }; function throwRequiredEnvVar(name: string): never { throw new Error(`${name} is required.`); @@ -29,6 +30,23 @@ export function string(name: string, opts?: BaseOptions): string { return throwRequiredEnvVar(name); } +/** + * Read an env var as a constrained enum string. + * + * When set, validates the value is included in opts.values. + * When unset, returns opts.default if provided, otherwise throws. + */ +export function enumString( + name: string, + opts: EnumOptions, +): T[number] { + const value = opts.default !== undefined ? string(name, { default: opts.default }) : string(name); + if (opts.values.includes(value)) { + return value; + } + throw new Error(`${name} must be one of: ${opts.values.join(", ")}`); +} + export function boolean(name: string, opts?: BaseOptions): boolean { const value = process.env[name]; const exists = value != null && value.length > 0; diff --git a/src/lib/config.unit.test.ts b/src/lib/config.unit.test.ts index 2f3a072..c3a46b4 100644 --- a/src/lib/config.unit.test.ts +++ b/src/lib/config.unit.test.ts @@ -137,4 +137,23 @@ describe("lib/config", () => { expect(cfg.number(VAR, { default: 9.9 })).toBeCloseTo(9.9); }); }); + + describe("enumString()", () => { + it("returns the env value when it is included in values", () => { + process.env[VAR] = "GROQ"; + expect(cfg.enumString(VAR, { values: ["GROQ"] as const })).toBe("GROQ"); + }); + + it("returns default when missing", () => { + unset(VAR); + expect(cfg.enumString(VAR, { default: "GROQ", values: ["GROQ"] as const })).toBe("GROQ"); + }); + + it("throws when the value is not included in values", () => { + process.env[VAR] = "NOPE"; + expect(() => cfg.enumString(VAR, { values: ["GROQ"] as const })).toThrow( + /must be one of: GROQ/, + ); + }); + }); }); diff --git a/src/types.ts b/src/types.ts index a1279e5..e49b367 100644 --- a/src/types.ts +++ b/src/types.ts @@ -64,7 +64,7 @@ export type CamelCasedProps = { * This is a camelCase version of the INTENT config object from config.ts. */ export type IntentConfig = { - model?: string; + provider?: "GROQ"; timeoutMs?: number; relevancyThreshold?: number; batchSize?: number; From 89ccb874bcd416147ff0321f1c826c1fe22ffc54 Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Sun, 21 Dec 2025 22:59:36 -0800 Subject: [PATCH 07/12] test(int): add default-constructor Intent examples Add real-wire integration coverage for the simplest usage pattern (new Intent()) using primitive string items, validating that default extractors and provider wiring work end-to-end without any configuration. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- src/intent.int.test.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/intent.int.test.ts b/src/intent.int.test.ts index 652c508..bf01dd8 100644 --- a/src/intent.int.test.ts +++ b/src/intent.int.test.ts @@ -28,6 +28,35 @@ describe("reranker integration", () => { 30000, ); + test.concurrent( + "supports the simplest usage: new Intent() with string items", + async () => { + const intent = new Intent(); + const items = ["apple", "banana", "orange", "grape"]; + + const ranked = await intent.rank("citrus fruits", items); + expect(ranked.length).toBeGreaterThanOrEqual(1); + expect(ranked.length).toBeLessThanOrEqual(items.length); + + // Prefer stable assertions that still validate the core behavior. + expect(ranked.includes("orange")).toBe(true); + }, + 30000, + ); + + test.concurrent( + "supports simplest usage with a different query", + async () => { + const intent = new Intent(); + const items = ["tiny", "small", "medium", "large", "huge"]; + + const ranked = await intent.rank("size", items); + expect(ranked.length).toBeGreaterThanOrEqual(1); + expect(ranked.length).toBeLessThanOrEqual(items.length); + }, + 30000, + ); + test.concurrent( "returns empty list when everything is unrelated (threshold > minScore)", async () => { From 354b567fd4727f86d34fd146420e2f8ecc70f11e Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Sun, 21 Dec 2025 23:00:55 -0800 Subject: [PATCH 08/12] test(int): improve integration test descriptions Rename Intent integration tests to be clearer, more user-facing examples while keeping behavior unchanged. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- src/intent.int.test.ts | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/intent.int.test.ts b/src/intent.int.test.ts index bf01dd8..3f94ec5 100644 --- a/src/intent.int.test.ts +++ b/src/intent.int.test.ts @@ -7,7 +7,7 @@ describe("reranker integration", () => { const scoreRange = { minScore: 0, maxScore: 10 }; test.concurrent( - "ranks obvious match first", + "ranks the most relevant candidate first (simple object candidates)", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, @@ -29,36 +29,32 @@ describe("reranker integration", () => { ); test.concurrent( - "supports the simplest usage: new Intent() with string items", + "supports defaults: new Intent() ranks plain strings", async () => { const intent = new Intent(); const items = ["apple", "banana", "orange", "grape"]; const ranked = await intent.rank("citrus fruits", items); - expect(ranked.length).toBeGreaterThanOrEqual(1); - expect(ranked.length).toBeLessThanOrEqual(items.length); - - // Prefer stable assertions that still validate the core behavior. + expect(ranked.length).toBe(1); expect(ranked.includes("orange")).toBe(true); }, 30000, ); test.concurrent( - "supports simplest usage with a different query", + "supports defaults: new Intent() accepts arbitrary queries", async () => { const intent = new Intent(); const items = ["tiny", "small", "medium", "large", "huge"]; const ranked = await intent.rank("size", items); - expect(ranked.length).toBeGreaterThanOrEqual(1); - expect(ranked.length).toBeLessThanOrEqual(items.length); + expect(ranked).toEqual(["tiny", "small", "medium", "large", "huge"]); }, 30000, ); test.concurrent( - "returns empty list when everything is unrelated (threshold > minScore)", + "returns [] when all candidates are unrelated and threshold > minScore", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, @@ -80,7 +76,7 @@ describe("reranker integration", () => { ); test.concurrent( - "preserves input order on ties (or near-ties) between similar candidates", + "preserves input order when all candidates are returned", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, @@ -111,7 +107,7 @@ describe("reranker integration", () => { ); test.concurrent( - "explain=true returns item+explanation and filters by threshold", + "explain=true returns { item, explanation } and still filters by threshold", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, @@ -138,7 +134,7 @@ describe("reranker integration", () => { ); test.concurrent( - "supports custom extractors over nested objects", + "supports custom extractors (nested objects)", async () => { type Doc = { id: string; meta: { title: string }; body: string }; const intent = new Intent({ @@ -160,7 +156,7 @@ describe("reranker integration", () => { ); test.concurrent( - "supports unicode keys and summaries", + "handles unicode keys and summaries", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, @@ -182,7 +178,7 @@ describe("reranker integration", () => { ); test.concurrent( - "supports keys with punctuation and whitespace", + "handles keys with punctuation, whitespace, and newlines", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, @@ -212,7 +208,7 @@ describe("reranker integration", () => { ); test.concurrent( - "disambiguates duplicate keys without failing", + "disambiguates duplicate keys", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, @@ -234,7 +230,7 @@ describe("reranker integration", () => { ); test.concurrent( - "supports non-0..10 score ranges", + "supports non-default score ranges (minScore/maxScore)", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, @@ -257,7 +253,7 @@ describe("reranker integration", () => { ); test.concurrent( - "handles batching for larger candidate sets", + "handles batching across multiple LLM calls", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, @@ -286,7 +282,7 @@ describe("reranker integration", () => { ); test.concurrent( - "stress: ranks with many candidates across batches", + "handles larger candidate sets (stress)", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, @@ -313,7 +309,7 @@ describe("reranker integration", () => { ); test.concurrent( - "explain=true stays aligned with items across batching", + "explain=true stays aligned with items across batches", async () => { const intent = new Intent<{ key: string; summary: string }>({ key: (x) => x.key, From 206f5f720bfe54e6c185cbbe2e96e1784e47f9e1 Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Sun, 21 Dec 2025 23:47:51 -0800 Subject: [PATCH 09/12] feat(intent): add filter() and choice() APIs Add two new high-level Intent operations alongside rank(): - filter(query, candidates): boolean relevancy decisions per candidate; returns only relevant items while preserving input order, with optional per-item explanations. - choice(query, candidates): always returns exactly one winner; uses an enum-based selectedKey schema and a batched tournament strategy for large inputs, with optional explanation. Implementation includes new schema builders and prompt/message builders for the two operations, plus comprehensive unit coverage (100%) and stricter, deterministic real-wire integration assertions for rank/filter/choice behavior. --- src/intent.int.test.ts | 152 +++++++++--- src/intent.ts | 330 +++++++++++++++++++++++++- src/intent.unit.test.ts | 505 +++++++++++++++++++++++++++++++++++++++- src/messages.ts | 68 ++++++ src/schema.ts | 93 ++++++++ 5 files changed, 1116 insertions(+), 32 deletions(-) diff --git a/src/intent.int.test.ts b/src/intent.int.test.ts index 3f94ec5..f87400b 100644 --- a/src/intent.int.test.ts +++ b/src/intent.int.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from "vitest"; import { Intent } from "./intent"; -describe("reranker integration", () => { +describe("intent integration", () => { const defaultTimeoutMs = 10000; const scoreRange = { minScore: 0, maxScore: 10 }; @@ -22,8 +22,7 @@ describe("reranker integration", () => { { key: "Eiffel Tower", summary: "Directions to the tower" }, ]); - expect(out.length).toBeGreaterThanOrEqual(1); - expect(out[0]?.key).toBe("JS Arrays"); + expect(out.map((x) => x.key)).toEqual(["JS Arrays"]); }, 30000, ); @@ -35,8 +34,7 @@ describe("reranker integration", () => { const items = ["apple", "banana", "orange", "grape"]; const ranked = await intent.rank("citrus fruits", items); - expect(ranked.length).toBe(1); - expect(ranked.includes("orange")).toBe(true); + expect(ranked).toEqual(["orange"]); }, 30000, ); @@ -82,6 +80,7 @@ describe("reranker integration", () => { key: (x) => x.key, summary: (x) => x.summary, timeoutMs: defaultTimeoutMs, + // All of these candidates should be obviously relevant, so all should pass a 0 threshold. relevancyThreshold: 0, }); @@ -92,16 +91,8 @@ describe("reranker integration", () => { ]; const out = await intent.rank("JavaScript array sorting", input); - expect(out.length).toBeGreaterThanOrEqual(1); - const outputKeys = out.map((x) => x.key); - for (const k of outputKeys) { - expect(input.some((i) => i.key === k)).toBe(true); - } - - if (out.length === input.length) { - expect(outputKeys).toEqual(input.map((x) => x.key)); - } + expect(out.map((x) => x.key)).toEqual(["Guide A", "Guide B", "Guide C"]); }, 30000, ); @@ -125,8 +116,7 @@ describe("reranker integration", () => { { explain: true }, ); - expect(out.length).toBeGreaterThanOrEqual(1); - expect(out[0]?.item.key).toBe("JS Arrays"); + expect(out.map((x) => x.item.key)).toEqual(["JS Arrays"]); expect(typeof out[0]?.explanation).toBe("string"); expect(out[0]?.explanation.length).toBeGreaterThan(0); }, @@ -167,12 +157,11 @@ describe("reranker integration", () => { const out = await intent.rank("JavaScript array sorting", [ { key: "Café", summary: "Guía para ordenar arrays en JavaScript" }, - { key: "東京", summary: "JavaScriptで配列をソートする方法" }, + { key: "東京", summary: "A travel guide to Tokyo neighborhoods" }, { key: "Banana Bread", summary: "How to bake banana bread" }, ]); - expect(out.length).toBeGreaterThanOrEqual(1); - expect(["Café", "東京"].includes(out[0]?.key ?? "")).toBe(true); + expect(out.map((x) => x.key)).toEqual(["Café"]); }, 30000, ); @@ -200,9 +189,16 @@ describe("reranker integration", () => { key: "Key with spaces\n(and newline)", summary: "Examples of sorting arrays in JavaScript", }, + { + key: "Unrelated", + summary: "How to bake banana bread", + }, ]); - expect(out.length).toBeGreaterThanOrEqual(1); + expect(out.length).toBe(3); + expect(out[0]?.key).toBe("Array.sort() / comparator"); + expect(out.some((x) => x.key === "Stable sort: ties, order")).toBe(true); + expect(out.some((x) => x.key === "Key with spaces\n(and newline)")).toBe(true); }, 30000, ); @@ -223,8 +219,7 @@ describe("reranker integration", () => { { key: "Other", summary: "Banana bread recipe" }, ]); - expect(out.length).toBeGreaterThanOrEqual(1); - expect(out.some((x) => x.key === "Same")).toBe(true); + expect(out.map((x) => x.key)).toEqual(["Same", "Same"]); }, 30000, ); @@ -275,8 +270,12 @@ describe("reranker integration", () => { ]; const out = await intent.rank("JavaScript array sorting", candidates); - expect(out.length).toBeGreaterThanOrEqual(1); - expect(out.length).toBeLessThanOrEqual(candidates.length); + expect(out.map((x) => x.key)).toEqual([ + "Array.sort", + "Comparator", + "Stable sort", + "Quickstart", + ]); }, 60000, ); @@ -302,8 +301,8 @@ describe("reranker integration", () => { })); const out = await intent.rank("JavaScript array sorting", candidates); - expect(out.length).toBeGreaterThanOrEqual(1); - expect(out.length).toBeLessThanOrEqual(candidates.length); + expect(out.length).toBe(5); + expect(out.map((x) => x.key)).toEqual(["Doc 1", "Doc 7", "Doc 13", "Doc 19", "Doc 25"]); }, 120000, ); @@ -326,11 +325,12 @@ describe("reranker integration", () => { })); const out = await intent.rank("JavaScript array sorting", input, { explain: true }); - expect(out.length).toBeGreaterThanOrEqual(1); + expect(out.length).toBe(4); + expect(out.map((r) => r.item.key)).toEqual(["Item 1", "Item 4", "Item 7", "Item 10"]); for (const r of out) { - expect(input.some((x) => x.key === r.item.key)).toBe(true); expect(typeof r.explanation).toBe("string"); + expect(r.explanation.length).toBeGreaterThan(0); } }, 120000, @@ -355,8 +355,102 @@ describe("reranker integration", () => { { userId: "integration-test-user" }, ); - expect(out.length).toBeGreaterThanOrEqual(1); + expect(out.map((x) => x.key)).toEqual(["JS Arrays"]); + }, + 30000, + ); + + test.concurrent( + "filter() returns only relevant items and preserves input order", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + }); + + const input = [ + { key: "A", summary: "How to bake banana bread" }, + { key: "B", summary: "Guide to sorting arrays in JavaScript" }, + { key: "C", summary: "JavaScript Array.prototype.sort examples" }, + { key: "D", summary: "History of the Eiffel Tower" }, + ]; + + const out = await intent.filter("JavaScript array sorting", input); + expect(out.map((x) => x.key)).toEqual(["B", "C"]); }, 30000, ); + + test.concurrent( + "filter({ explain: true }) returns aligned explanations", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + }); + + const input = [ + { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, + { key: "Banana Bread", summary: "How to bake banana bread" }, + ]; + + const out = await intent.filter("JavaScript array sorting", input, { explain: true }); + expect(out.length).toBe(1); + expect(out[0]?.item.key).toBe("JS Arrays"); + expect(typeof out[0]?.explanation).toBe("string"); + expect(out[0]?.explanation.length).toBeGreaterThan(0); + }, + 30000, + ); + + test.concurrent( + "choice() returns exactly one item from the input", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + batchSize: 2, + tinyBatchFraction: 0, + }); + + const input = [ + { key: "Banana Bread", summary: "How to bake banana bread" }, + { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, + { key: "Eiffel Tower", summary: "History of the Eiffel Tower" }, + { key: "Comparator", summary: "How to write a comparator function in JS" }, + ]; + + const winner = await intent.choice("JavaScript array sorting", input); + expect(winner.key).toBe("JS Arrays"); + }, + 60000, + ); + + test.concurrent( + "choice({ explain: true }) returns { item, explanation }", + async () => { + const intent = new Intent<{ key: string; summary: string }>({ + key: (x) => x.key, + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + batchSize: 2, + tinyBatchFraction: 0, + }); + + const input = [ + { key: "Banana Bread", summary: "How to bake banana bread" }, + { key: "JS Arrays", summary: "Guide to sorting arrays in JavaScript" }, + { key: "Comparator", summary: "How to write a comparator function in JS" }, + ]; + + const res = await intent.choice("JavaScript array sorting", input, { explain: true }); + expect(res.item.key).toBe("JS Arrays"); + expect(typeof res.explanation).toBe("string"); + expect(res.explanation.length).toBeGreaterThan(0); + }, + 60000, + ); }); diff --git a/src/intent.ts b/src/intent.ts index 9d437f1..4578c39 100644 --- a/src/intent.ts +++ b/src/intent.ts @@ -3,8 +3,8 @@ import { CONFIG } from "./config"; import { DEFAULT_KEY_EXTRACTOR, DEFAULT_SUMMARY_EXTRACTOR } from "./extractors"; import { clamp } from "./lib/number"; import { selectLlmClient } from "./llm_client"; -import { buildMessages } from "./messages"; -import { buildRelevancySchema } from "./schema"; +import { buildChoiceMessages, buildFilterMessages, buildMessages } from "./messages"; +import { buildChoiceSchema, buildFilterSchema, buildRelevancySchema } from "./schema"; import type { ChatMessage, @@ -325,6 +325,158 @@ export class Intent { } } + /** + * Filter candidates based on relevance to a query. + * + * Calls the LLM to decide whether each candidate is relevant, then returns + * only the relevant items in the same order they were provided. + * + * Explanations are only returned when `options.explain` is true. + * + * Fast paths: + * - Returns [] for 0 candidates without LLM call + * - Returns the single candidate unchanged without LLM call + * + * Error handling: + * - On any batch error, preserves that batch's original order + * - On top-level error, preserves original order + * + * @param query - The search query or user intent to filter against + * @param candidates - Array of items to filter + * @param options - Optional per-call configuration + * @param options.explain - When true, return `{ item, explanation }[]` instead of `T[]` + * @param options.userId - Optional user ID for this specific call, overrides ctx.userId + * @returns Filtered array of items, preserving input order + */ + public async filter( + query: string, + candidates: T[], + options?: { explain?: false; userId?: string }, + ): Promise; + + public async filter( + query: string, + candidates: T[], + options: { explain: true; userId?: string }, + ): Promise>; + + public async filter( + query: string, + candidates: T[], + options?: { explain?: boolean; userId?: string }, + ): Promise> { + if (candidates.length === 0) { + return []; + } + + if (candidates.length === 1) { + if (options?.explain) { + const [firstCandidate] = candidates; + return [{ item: firstCandidate!, explanation: "" }]; + } + return candidates; + } + + const prepared = this.prepareCandidates(candidates); + + const filteredWithExplanations = await batchProcess( + prepared, + this.cfg.batchSize, + this.cfg.tinyBatchFraction, + async (batch) => + (await this.processFilterBatch( + query, + batch, + options?.userId !== undefined ? { userId: options.userId } : undefined, + )) as Array<{ item: T; explanation: string }>, + this.ctx.logger, + (batch) => batch.map(({ item }) => ({ item, explanation: "" })), + ); + + if (options?.explain) { + return filteredWithExplanations; + } + + return filteredWithExplanations.map(({ item }) => item); + } + + /** + * Choose exactly one candidate as the best match for a query. + * + * Uses a tournament strategy when inputs exceed the batch size: + * - Choose one from each batch + * - Then choose one from the batch winners + * + * This method always returns a single item. + * + * @param query - The search query or user intent + * @param candidates - Array of items to choose from + * @param options - Optional per-call configuration + * @param options.explain - When true, return `{ item, explanation }` instead of `T` + * @param options.userId - Optional user ID for this specific call, overrides ctx.userId + * @returns The single chosen item (or item + explanation) + */ + public async choice( + query: string, + candidates: T[], + options?: { explain?: false; userId?: string }, + ): Promise; + + public async choice( + query: string, + candidates: T[], + options: { explain: true; userId?: string }, + ): Promise<{ item: T; explanation: string }>; + + public async choice( + query: string, + candidates: T[], + options?: { explain?: boolean; userId?: string }, + ): Promise { + if (candidates.length === 0) { + throw new Error("intent: choice requires at least one candidate"); + } + + if (candidates.length === 1) { + const [firstCandidate] = candidates; + if (options?.explain) { + return { item: firstCandidate!, explanation: "" }; + } + return firstCandidate!; + } + + const prepared = this.prepareCandidates(candidates); + const keyed = this.ensureUniqueKeys(prepared); + + const winners = await batchProcess( + keyed, + this.cfg.batchSize, + this.cfg.tinyBatchFraction, + async (batch) => [await this.processChoiceBatch(query, batch, options?.userId)], + this.ctx.logger, + (batch) => [{ item: batch[0]!.item, explanation: "" }], + ); + + // batchProcess guarantees at least one winner via its per-batch fallback. + + if (winners.length === 1) { + const [onlyWinner] = winners; + if (options?.explain) { + return onlyWinner!; + } + return onlyWinner!.item; + } + + const preparedFinalists = this.prepareCandidates(winners.map((w) => w.item)); + const keyedFinalists = this.ensureUniqueKeys(preparedFinalists); + + const final = await this.processChoiceBatch(query, keyedFinalists, options?.userId); + if (options?.explain) { + return final; + } + return final.item; + } + /** * Normalize incoming items into a consistent shape for downstream processing. * @@ -396,6 +548,42 @@ export class Intent { return { schema, messages }; } + /** + * Build the JSON schema and chat messages payload for the LLM filter call. + * + * @param query - The search query to evaluate candidates against + * @param items - Candidates with unique keys and summaries + * @returns Object containing JSON schema and chat messages array + * @private + */ + private buildFilterRequest( + query: string, + items: Array<{ key: string; summary: string }>, + ): { schema: JSONObject; messages: ChatMessage[] } { + const keys = items.map((x) => x.key); + const schema: JSONObject = buildFilterSchema(keys); + const messages = buildFilterMessages(query, items); + return { schema, messages }; + } + + /** + * Build the JSON schema and chat messages payload for the LLM choice call. + * + * @param query - The search query to choose against + * @param items - Candidates with unique keys and summaries + * @returns Object containing JSON schema and chat messages array + * @private + */ + private buildChoiceRequest( + query: string, + items: Array<{ key: string; summary: string }>, + ): { schema: JSONObject; messages: ChatMessage[] } { + const keys = items.map((x) => x.key); + const schema: JSONObject = buildChoiceSchema(keys); + const messages = buildChoiceMessages(query, items); + return { schema, messages }; + } + /** * Invoke the LLM and return the parsed map of candidate scores. * @@ -429,6 +617,144 @@ export class Intent { return data as Record; } + /** + * Invoke the LLM and return boolean relevancy decisions. + * + * @param messages - Chat messages to send + * @param schema - Strict JSON schema defining expected response structure + * @param userId - Optional user id + * @returns Map of candidate keys to filter decisions, or null if invalid + * @private + */ + private async fetchFilterDecisions( + messages: ChatMessage[], + schema: JSONObject, + userId?: string, + ): Promise | null> { + const config: LlmCallConfig = { + model: this.resolveModel(), + temperature: 0, + timeoutMs: this.cfg.timeoutMs, + }; + const { data } = await this.llm.call< + Record + >(messages, schema, config, userId ?? this.ctx.userId); + + if (data == null || typeof data !== "object") return null; + return data as Record; + } + + /** + * Invoke the LLM and return a single selected key. + * + * @param messages - Chat messages to send + * @param schema - Strict JSON schema defining expected response structure + * @param userId - Optional user id + * @returns Choice result, or null if invalid + * @private + */ + private async fetchChoice( + messages: ChatMessage[], + schema: JSONObject, + userId?: string, + ): Promise<{ explanation: string; selectedKey: string } | null> { + const config: LlmCallConfig = { + model: this.resolveModel(), + temperature: 0, + timeoutMs: this.cfg.timeoutMs, + }; + const { data } = await this.llm.call<{ explanation: string; selectedKey: string }>( + messages, + schema, + config, + userId ?? this.ctx.userId, + ); + + if (data == null || typeof data !== "object") return null; + const record = data as Record; + const explanation = typeof record.explanation === "string" ? record.explanation : ""; + const selectedKey = typeof record.selectedKey === "string" ? record.selectedKey : ""; + if (selectedKey === "") return null; + return { explanation, selectedKey }; + } + + /** + * Apply boolean filter decisions while preserving input order. + * + * @param items - Candidates with unique keys + * @param decisions - Map of candidate keys to decisions + * @returns Filtered items with explanations + * @private + */ + private applyFilter( + items: Array<{ item: T; idx: number; key: string; summary: string }>, + decisions: Record, + ): Array<{ item: T; explanation: string }> { + const kept: Array<{ item: T; explanation: string }> = []; + for (const it of items) { + const decision = decisions[it.key]; + if (decision?.isRelevant === true) { + kept.push({ + item: it.item, + explanation: typeof decision.explanation === "string" ? decision.explanation : "", + }); + } + } + return kept; + } + + /** + * Process a single batch of candidates through the LLM filter. + * + * @param query - Query to filter against + * @param batch - Prepared candidates + * @param options - Optional userId override + * @returns Filtered items (stable order), with explanations + * @private + */ + private async processFilterBatch( + query: string, + batch: Array<{ item: T; idx: number; baseKey: string; summary: string }>, + options?: { userId?: string }, + ): Promise> { + const keyed = this.ensureUniqueKeys(batch); + const { schema, messages } = this.buildFilterRequest(query, keyed); + const decisions = await this.fetchFilterDecisions(messages, schema, options?.userId); + if (decisions == null) { + return keyed.map(({ item }) => ({ item, explanation: "" })); + } + + return this.applyFilter(keyed, decisions); + } + + /** + * Process a batch of candidates and choose a single winner. + * + * @param query - Query to choose against + * @param batch - Candidates with unique keys + * @param userId - Optional userId override + * @returns Winner item with explanation + * @private + */ + private async processChoiceBatch( + query: string, + batch: Array<{ item: T; idx: number; key: string; summary: string }>, + userId?: string, + ): Promise<{ item: T; explanation: string }> { + const { schema, messages } = this.buildChoiceRequest(query, batch); + const choice = await this.fetchChoice(messages, schema, userId); + if (choice == null) { + return { item: batch[0]!.item, explanation: "" }; + } + + const winner = batch.find((x) => x.key === choice.selectedKey); + if (!winner) { + return { item: batch[0]!.item, explanation: choice.explanation }; + } + + return { item: winner.item, explanation: choice.explanation }; + } + /** * Apply relevancy threshold filtering and stable sorting. * diff --git a/src/intent.unit.test.ts b/src/intent.unit.test.ts index 1173095..1865fe4 100644 --- a/src/intent.unit.test.ts +++ b/src/intent.unit.test.ts @@ -52,7 +52,7 @@ describe("Intent.rank", () => { }); test("candidate evaluation schema defines explanation before score", () => { - const schema = buildCandidateEvaluationSchema(0, 10); + const schema = buildCandidateEvaluationSchema(); expect(Object.keys(schema.properties)).toEqual(["explanation", "score"]); }); @@ -616,3 +616,506 @@ describe("Intent.rank", () => { expect(out.map((c) => c.summary)).toEqual(["S0", "S1"]); }); }); + +describe("Intent.filter", () => { + test("returns empty list for zero candidates", async () => { + const ctx = makeCtx(); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const res = await intent.filter("query", []); + expect(res).toEqual([]); + expect(ctx.llm.call).not.toHaveBeenCalled(); + }); + + test("returns input unchanged for single candidate (no LLM call)", async () => { + const ctx = makeCtx(); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const input = [{ key: "Only", summary: "s" }]; + const res = await intent.filter("query", input); + expect(res).toEqual(input); + expect(ctx.llm.call).not.toHaveBeenCalled(); + }); + + test("with explain=true returns explanation wrapper for single candidate (no LLM call)", async () => { + const ctx = makeCtx(); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const input = [{ key: "Only", summary: "s" }]; + const res = await intent.filter("query", input, { explain: true }); + expect(res).toEqual([{ item: input[0], explanation: "" }]); + expect(ctx.llm.call).not.toHaveBeenCalled(); + }); + + test("filters by boolean decision and preserves input order", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { + A: { explanation: "yes", isRelevant: true }, + B: { explanation: "no", isRelevant: false }, + C: { explanation: "yes", isRelevant: true }, + }, + }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + { key: "C", summary: "" }, + ]; + + const res = await intent.filter("query", input); + expect(res.map((x) => x.key)).toEqual(["A", "C"]); + }); + + test("explain=true returns explanations for kept items", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { + A: { explanation: 123, isRelevant: true }, + B: { explanation: "no", isRelevant: false }, + }, + }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.filter("query", input, { explain: true }); + expect(res).toEqual([{ item: input[0], explanation: "" }]); + }); + + test("returns original list on error and logs warning", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockRejectedValueOnce(new Error("boom")); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.filter("query", input); + expect(res).toEqual(input); + expect(ctx.logger.warn).toHaveBeenCalled(); + }); + + test("on error returns wrapped input when explain=true", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockImplementationOnce(() => Promise.reject(new Error("boom"))); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + + const res = await intent.filter("query", input, { explain: true }); + expect(res).toEqual([ + { item: input[0], explanation: "" }, + { item: input[1], explanation: "" }, + ]); + expect(ctx.logger.warn).toHaveBeenCalled(); + }); + + test("returns original list when LLM response is null", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ data: null }); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.filter("query", input); + expect(res).toEqual(input); + }); + + test("passes userId override through filter requests", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { + A: { explanation: "", isRelevant: true }, + B: { explanation: "", isRelevant: false }, + }, + }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + + await intent.filter( + "query", + [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ], + { userId: "call-user" }, + ); + + const call = (ctx.llm.call as any).mock.calls[0]; + expect(call[3]).toBe("call-user"); + }); + + test("with explain=true returns wrapped input on top-level error", async () => { + const ctx = makeCtx({ llm: undefined }); + const configOverride = { + ...CONFIG, + GROQ: { ...CONFIG.GROQ, API_KEY: "fake" }, + } as typeof CONFIG; + + const intent = new Intent({ + ...ctx, + config: configOverride, + key: (c) => c.key, + summary: (c) => c.summary, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.filter("query", input, { explain: true }); + expect(res).toEqual([ + { item: input[0], explanation: "" }, + { item: input[1], explanation: "" }, + ]); + expect(ctx.logger.warn).toHaveBeenCalled(); + }); +}); + +describe("Intent.choice", () => { + test("throws for zero candidates", async () => { + const ctx = makeCtx(); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + + await expect(intent.choice("query", [])).rejects.toThrow(/requires at least one candidate/); + }); + + test("returns single candidate unchanged (no LLM call)", async () => { + const ctx = makeCtx(); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + const input = [{ key: "Only", summary: "s" }]; + const res = await intent.choice("query", input); + expect(res).toEqual(input[0]); + expect(ctx.llm.call).not.toHaveBeenCalled(); + }); + + test("explain=true returns explanation wrapper for single candidate (no LLM call)", async () => { + const ctx = makeCtx(); + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + }); + + const input = [{ key: "Only", summary: "s" }]; + const res = await intent.choice("query", input, { explain: true }); + expect(res).toEqual({ item: input[0], explanation: "" }); + expect(ctx.llm.call).not.toHaveBeenCalled(); + }); + + test("returns chosen item based on selectedKey", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ data: { selectedKey: "B" } }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 10, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + { key: "C", summary: "" }, + ]; + const res = await intent.choice("query", input); + expect(res.key).toBe("B"); + }); + + test("falls back to first item when LLM returns invalid data", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ data: null }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 10, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.choice("query", input); + expect(res).toEqual(input[0]); + }); + + test("falls back to first item when selectedKey is not in the batch", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { explanation: "x", selectedKey: "Z" }, + }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 10, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.choice("query", input); + expect(res).toEqual(input[0]); + }); + + test("explain=true returns chosen item with explanation", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { explanation: "best", selectedKey: "A" }, + }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 10, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.choice("query", input, { explain: true }); + expect(res).toEqual({ item: input[0], explanation: "best" }); + }); + + test("uses a tournament strategy when candidates exceed batchSize", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any) + .mockResolvedValueOnce({ data: { explanation: "b1", selectedKey: "K1" } }) + .mockResolvedValueOnce({ data: { explanation: "b2", selectedKey: "K3" } }) + .mockResolvedValueOnce({ data: { explanation: "final", selectedKey: "K3" } }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 2, + }); + + const input: IntentCandidate[] = [ + { key: "K0", summary: "" }, + { key: "K1", summary: "" }, + { key: "K2", summary: "" }, + { key: "K3", summary: "" }, + ]; + + const res = await intent.choice("query", input); + expect(res.key).toBe("K3"); + expect((ctx.llm.call as any).mock.calls.length).toBe(3); + }); + + test("does not run a final round when there is only one batch winner", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { explanation: "b", selectedKey: "K1" }, + }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 10, + }); + + const input: IntentCandidate[] = [ + { key: "K0", summary: "" }, + { key: "K1", summary: "" }, + { key: "K2", summary: "" }, + ]; + + const res = await intent.choice("query", input); + expect(res.key).toBe("K1"); + expect((ctx.llm.call as any).mock.calls.length).toBe(1); + }); + + test("with explain=true returns the single batch winner without running a final round", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { explanation: "b", selectedKey: "K1" }, + }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 10, + }); + + const input: IntentCandidate[] = [ + { key: "K0", summary: "" }, + { key: "K1", summary: "" }, + { key: "K2", summary: "" }, + ]; + + const res = await intent.choice("query", input, { explain: true }); + expect(res).toEqual({ item: input[1], explanation: "b" }); + expect((ctx.llm.call as any).mock.calls.length).toBe(1); + }); + + test("with explain=true returns wrapped first item on error", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockImplementationOnce(() => Promise.reject(new Error("boom"))); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 10, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + + const res = await intent.choice("query", input, { explain: true }); + expect(res).toEqual({ item: input[0], explanation: "" }); + expect(ctx.logger.warn).toHaveBeenCalled(); + }); + + test("passes userId override through choice requests", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ + data: { explanation: "x", selectedKey: "A" }, + }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 10, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + + await intent.choice("query", input, { userId: "call-user" }); + const call = (ctx.llm.call as any).mock.calls[0]; + expect(call[3]).toBe("call-user"); + }); + + test("runs a final round when there are multiple batch winners", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any) + .mockResolvedValueOnce({ data: { selectedKey: "K1" } }) + .mockResolvedValueOnce({ data: { selectedKey: "K2" } }) + .mockResolvedValueOnce({ data: { selectedKey: "K2" } }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 2, + tinyBatchFraction: 0, + }); + + const input: IntentCandidate[] = [ + { key: "K0", summary: "" }, + { key: "K1", summary: "" }, + { key: "K2", summary: "" }, + { key: "K3", summary: "" }, + ]; + + const res = await intent.choice("query", input, { explain: true }); + expect(res.item.key).toBe("K2"); + expect((ctx.llm.call as any).mock.calls.length).toBe(3); + }); + + test("returns the first item when there are no batch winners", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockResolvedValueOnce({ data: { selectedKey: 123 } }); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 10, + }); + + const input: IntentCandidate[] = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + + const res = await intent.choice("query", input); + expect(res).toEqual(input[0]); + }); + + test("on error returns the first item and logs warning", async () => { + const ctx = makeCtx(); + (ctx.llm.call as any).mockImplementationOnce(() => Promise.reject(new Error("boom"))); + + const intent = new Intent({ + ...ctx, + key: (c) => c.key, + summary: (c) => c.summary, + batchSize: 10, + }); + + const input = [ + { key: "A", summary: "" }, + { key: "B", summary: "" }, + ]; + const res = await intent.choice("query", input); + expect(res).toEqual(input[0]); + expect(ctx.logger.warn).toHaveBeenCalled(); + }); +}); diff --git a/src/messages.ts b/src/messages.ts index 3b352bc..961995f 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -53,3 +53,71 @@ Pretty-print the JSON for readability.`; { role: "user", content: jsonStringify(payload) }, ]; } + +/** + * Build system + user messages instructing the model to filter candidates. + * + * The model must output a JSON object mapping each candidate key to: + * Example: {"Some key": {"explanation": "...", "isRelevant": true}}. + * + * @param query - The search query or user intent + * @param candidates - Array of candidates with keys and summaries + * @returns Array of chat messages ready for LLM consumption + */ +export function buildFilterMessages(query: string, candidates: IntentCandidate[]): ChatMessage[] { + const system = `The user will provide a short description of a query they are trying to automate, along with a JSON blob containing candidate_search_results. Your task is to assess each candidate and return a JSON object that maps candidate keys to objects of the form {"explanation": string, "isRelevant": boolean}. + +Return isRelevant=true only when the candidate clearly helps satisfy the query intent. Otherwise return isRelevant=false. + +Every candidate MUST include an explanation. Write the explanation first, then the boolean. The explanation should be concise (1-3 sentences), concrete, and reference the query intent and the candidate summary. + +Every key in candidate_search_results must be present in your output mapping. Do not add any keys that are not present in candidate_search_results. +Every key in candidate_search_results must map to an object with: +- explanation: string +- isRelevant: boolean +Do not include anything other than the mapping JSON object. Return only JSON matching the enforced schema. + +Pretty-print the JSON for readability.`; + + const payload = { + query, + candidate_search_results: candidates.map((c) => ({ key: c.key, summary: c.summary })), + } as const; + + return [ + { role: "system", content: system }, + { role: "user", content: jsonStringify(payload) }, + ]; +} + +/** + * Build system + user messages instructing the model to choose exactly one candidate. + * + * The model must output a JSON object of the form: + * Example: {"explanation": "...", "selectedKey": "Some key"}. + * + * @param query - The search query or user intent + * @param candidates - Array of candidates with keys and summaries + * @returns Array of chat messages ready for LLM consumption + */ +export function buildChoiceMessages(query: string, candidates: IntentCandidate[]): ChatMessage[] { + const system = `The user will provide a short description of a query they are trying to automate, along with a JSON blob containing candidate_search_results. Your task is to choose exactly one candidate as the best match for the query. + +You MUST choose one candidate key from the provided list. Do not choose multiple. + +Return ONLY JSON of the form: {"explanation": string, "selectedKey": string} where selectedKey is exactly one of the candidate keys. The explanation should be concise (1-3 sentences), concrete, and reference the query intent and the candidate summary. + +Do not include any other text outside the JSON. Return only JSON matching the enforced schema. + +Pretty-print the JSON for readability.`; + + const payload = { + query, + candidate_search_results: candidates.map((c) => ({ key: c.key, summary: c.summary })), + } as const; + + return [ + { role: "system", content: system }, + { role: "user", content: jsonStringify(payload) }, + ]; +} diff --git a/src/schema.ts b/src/schema.ts index 239ef85..df3e5ef 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -2,6 +2,7 @@ import type { JSONObject } from "./types"; type IntegerSchema = { type: "integer" }; type StringSchema = { type: "string" }; +type BooleanSchema = { type: "boolean" }; type CandidateEvaluationSchema = { type: "object"; @@ -13,6 +14,26 @@ type CandidateEvaluationSchema = { additionalProperties: false; }; +type CandidateFilterSchema = { + type: "object"; + properties: { + explanation: StringSchema; + isRelevant: BooleanSchema; + }; + required: ["explanation", "isRelevant"]; + additionalProperties: false; +}; + +type ChoiceSchema = { + type: "object"; + properties: { + explanation: StringSchema; + selectedKey: { type: "string"; enum: string[] }; + }; + required: ["explanation", "selectedKey"]; + additionalProperties: false; +}; + /** * Build the schema used for a single candidate's evaluation. * @@ -33,6 +54,25 @@ export function buildCandidateEvaluationSchema(): CandidateEvaluationSchema { }; } +/** + * Build the schema used for a single candidate's filter decision. + * + * Property order is intentional: `explanation` is defined before `isRelevant`. + * + * @returns JSON schema for a single candidate filter decision + */ +export function buildCandidateFilterSchema(): CandidateFilterSchema { + return { + type: "object", + properties: { + explanation: { type: "string" }, + isRelevant: { type: "boolean" }, + }, + required: ["explanation", "isRelevant"], + additionalProperties: false, + }; +} + /** * Build a strict JSON schema mapping candidate keys to evaluation objects. * @@ -67,3 +107,56 @@ export function buildRelevancySchema( additionalProperties: false, } as JSONObject; } + +/** + * Build a strict JSON schema mapping candidate keys to boolean relevancy decisions. + * + * Each candidate key maps to an object containing: + * - explanation: a short justification + * - isRelevant: boolean + * + * @param keys - Array of unique candidate keys + * @returns JSON schema object enforcing exact structure of response + */ +export function buildFilterSchema(keys: string[]): JSONObject { + const decisionSchema = buildCandidateFilterSchema(); + + const properties: Record = {}; + for (const k of keys) { + properties[k] = decisionSchema; + } + + return { + title: "Query / Candidate Relevancy Filter", + description: + "Map candidate results for a search query to boolean relevancy decisions with explanations.", + type: "object", + properties, + required: keys, + additionalProperties: false, + } as JSONObject; +} + +/** + * Build a strict JSON schema for choosing a single candidate key. + * + * The model must return: + * - explanation: string + * - selectedKey: one of the provided candidate keys (enum) + * + * @param keys - Array of unique candidate keys + * @returns JSON schema enforcing a single selected key + */ +export function buildChoiceSchema(keys: string[]): JSONObject { + return { + title: "Query / Candidate Single Choice", + description: "Choose exactly one candidate key for the query and explain why.", + type: "object", + properties: { + explanation: { type: "string" }, + selectedKey: { type: "string", enum: keys }, + }, + required: ["explanation", "selectedKey"], + additionalProperties: false, + } as ChoiceSchema as JSONObject; +} From 8e186481d679fb39a6450c9879486ba8416ee440 Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Mon, 22 Dec 2025 00:15:46 -0800 Subject: [PATCH 10/12] feat(groq): replace temperature with reasoningEffort Replace the temperature-based tuning surface with Groq's reasoning_effort control to better match reasoning-style models. - Add GROQ_DEFAULT_REASONING_EFFORT (low|medium|high, default: medium) and remove GROQ_DEFAULT_TEMPERATURE - Update LlmCallConfig to accept reasoningEffort and plumb it through Intent + selectLlmClient - Update Groq provider request building to send reasoning_effort (while preserving strict JSON schema response_format) - Update unit/integration tests for config parsing and request shape changes Backwards compatibility is intentionally not preserved. --- src/config.ts | 9 ++++++--- src/intent.int.test.ts | 18 +++--------------- src/intent.ts | 6 +++--- src/lib/config.ts | 2 +- src/lib/config.unit.test.ts | 8 ++++---- src/llm_client.ts | 2 +- src/llm_client.unit.test.ts | 8 ++++---- src/providers/groq.int.test.ts | 2 +- src/providers/groq.ts | 18 +++++++++--------- src/providers/groq.unit.test.ts | 6 +++--- src/types.ts | 2 +- 11 files changed, 36 insertions(+), 45 deletions(-) diff --git a/src/config.ts b/src/config.ts index 746407a..0563eda 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,5 @@ import "dotenv/config"; -import { enumString, int, number, string } from "./lib/config"; +import { enumeration, int, number, string } from "./lib/config"; /** * Exported config object (no function call required) following API patterns. @@ -8,11 +8,14 @@ export const CONFIG = { GROQ: { API_KEY: string("GROQ_API_KEY", { default: "" }), DEFAULT_MODEL: string("GROQ_DEFAULT_MODEL", { default: "openai/gpt-oss-20b" }), - DEFAULT_TEMPERATURE: number("GROQ_DEFAULT_TEMPERATURE", { default: 0, min: 0, max: 1 }), + DEFAULT_REASONING_EFFORT: enumeration("GROQ_DEFAULT_REASONING_EFFORT", { + default: "medium", + values: ["low", "medium", "high"] as const, + }), JSON_REPAIR_ATTEMPTS: int("GROQ_JSON_REPAIR_ATTEMPTS", { default: 3, min: 0 }), }, INTENT: { - PROVIDER: enumString("INTENT_PROVIDER", { default: "GROQ", values: ["GROQ"] as const }), + PROVIDER: enumeration("INTENT_PROVIDER", { default: "GROQ", values: ["GROQ"] as const }), TIMEOUT_MS: int("INTENT_TIMEOUT_MS", { default: 3000, min: 1 }), MIN_SCORE: int("INTENT_MIN_SCORE", { default: 0 }), MAX_SCORE: int("INTENT_MAX_SCORE", { default: 10, min: 1 }), diff --git a/src/intent.int.test.ts b/src/intent.int.test.ts index f87400b..5e8e4b0 100644 --- a/src/intent.int.test.ts +++ b/src/intent.int.test.ts @@ -39,18 +39,6 @@ describe("intent integration", () => { 30000, ); - test.concurrent( - "supports defaults: new Intent() accepts arbitrary queries", - async () => { - const intent = new Intent(); - const items = ["tiny", "small", "medium", "large", "huge"]; - - const ranked = await intent.rank("size", items); - expect(ranked).toEqual(["tiny", "small", "medium", "large", "huge"]); - }, - 30000, - ); - test.concurrent( "returns [] when all candidates are unrelated and threshold > minScore", async () => { @@ -90,9 +78,9 @@ describe("intent integration", () => { { key: "Guide C", summary: "JavaScript array sorting examples and best practices" }, ]; - const out = await intent.rank("JavaScript array sorting", input); + const out = await intent.filter("JavaScript array sorting", input); - expect(out.map((x) => x.key)).toEqual(["Guide A", "Guide B", "Guide C"]); + expect(out).toEqual(input); }, 30000, ); @@ -266,7 +254,7 @@ describe("intent integration", () => { { key: "Quickstart", summary: "Basic JS arrays tutorial" }, { key: "Banana bread", summary: "Recipe for banana bread" }, { key: "Eiffel Tower", summary: "History of the Eiffel Tower" }, - { key: "Node streams", summary: "Guide to Node.js streams" }, + { key: "Moon Bases", summary: "Guide to living on the moon" }, ]; const out = await intent.rank("JavaScript array sorting", candidates); diff --git a/src/intent.ts b/src/intent.ts index 4578c39..35b2151 100644 --- a/src/intent.ts +++ b/src/intent.ts @@ -603,7 +603,7 @@ export class Intent { ): Promise | null> { const config: LlmCallConfig = { model: this.resolveModel(), - temperature: 0, + reasoningEffort: "medium", timeoutMs: this.cfg.timeoutMs, }; const { data } = await this.llm.call>( @@ -633,7 +633,7 @@ export class Intent { ): Promise | null> { const config: LlmCallConfig = { model: this.resolveModel(), - temperature: 0, + reasoningEffort: "medium", timeoutMs: this.cfg.timeoutMs, }; const { data } = await this.llm.call< @@ -660,7 +660,7 @@ export class Intent { ): Promise<{ explanation: string; selectedKey: string } | null> { const config: LlmCallConfig = { model: this.resolveModel(), - temperature: 0, + reasoningEffort: "medium", timeoutMs: this.cfg.timeoutMs, }; const { data } = await this.llm.call<{ explanation: string; selectedKey: string }>( diff --git a/src/lib/config.ts b/src/lib/config.ts index cec1ff8..628b8ac 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -36,7 +36,7 @@ export function string(name: string, opts?: BaseOptions): string { * When set, validates the value is included in opts.values. * When unset, returns opts.default if provided, otherwise throws. */ -export function enumString( +export function enumeration( name: string, opts: EnumOptions, ): T[number] { diff --git a/src/lib/config.unit.test.ts b/src/lib/config.unit.test.ts index c3a46b4..7c70401 100644 --- a/src/lib/config.unit.test.ts +++ b/src/lib/config.unit.test.ts @@ -138,20 +138,20 @@ describe("lib/config", () => { }); }); - describe("enumString()", () => { + describe("enumeration()", () => { it("returns the env value when it is included in values", () => { process.env[VAR] = "GROQ"; - expect(cfg.enumString(VAR, { values: ["GROQ"] as const })).toBe("GROQ"); + expect(cfg.enumeration(VAR, { values: ["GROQ"] as const })).toBe("GROQ"); }); it("returns default when missing", () => { unset(VAR); - expect(cfg.enumString(VAR, { default: "GROQ", values: ["GROQ"] as const })).toBe("GROQ"); + expect(cfg.enumeration(VAR, { default: "GROQ", values: ["GROQ"] as const })).toBe("GROQ"); }); it("throws when the value is not included in values", () => { process.env[VAR] = "NOPE"; - expect(() => cfg.enumString(VAR, { values: ["GROQ"] as const })).toThrow( + expect(() => cfg.enumeration(VAR, { values: ["GROQ"] as const })).toThrow( /must be one of: GROQ/, ); }); diff --git a/src/llm_client.ts b/src/llm_client.ts index d3d813c..45a8d9e 100644 --- a/src/llm_client.ts +++ b/src/llm_client.ts @@ -26,7 +26,7 @@ export function selectLlmClient( return createDefaultGroqClient(groqKey, { defaults: { model: config.GROQ.DEFAULT_MODEL, - temperature: config.GROQ.DEFAULT_TEMPERATURE, + reasoningEffort: config.GROQ.DEFAULT_REASONING_EFFORT, }, }); } diff --git a/src/llm_client.unit.test.ts b/src/llm_client.unit.test.ts index 10d4688..f22ddb0 100644 --- a/src/llm_client.unit.test.ts +++ b/src/llm_client.unit.test.ts @@ -9,7 +9,7 @@ describe("selectLlmClient", () => { const selected = selectLlmClient( { llm } as any, { - GROQ: { API_KEY: "k", DEFAULT_MODEL: "m", DEFAULT_TEMPERATURE: 0 }, + GROQ: { API_KEY: "k", DEFAULT_MODEL: "m", DEFAULT_REASONING_EFFORT: "medium" }, } as any, ); expect(selected).toBe(llm); @@ -17,7 +17,7 @@ describe("selectLlmClient", () => { test("returns undefined when GROQ api key missing", () => { const selected = selectLlmClient({}, { - GROQ: { API_KEY: "", DEFAULT_MODEL: "m", DEFAULT_TEMPERATURE: 0 }, + GROQ: { API_KEY: "", DEFAULT_MODEL: "m", DEFAULT_REASONING_EFFORT: "medium" }, } as any); expect(selected).toBeUndefined(); }); @@ -28,7 +28,7 @@ describe("selectLlmClient", () => { .mockImplementation((apiKey, options) => { expect(apiKey).toBe("test-key"); expect(options?.defaults?.model).toBe("test-model"); - expect(options?.defaults?.temperature).toBe(0.12); + expect(options?.defaults?.reasoningEffort).toBe("high"); return { call: vi.fn(async (messages: any, schema: any) => ({ data: { A: 1 }, messages, schema })), @@ -39,7 +39,7 @@ describe("selectLlmClient", () => { GROQ: { API_KEY: "test-key", DEFAULT_MODEL: "test-model", - DEFAULT_TEMPERATURE: 0.12, + DEFAULT_REASONING_EFFORT: "high", }, }; diff --git a/src/providers/groq.int.test.ts b/src/providers/groq.int.test.ts index 256d6f6..bd95f4e 100644 --- a/src/providers/groq.int.test.ts +++ b/src/providers/groq.int.test.ts @@ -10,7 +10,7 @@ describe("groq provider integration", () => { test.concurrent("repairs server-side schema validation failures", async () => { const client = createDefaultGroqClient(CONFIG.GROQ.API_KEY, { jsonRepairAttempts: 3, - defaults: { temperature: 0 }, + defaults: { reasoningEffort: "medium" }, }); const schema = { diff --git a/src/providers/groq.ts b/src/providers/groq.ts index 56e48ae..e4d0d54 100644 --- a/src/providers/groq.ts +++ b/src/providers/groq.ts @@ -18,7 +18,7 @@ type GroqJsonSchemaResponseFormat = { type GroqChatCompletionRequest = { model: string; - temperature: number; + reasoning_effort: "low" | "medium" | "high"; messages: ChatCompletionMessageParam[]; user?: string; response_format: GroqJsonSchemaResponseFormat; @@ -77,13 +77,13 @@ function mapToGroqMessages(messages: ChatMessage[]): ChatCompletionMessageParam[ /** * Build the request payload expected by groq-sdk with strict JSON schema. * - * Constructs the complete request object including model, temperature, messages, + * Constructs the complete request object including model, reasoning effort, messages, * optional user ID, and the response_format configuration that enforces strict * JSON schema validation on the model's output. * * @param outputSchema - JSON schema defining expected response structure * @param groqMessages - Formatted chat messages - * @param config - Optional config overriding model/temperature + * @param config - Optional config overriding model and reasoning effort * @param userId - Optional user identifier for Groq's abuse monitoring * @returns Request payload ready for groq-sdk * @private @@ -93,11 +93,11 @@ function buildGroqRequest( groqMessages: ChatCompletionMessageParam[], config: LlmCallConfig | undefined, userId: string | undefined, - defaults: { model: string; temperature: number }, + defaults: { model: string; reasoningEffort: "low" | "medium" | "high" }, ): GroqChatCompletionRequest { return { model: config?.model ?? defaults.model, - temperature: config?.temperature ?? defaults.temperature, + reasoning_effort: config?.reasoningEffort ?? defaults.reasoningEffort, messages: groqMessages, ...(userId ? { user: userId } : {}), response_format: { @@ -326,7 +326,7 @@ function extractJsonValidateFailedRepairInput(err: unknown): JsonRepairInput | u * Returns an LlmClient implementation that uses the Groq SDK with: * - Strict JSON schema enforcement via response_format * - Automatic retry on schema validation failures (up to 3 attempts) - * - Support for custom model, temperature, timeout, and user ID + * - Support for custom model, reasoning effort, timeout, and user ID * * @param apiKey - Groq API key * @returns LlmClient implementation for Groq @@ -394,14 +394,14 @@ export function createGroqSdk(options: { apiKey: string }): unknown { export function createDefaultGroqClient( apiKey: string, options?: { - defaults?: { model?: string; temperature?: number }; + defaults?: { model?: string; reasoningEffort?: "low" | "medium" | "high" }; makeSdk?: (apiKey: string) => GroqSdkLike; jsonRepairAttempts?: number; }, ): LlmClient { const defaults = { model: options?.defaults?.model ?? CONFIG.GROQ.DEFAULT_MODEL, - temperature: options?.defaults?.temperature ?? CONFIG.GROQ.DEFAULT_TEMPERATURE, + reasoningEffort: options?.defaults?.reasoningEffort ?? CONFIG.GROQ.DEFAULT_REASONING_EFFORT, } as const; const makeSdk: GroqClientFactory = options?.makeSdk ?? createGroqSdkLike; const jsonRepairAttempts = options?.jsonRepairAttempts ?? CONFIG.GROQ.JSON_REPAIR_ATTEMPTS; @@ -418,7 +418,7 @@ export function createDefaultGroqClient( * * @param messages - Chat messages to send to the model * @param outputSchema - JSON schema defining expected response structure - * @param config - Optional model, temperature, and timeout overrides + * @param config - Optional model, reasoning effort, and timeout overrides * @param userId - Optional user ID for Groq's abuse monitoring * @returns Parsed response data wrapped in { data } object * @throws {Error} If all retry attempts fail or response is invalid diff --git a/src/providers/groq.unit.test.ts b/src/providers/groq.unit.test.ts index ef30184..6bfa9ea 100644 --- a/src/providers/groq.unit.test.ts +++ b/src/providers/groq.unit.test.ts @@ -59,13 +59,13 @@ describe("groq provider", () => { test("defaults come from options.defaults when provided", async () => { const callMock = vi.fn(async (req: any) => { expect(req.model).toBe("m1"); - expect(req.temperature).toBe(0.33); + expect(req.reasoning_effort).toBe("high"); return { choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], }; }); const client = GroqProvider.createDefaultGroqClient("k", { - defaults: { model: "m1", temperature: 0.33 }, + defaults: { model: "m1", reasoningEffort: "high" }, makeSdk: () => ({ chat: { completions: { create: callMock } } }), }); await client.call([{ role: "user", content: "{}" }], schema as any); @@ -89,7 +89,7 @@ describe("groq provider", () => { test("defaults fall back when options.defaults omitted", async () => { const callMock = vi.fn(async (req: any) => { expect(typeof req.model).toBe("string"); - expect(typeof req.temperature).toBe("number"); + expect(["low", "medium", "high"].includes(req.reasoning_effort)).toBe(true); return { choices: [{ message: { role: "assistant", content: JSON.stringify({ A: 1 }) } }], }; diff --git a/src/types.ts b/src/types.ts index e49b367..36b8af5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,7 +12,7 @@ export type ChatMessage = { export type LlmCallConfig = { model?: string; - temperature?: number; + reasoningEffort?: "low" | "medium" | "high"; timeoutMs?: number; }; From 89faeaf6ba57314287cd448f64f5f6229c0e86d9 Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Mon, 22 Dec 2025 01:08:21 -0800 Subject: [PATCH 11/12] docs(readme): quickstart + config clarity; refine explanations Update README to emphasize choice API, add a concise Quickstart example, and clarify configuration/performance knobs (relevancy threshold behavior, min/max score range, timeout, batching). Add an integration test that validates the README Quickstart example output. Refine system prompts for rank and choice to produce end-user-facing explanations (avoid prompt mechanics and first-person language). --- README.md | 423 +++++++++++++++++------------------------ src/intent.int.test.ts | 138 ++++++++++++++ src/messages.ts | 33 +++- 3 files changed, 347 insertions(+), 247 deletions(-) diff --git a/README.md b/README.md index dda690a..25fab27 100644 --- a/README.md +++ b/README.md @@ -1,342 +1,277 @@ # Intent -`intent` is an LLM-based reranker library that offers ranking, filtering, and selection all with explicit, inspectable reasoning. +`intent` is an LLM-based reranker library that offers ranking, filtering, and choice all with explicit, inspectable reasoning. Unlike black-box models, `intent` generates an explanation alongside every score. This transparency allows for easier debugging and enables you to surface reasoning directly to users. -## Usage +By being LLM powered, it also offers flexibility and dynamic tunability to +your ranking logic without needing to come up with custom embedding models or +retrain existing ones. -Intent is designed to be simple to use while remaining flexible. Start with the basics and add configuration as needed. +## Install -### Simplest Case: Rank Primitives +```bash +npm install intent +``` -When working with simple types like strings or numbers, just create an Intent and call `rank()`. Intent will use sensible defaults for everything: +## Quickstart -```typescript +```ts import { Intent } from "intent"; -const intent = new Intent(); -const items = ["apple", "banana", "orange", "grape"]; +const intent = new Intent({ relevancyThreshold: 1 }); + +const docs = [ + "Many network requests can fail intermittently due to transient issues.", + "To reduce flaky tests, add exponential backoff with jitter to HTTP retries.", + "Citrus fruits like oranges and lemons are high in vitamin C.", +]; -const ranked = await intent.rank("citrus fruits", items); -// -// Returns: ["orange"] (if threshold filters out non-citrus) +const ranked = await intent.rank("exponential backoff retries", docs); +// => [doc1, doc2] (doc3 is filtered out via relevancy threshold) ``` -**What's happening:** +Intent will use a default Groq client when `GROQ_API_KEY` is set. + +## Core API + +- `rank(query, candidates)` → rerank + threshold filter (score-based) +- `filter(query, candidates)` → keep only relevant items (boolean) +- `choice(query, candidates)` → choose exactly one best item + +All three support `{ explain: true }` to return explanations. -- No configuration needed – Intent auto-detects your `GROQ_API_KEY` from environment variables -- Keys and summaries are generated automatically using pretty-printed JSON and hashing -- Default threshold of 0 includes all results scored above zero -- Works great for prototyping and simple use cases +## Example Use Cases -### With Type Safety: Custom Item Types +### 1) Ordering search results with `rank()` -For structured data, specify the type and provide extractors to tell Intent how to identify and describe your items: +Use `rank()` when you want ranked, ordered results. -```typescript +```ts import { Intent } from "intent"; -type Document = { +type Doc = { id: string; title: string; - content: string; - category: string; + body: string; + tags: string[]; }; -const intent = new Intent({ - key: (doc) => doc.title, - summary: (doc) => `${doc.category}: ${doc.content.slice(0, 200)}`, +const intent = new Intent({ + key: (d) => d.id, + relevancyThreshold: 5, }); -const docs: Document[] = [ - { id: "1", title: "Q2 Expenses", content: "Travel and meals...", category: "Finance" }, - { id: "2", title: "OKR Planning", content: "Team goals for...", category: "Strategy" }, - { id: "3", title: "Equipment Purchases", content: "New laptops...", category: "Finance" }, +const docs: Doc[] = [ + { id: "1", title: "Q2 expenses", body: "Travel, meals, ...", tags: ["finance"] }, + { id: "2", title: "OKR planning", body: "Goals for ...", tags: ["strategy"] }, + { id: "3", title: "Laptop purchases", body: "New laptops ...", tags: ["finance", "it"] }, ]; -const results = await intent.rank("expense reports", docs); -// Returns finance-related docs, scored by relevance to "expense reports" +const results = await intent.rank("Find expense reports and anything about spend approvals", docs); ``` -**What's happening:** +### Include explanations -- `key` provides a human-readable identifier for each item (used in LLM prompts) -- `summary` gives the LLM context about each item to make scoring decisions -- Type parameter `` ensures type safety for your extractors -- Still using GROQ_API_KEY auto-detection and default config +```ts +const results = await intent.rank("expense reports", docs, { explain: true }); +// => [{ item: Doc, explanation: string }, ...] +``` -### Tuning Results: Configuration Options +## 2) Tool filtering with `filter()` -Adjust Intent's behavior using configuration options: +Use `filter()` when you want to keep the subset of items in a collection that +are relevant to a query. -```typescript +```ts import { Intent } from "intent"; -const intent = new Intent({ - key: (doc) => doc.title, - summary: (doc) => doc.content.slice(0, 150), - relevancyThreshold: 5, // Only return items scored 6+ (0-10 scale) - batchSize: 30, // Process 30 items per LLM call - timeoutMs: 5000, // Wait up to 5 seconds for LLM responses -}); -``` - -**What's happening:** - -- `relevancyThreshold` controls selectivity – higher values = fewer, more relevant results -- `batchSize` affects token usage and latency (larger = fewer LLM calls, more tokens per call) -- `timeoutMs` prevents long waits on slow LLM responses -- These can also be set via environment variables (`INTENT_RELEVANCY_THRESHOLD`, etc.) - -### Custom LLM: Bring Your Own Client - -Use any LLM provider by implementing the simple `LlmClient` interface: - -```typescript -import { Intent, type LlmClient } from "intent"; -import Anthropic from "@anthropic-ai/sdk"; - -// Adapt your LLM SDK to Intent's interface -const myClient: LlmClient = { - async call(messages, outputSchema, config, userId) { - const client = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }); - const response = await client.messages.create({ - model: config?.model ?? "claude-3-5-sonnet-20241022", - max_tokens: 1024, - messages: messages.map((m) => ({ role: m.role, content: m.content })), - // Use outputSchema to validate response structure - }); - // Parse and return { data: Record } - const content = response.content[0].text; - return { data: JSON.parse(content) }; - }, +type Tool = { + name: string; + description: string; }; -const intent = new Intent({ - llm: myClient, - key: (doc) => doc.title, - summary: (doc) => doc.content, +const intent = new Intent({ + key: (t) => t.name, + summary: (t) => t.description, }); -``` - -**What's happening:** -- Provide your own `llm` client that implements the `LlmClient` interface -- Intent works with any LLM that can return structured JSON -- The `messages` parameter contains the full prompt with query and candidates -- The `outputSchema` parameter specifies the expected response structure - -### Full Configuration: All Options Together - -Combine everything for complete control: - -```typescript -import { Intent } from "intent"; - -const intent = new Intent({ - // LLM client - llm: myCustomClient, - userId: "org-12345", // Optional: for provider abuse monitoring - - // Extractors - key: (doc) => doc.title, - summary: (doc) => `[${doc.category}] ${doc.content.slice(0, 200)}`, +const tools: Tool[] = [ + { name: "search", description: "Search the web for up-to-date information" }, + { name: "sendEmail", description: "Send an email to a recipient" }, + { name: "runSQL", description: "Run a SQL query against the analytics DB" }, + { name: "createInvoice", description: "Create an invoice for a customer" }, +]; - // Configuration - model: "openai/gpt-4o", - relevancyThreshold: 7, - batchSize: 25, - timeoutMs: 10000, - tinyBatchFraction: 0.15, +const task = "Find the customer's last invoice total and email it to them."; - // Logging - logger: console, -}); +const relevantTools = await intent.filter(task, tools); +// [sendEmail, runSQL] ``` -**What's happening:** - -- All options in one place for maximum flexibility -- `userId` is passed to your LLM provider (useful for rate limiting, abuse detection) -- `model` overrides the default model (when using Groq or compatible clients) -- `logger` receives warnings and errors (any object with `info`, `warn`, `error` methods) -- `tinyBatchFraction` controls batch merging behavior (avoids inefficient tiny final batches) - -## Highlights - -- Pluggable LLM client interface (Groq/OpenAI/etc.) -- Stable, safe behavior with fallbacks -- Strict JSON schema scoring, duplicate-key handling -- Fully typed TypeScript API with tests - -## How It Works - -- Listwise LLM reranker: given a user query and a set of candidate items (each with a short key and optional summary), the LLM sees the query and all candidates together and assigns each a relevance score from 0–10. -- Intent-aware ranking: the prompt emphasizes the user’s intent, task framing, and constraints (not just surface similarity). Items that best satisfy the intent rise, even when lexical overlap is low. -- Threshold + stable ordering: scores are filtered by a configurable threshold and returned in descending order; ties preserve original input order. -- Retriever-agnostic: use with any first-stage retriever (vector, BM25, hybrid) or any arbitrary list of items. Summaries help the LLM reason efficiently within token limits. +### Filter with explanations -Why It Excels At User Intent - -- Interprets nuance: weighs the goal behind the query (task, specificity, constraints, entities, and outcome) instead of matching keywords alone. -- Cross-candidate reasoning: considers all candidates in one pass, comparing which best fulfills the intent relative to the rest. -- Robust to wording: prioritizes items that truly answer the need, even if phrased differently than the query. -- Configurable strictness: tune the relevancy threshold to be more selective for high-precision top results. - -Best Practices - -### Data Quality - -- **Keep summaries short and structured**: Include title, 1–2 key facts, entities, dates, and outcomes. Aim for consistent length across items so the LLM compares fairly. -- **Encode user intent explicitly**: Pass the user's goal, constraints, timeframe, and domain context in the query string you provide to Intent. -- **Use helpful metadata**: Incorporate type, tags, author, and dates into the summary string to improve intent alignment. - -### Performance & Cost - -- **Size the candidate set to fit context**: Start with 50–100 items with concise summaries. Tune `BATCH_SIZE` (or `INTENT_BATCH_SIZE`) to your model's token budget. -- **Favor determinism for stable ranking**: Run with temperature 0 and a fixed prompt template. Ties are already stable by input order. +```ts +const relevantTools = await intent.filter(task, tools, { explain: true }); +// => [{ item: Tool, explanation: string }, ...] +``` -### Accuracy & Results +## 3) Model routing with `choice()` -- **Tune selectivity**: Set `RELEVANCY_THRESHOLD` (or `INTENT_RELEVANCY_THRESHOLD`) to emphasize high-precision top results for RAG and QA. Values 0-10 are supported, with 0 including all results. -- **Optional score fusion**: For even stronger robustness, combine LLM scores with retriever scores (e.g., a weighted sum) when you have them. +Use `choice()` when you need exactly one selection from a set of items. -### Monitoring +```ts +import { Intent } from "intent"; -- **Monitor and iterate**: Log query, candidate count, raw/normalized scores, and token usage to fine-tune thresholds and batch sizes. +type Model = { + id: string; + strengths: string; +}; -Install +const intent = new Intent({ + key: (m) => m.id, + summary: (m) => m.strengths, +}); -```bash -npm install intent -``` +const models: Model[] = [ + { + id: "gemini-3-pro", + strengths: "Hard reasoning, math, complex debugging. Slower but very strong.", + }, + { + id: "gpt-5.2", + strengths: "Best for code generation, refactors, feature implementation.", + }, + { + id: "haiku-4.5" + strengths: "Fast and cheap. Good for triage and simple edits.", + }, + { + id: "nano-banana-pro", + strengths: "Image generation and visual content.", + }, +]; -Or with yarn: +const task = "Implement a feature to add retries with exponential backoff and tests."; -```bash -yarn add intent +const { item: selected, explanation } = await intent.choice(task, models, { explain: true }); +// selected.id => gpt-5.2 ``` -Configuration +## Configuration -The library reads `.env` automatically when imported. Create a `.env` file in your project root: +Intent reads `.env` automatically when imported. ```env -# Required only if not providing ctx.llm GROQ_API_KEY=your_groq_api_key_here -# Optional: customize defaults -INTENT_MODEL=openai/gpt-oss-120b -INTENT_TIMEOUT_MS=30000 +# Optional defaults +GROQ_DEFAULT_MODEL=openai/gpt-oss-20b +GROQ_DEFAULT_REASONING_EFFORT=medium +INTENT_TIMEOUT_MS=3000 +INTENT_MIN_SCORE=0 +INTENT_MAX_SCORE=10 INTENT_RELEVANCY_THRESHOLD=0 INTENT_BATCH_SIZE=20 -INTENT_TINY_BATCH_FRACTION=0.15 +INTENT_TINY_BATCH_FRACTION=0.2 ``` -**Configuration Options:** +### How configuration works -- **`GROQ_API_KEY`**: Your Groq API key. Required if not providing a custom `ctx.llm` client. -- **`INTENT_MODEL`**: LLM model to use for reranking (default: `openai/gpt-oss-20b`). Any Groq-supported model works here. -- **`INTENT_TIMEOUT_MS`**: Maximum time in milliseconds to wait for LLM responses (default: `3000`). Increase for larger batches or slower models. -- **`INTENT_RELEVANCY_THRESHOLD`**: Minimum relevance score (0-10) to include in results (default: `0`). Higher values = more selective filtering. -- **`INTENT_BATCH_SIZE`**: Number of candidates to score per LLM call (default: `20`). Tune based on your model's context window and candidate summary length. -- **`INTENT_TINY_BATCH_FRACTION`**: Threshold for merging small trailing batches (default: `0.2`). If the last batch is smaller than this fraction of `BATCH_SIZE`, it gets merged with the previous batch to avoid inefficient LLM calls. +- You can configure Intent via **environment variables** (shown above) or via the `Intent` constructor. +- Constructor options override environment defaults for that instance. +- Most tuning is a trade-off between **quality**, **latency**, and **cost**. -Development +### Provider + model -- Build: `npm run build` (requires TypeScript) -- Lint and format: `npm run lint:check` (check) or `npm run lint` (auto-fix) -- Tests with coverage: `npm test` (uses Vitest + v8 coverage) +#### `GROQ_API_KEY` -Tests +If you don't pass a custom `llm` client, Intent will create a default Groq client when this is set. -- Co-located alongside source for easy association. - - Unit: `src/**/*.unit.test.ts` (100% coverage enforced) - - Integration: `src/**/*.int.test.ts` (live Groq; requires `GROQ_API_KEY`) -- Scripts - - `npm run test:unit` — unit tests only - - `npm run test:int` — integration tests only (concurrent) - - `npm test` — all tests - - Optional: set `TEST_SCOPE=unit|int|all` to control scope +#### `GROQ_DEFAULT_MODEL` -Groq default +Sets the model name used by the built-in Groq client. -- Uses `groq-sdk` under the hood. If `GROQ_API_KEY` is set in the environment, you can omit `ctx.llm` and Intent will use a built‑in Groq adapter automatically. -- Otherwise, provide your own `llm` client via `ctx.llm`. +- Choose a stronger model when you care about nuanced ranking or long candidate summaries. +- Choose a smaller model when you want lower latency/cost and your candidates are simple. -## Config +#### `GROQ_DEFAULT_REASONING_EFFORT` -- Config can be supplied at construction or via environment variables (see Configuration section above for details). -- The library reads `.env` automatically when imported, so `INTENT_*` keys in your `.env` are honored. -- Constructor config overrides environment variables for fine-grained control per instance. +Controls how much reasoning the model should do (`low | medium | high`). -## API Reference +- `low`: fastest; best for obvious matches. +- `medium`: good default. +- `high`: better for subtle intent, but typically slower/more expensive. -### Constructor +### Ranking behavior -```typescript -new Intent(options?: IntentOptions) -``` +#### `INTENT_RELEVANCY_THRESHOLD` -Creates a new Intent instance with optional configuration. All parameters are optional with sensible defaults. +Controls how selective the output is. -**Type Parameters:** +- Higher threshold → fewer results (higher precision) +- Lower threshold → more results (higher recall) -- `T` - The type of items to rerank (defaults to `any`) +Important: threshold filtering is **strictly greater-than** (`score > threshold`). +So with the default score range `0..10`: -**Options:** +- `relevancyThreshold=0` keeps scores `1..10` +- `relevancyThreshold=5` keeps scores `6..10` -_LLM Client & Context:_ +#### `INTENT_MIN_SCORE` / `INTENT_MAX_SCORE` -- `llm?: LlmClient` - Custom LLM client. If omitted, auto-detects Groq via `GROQ_API_KEY` -- `logger?: LoggerLike` - Logger for warnings and errors (any object with `info`, `warn`, `error` methods) -- `userId?: string` - User identifier passed to LLM provider for monitoring +Controls the score range given to the LLM. -_Extractors:_ +- Narrower ranges (e.g. `1..5`) can make scoring easier to calibrate. +- Wider ranges (e.g. `0..10`) give more resolution for ranking. -- `key?: (item: T) => string` - Extracts human-readable identifier from items. Default: hash of pretty-printed JSON -- `summary?: (item: T) => string` - Extracts description for LLM reasoning. Default: pretty-printed JSON (2-space indentation) +Note: Since the is an LLM's judgement, as opposed to an objective measurement, +scores may not use the full range perfectly and you may seem similar biases +that you'd see with human raters. -_Configuration:_ +This also means that massive ranges (e.g. `0..1000`) may not yield more +precise results. -- `model?: string` - LLM model name (default: `INTENT_MODEL` env or `"openai/gpt-oss-20b"`) -- `timeoutMs?: number` - Request timeout in milliseconds (default: `INTENT_TIMEOUT_MS` env or `3000`) -- `relevancyThreshold?: number` - Minimum score (0-10) to include in results (default: `INTENT_RELEVANCY_THRESHOLD` env or `0`) -- `batchSize?: number` - Candidates per LLM call (default: `INTENT_BATCH_SIZE` env or `20`) -- `tinyBatchFraction?: number` - Threshold for merging small batches (default: `INTENT_TINY_BATCH_FRACTION` env or `0.2`) +If you change the range, ensure your `relevancyThreshold` stays within it. -**Throws:** +### Performance knobs -- `Error` - If no LLM client provided and `GROQ_API_KEY` not set -- `Error` - If `relevancyThreshold` not between 0 and 10 +#### `INTENT_TIMEOUT_MS` -### rank Method +Hard timeout per LLM call. -```typescript -rank(query: string, candidates: T[], options?: { userId?: string }): Promise -``` +- Increase it when you have larger batches, longer summaries, or slower models. +- Decrease it when you prefer quick fallbacks over waiting. -Ranks candidates based on relevance to the query. +If we timeout, we never throw an error; instead, we return the original +results. -**Parameters:** +#### `INTENT_BATCH_SIZE` -- `query` - The search query or user intent to rank against -- `candidates` - Array of items to rerank -- `options.userId` - Optional user ID for this call (overrides constructor `userId`) +How many candidates are evaluated per LLM call. -**Returns:** Filtered and sorted array of items +- Larger batch size → fewer calls (often cheaper/faster), but higher token usage per call. +- Smaller batch size → more calls (often slower), but each call is smaller. -**Behavior:** +#### `INTENT_TINY_BATCH_FRACTION` -- Fast-path: Returns empty array for 0 candidates, unchanged array for 1 candidate (no LLM calls) -- Scores each candidate 0-10 based on relevance -- Filters results by `relevancyThreshold` -- Sorts by score descending, preserving input order for ties -- On error: Returns items in original order (graceful degradation) +When the last batch is “too small”, Intent will merge it into the previous batch. -## Notes +- Increase to avoid tiny extra calls (better latency/cost). +- Decrease if you frequently run near context limits. -- Always returns a list; on any failure, it preserves the original order for the affected batch. -- Ties keep original order (stable sort by input index). -- Duplicate keys are internally disambiguated: `"Key (idx)"`. +### Programmatic configuration (constructor) + +Everything above can be set per instance: + +```ts +import { Intent } from "intent"; + +const intent = new Intent({ + timeoutMs: 10_000, + batchSize: 25, + relevancyThreshold: 3, + minScore: 0, + maxScore: 10, +}); +``` diff --git a/src/intent.int.test.ts b/src/intent.int.test.ts index 5e8e4b0..e55b628 100644 --- a/src/intent.int.test.ts +++ b/src/intent.int.test.ts @@ -6,6 +6,27 @@ describe("intent integration", () => { const defaultTimeoutMs = 10000; const scoreRange = { minScore: 0, maxScore: 10 }; + test.concurrent( + "README quickstart: ranks simple strings and filters out unrelated items", + async () => { + const intent = new Intent({ relevancyThreshold: 1, timeoutMs: defaultTimeoutMs }); + + const docs = [ + "Many network requests can fail intermittently due to transient issues.", + "To reduce flaky tests, add exponential backoff with jitter to HTTP retries.", + "Citrus fruits like oranges and lemons are high in vitamin C.", + ]; + + const ranked = await intent.rank("exponential backoff retries", docs); + + expect(ranked).toEqual([ + "To reduce flaky tests, add exponential backoff with jitter to HTTP retries.", + "Many network requests can fail intermittently due to transient issues.", + ]); + }, + 30000, + ); + test.concurrent( "ranks the most relevant candidate first (simple object candidates)", async () => { @@ -39,6 +60,46 @@ describe("intent integration", () => { 30000, ); + test.concurrent( + "supports candidates with summary only (no key extractor)", + async () => { + const intent = new Intent<{ summary: string }>({ + summary: (x) => x.summary, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: 0, + }); + + const out = await intent.rank("Help me sort a JavaScript array", [ + { summary: "The chemical composition of Saturn's moons" }, + { summary: "Guide to sorting arrays in JavaScript" }, + { summary: "Directions to the tower" }, + ]); + + expect(out).toEqual([{ summary: "Guide to sorting arrays in JavaScript" }]); + }, + 30000, + ); + + test.concurrent( + "supports candidates with key only (no summary extractor)", + async () => { + const intent = new Intent<{ key: string }>({ + key: (x) => x.key, + timeoutMs: defaultTimeoutMs, + relevancyThreshold: 0, + }); + + const out = await intent.rank("Help me sort a JavaScript array", [ + { key: "Saturns Moons" }, + { key: "JS Arrays" }, + { key: "Eiffel Tower" }, + ]); + + expect(out).toEqual([{ key: "JS Arrays" }]); + }, + 30000, + ); + test.concurrent( "returns [] when all candidates are unrelated and threshold > minScore", async () => { @@ -370,6 +431,39 @@ describe("intent integration", () => { 30000, ); + test.concurrent( + "README filter example: filters tools down to the relevant subset", + async () => { + type Tool = { + name: string; + description: string; + }; + + const intent = new Intent({ + key: (t) => t.name, + summary: (t) => t.description, + timeoutMs: defaultTimeoutMs, + }); + + const tools: Tool[] = [ + { name: "search", description: "Search the web for up-to-date information" }, + { name: "sendEmail", description: "Send an email to a recipient" }, + { name: "runSQL", description: "Run a SQL query against the analytics DB" }, + { name: "createInvoice", description: "Create an invoice for a customer" }, + ]; + + const task = "Find the customer's last invoice total and email it to them."; + + const relevantTools = await intent.filter(task, tools); + + // Filter is order-preserving, so we only assert membership. + const names = relevantTools.map((t) => t.name); + expect(names).toEqual(expect.arrayContaining(["sendEmail", "runSQL"])); + expect(names).not.toEqual(expect.arrayContaining(["createInvoice"])); + }, + 30000, + ); + test.concurrent( "filter({ explain: true }) returns aligned explanations", async () => { @@ -441,4 +535,48 @@ describe("intent integration", () => { }, 60000, ); + + test.concurrent( + "README choice example: routes to a best-fit model and explains why", + async () => { + type Model = { + id: string; + strengths: string; + }; + + const intent = new Intent({ + key: (m) => m.id, + summary: (m) => m.strengths, + timeoutMs: defaultTimeoutMs, + }); + + const models: Model[] = [ + { + id: "gemini-3-pro", + strengths: "Hard reasoning, math, complex debugging. Slower but very strong.", + }, + { + id: "gpt-5.2", + strengths: "Best for code generation, refactors, feature implementation.", + }, + { + id: "haiku-4.5", + strengths: "Fast and cheap. Good for triage and simple edits.", + }, + { + id: "nano-banana-pro", + strengths: "Image generation and visual content.", + }, + ]; + + const task = "Implement a feature to add retries with exponential backoff and tests."; + + const { item: selected, explanation } = await intent.choice(task, models, { explain: true }); + + expect(selected.id).toBe("gpt-5.2"); + expect(typeof explanation).toBe("string"); + expect(explanation.length).toBeGreaterThan(0); + }, + 60000, + ); }); diff --git a/src/messages.ts b/src/messages.ts index 961995f..c2580ef 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -21,12 +21,25 @@ export function buildMessages( candidates: IntentCandidate[], scoreRange: { minScore: number; maxScore: number }, ): ChatMessage[] { - const system = `The user will provide a short description of a query they are trying to automate, along with a JSON blob containing candidate_search_results. Each candidate result has a uniquely identifying key and a short summary. Your task is to assess each candidate and return a JSON object that maps candidate keys to objects of the form {"explanation": string, "score": integer} where score is from ${scoreRange.minScore} to ${scoreRange.maxScore}: ${scoreRange.minScore} means not relevant at all, and ${scoreRange.maxScore} means highly relevant. Sometimes none are relevant, sometimes all are relevant. Be aggressive and decisive on relevancy. + const system = `You will receive a JSON blob containing candidate_search_results (each candidate has a key and a short summary) plus a short user request. + +Your task is to assess each candidate and return a JSON object that maps candidate keys to objects of the form {"explanation": string, "score": integer} avoiding ambiguity. + +The score must be an integer from ${scoreRange.minScore} to ${scoreRange.maxScore}: +- ${scoreRange.minScore} means not relevant at all +- ${scoreRange.maxScore} means highly relevant + +Sometimes none are relevant, sometimes all are relevant. Be decisive. It is okay to return ${scoreRange.minScore} if the candidate is not relevant to the query. It is okay to return ${scoreRange.maxScore} if the candidate is highly relevant to the query. Use the full range of scores. Every candidate MUST include an explanation. Write the explanation first, then the score. The explanation should be concise (1-3 sentences), concrete, and reference the query intent and the candidate summary. +Write explanations as end-user-facing justifications: +- Do NOT say "the query" or talk about prompt mechanics. +- Write in a direct, item-first voice (e.g., "gpt-5.2 is best here because it specializes in feature implementation and testing."). +- Avoid "I"/"we". + Every key in candidate_search_results must be present in your output mapping. Do not add any keys that are not present in candidate_search_results. Every key in candidate_search_results must map to an object with: - explanation: string @@ -65,12 +78,19 @@ Pretty-print the JSON for readability.`; * @returns Array of chat messages ready for LLM consumption */ export function buildFilterMessages(query: string, candidates: IntentCandidate[]): ChatMessage[] { - const system = `The user will provide a short description of a query they are trying to automate, along with a JSON blob containing candidate_search_results. Your task is to assess each candidate and return a JSON object that maps candidate keys to objects of the form {"explanation": string, "isRelevant": boolean}. + const system = `You will receive a JSON blob containing candidate_search_results (each candidate has a key and a short summary) plus a short user request. + +Your task is to assess each candidate and return a JSON object that maps candidate keys to objects of the form {"explanation": string, "isRelevant": boolean}. Return isRelevant=true only when the candidate clearly helps satisfy the query intent. Otherwise return isRelevant=false. Every candidate MUST include an explanation. Write the explanation first, then the boolean. The explanation should be concise (1-3 sentences), concrete, and reference the query intent and the candidate summary. +Write explanations as end-user-facing justifications: +- Do NOT say "the query" or talk about prompt mechanics. +- Write in a direct, item-first voice. +- Avoid "I"/"we". + Every key in candidate_search_results must be present in your output mapping. Do not add any keys that are not present in candidate_search_results. Every key in candidate_search_results must map to an object with: - explanation: string @@ -101,12 +121,19 @@ Pretty-print the JSON for readability.`; * @returns Array of chat messages ready for LLM consumption */ export function buildChoiceMessages(query: string, candidates: IntentCandidate[]): ChatMessage[] { - const system = `The user will provide a short description of a query they are trying to automate, along with a JSON blob containing candidate_search_results. Your task is to choose exactly one candidate as the best match for the query. + const system = `You will receive a JSON blob containing candidate_search_results (each candidate has a key and a short summary) plus a short user request. + +Your task is to choose exactly one candidate as the best match for what the user wants. You MUST choose one candidate key from the provided list. Do not choose multiple. Return ONLY JSON of the form: {"explanation": string, "selectedKey": string} where selectedKey is exactly one of the candidate keys. The explanation should be concise (1-3 sentences), concrete, and reference the query intent and the candidate summary. +Write the explanation as an end-user-facing justification: +- Do NOT say "the query" or talk about prompt mechanics. +- Write in a direct, item-first voice. +- Avoid "I"/"we". + Do not include any other text outside the JSON. Return only JSON matching the enforced schema. Pretty-print the JSON for readability.`; From e2fddb1a6db775e014e3fcd64bf9e6ea84387320 Mon Sep 17 00:00:00 2001 From: Steve Krenzel Date: Mon, 22 Dec 2025 01:23:00 -0800 Subject: [PATCH 12/12] chore(package): publish as @with-logic/intent Rename the npm package to the @with-logic scope and update README examples to import from the scoped name. Also set publishConfig.access=public so scoped publishes default to public access. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- README.md | 10 +++++----- package.json | 5 ++++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 25fab27..c1ec92b 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,13 @@ retrain existing ones. ## Install ```bash -npm install intent +npm install @with-logic/intent ``` ## Quickstart ```ts -import { Intent } from "intent"; +import { Intent } from "@with-logic/intent"; const intent = new Intent({ relevancyThreshold: 1 }); @@ -48,7 +48,7 @@ All three support `{ explain: true }` to return explanations. Use `rank()` when you want ranked, ordered results. ```ts -import { Intent } from "intent"; +import { Intent } from "@with-logic/intent"; type Doc = { id: string; @@ -84,7 +84,7 @@ Use `filter()` when you want to keep the subset of items in a collection that are relevant to a query. ```ts -import { Intent } from "intent"; +import { Intent } from "@with-logic/intent"; type Tool = { name: string; @@ -121,7 +121,7 @@ const relevantTools = await intent.filter(task, tools, { explain: true }); Use `choice()` when you need exactly one selection from a set of items. ```ts -import { Intent } from "intent"; +import { Intent } from "@with-logic/intent"; type Model = { id: string; diff --git a/package.json b/package.json index d5b1cfa..2484920 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "name": "intent", + "name": "@with-logic/intent", "version": "0.1.0", "private": false, "description": "Intent: a small, well-typed LLM-based reranker.", @@ -20,6 +20,9 @@ "url": "https://github.com/with-logic/intent/issues" }, "homepage": "https://github.com/with-logic/intent#readme", + "publishConfig": { + "access": "public" + }, "type": "module", "sideEffects": false, "main": "dist/index.js",