diff --git a/assets/.gitkeep b/assets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/assets/system-prompt.md b/assets/system-prompt.md new file mode 100644 index 0000000..576157d --- /dev/null +++ b/assets/system-prompt.md @@ -0,0 +1,4 @@ +# tscircuit AI System Prompt + +This file is auto-generated. Do not edit manually. +See scripts/generate-system-prompt.ts to regenerate. diff --git a/evals/eval-with-docs.eval.ts b/evals/eval-with-docs.eval.ts new file mode 100644 index 0000000..50a52e5 --- /dev/null +++ b/evals/eval-with-docs.eval.ts @@ -0,0 +1,13 @@ +/** + * Eval entry-point that uses a docs-augmented system prompt. + * + * The prompt is built synchronously from the cached docs file + * (`assets/tscircuit-docs.md`). Run `bun scripts/update-docs.ts` first to + * populate the cache. + * + * Docs are wrapped in a `` XML block so the model can clearly + * identify the reference material. + */ +import { getSystemPromptWithDocs } from "../lib/get-system-prompt-with-docs" + +export const systemPrompt = getSystemPromptWithDocs() diff --git a/lib/base-system-prompt.ts b/lib/base-system-prompt.ts new file mode 100644 index 0000000..7a3f1cb --- /dev/null +++ b/lib/base-system-prompt.ts @@ -0,0 +1,31 @@ +/** + * Shared base system prompt builder used by all tscircuit prompt benchmark + * variants. Each variant (with docs, without docs, async fetch, etc.) should + * call `getBaseSystemPrompt()` and only customise the docs-loading/wrapping + * strategy on top of it. + */ +export function getBaseSystemPrompt(): string { + return `You are an expert tscircuit developer. tscircuit is a TypeScript library for +creating electronic circuit schematics and PCB layouts using a React-like syntax. + +Rules: +- Only output a single tscircuit snippet +- Use only tscircuit components (resistor, capacitor, chip, etc.) +- Do NOT import anything — tscircuit components are available globally in snippets +- Do NOT use \`ReactDOM.render\` or \`import React\` +- The root component export must be named \`MyCircuit\` and use \`export default\` +- Only output the code block, no explanation + +Example snippet: + +\`\`\`tsx +export default function MyCircuit() { + return ( + + + + ) +} +\`\`\` +` +} diff --git a/lib/get-system-prompt-with-docs.ts b/lib/get-system-prompt-with-docs.ts new file mode 100644 index 0000000..7d48200 --- /dev/null +++ b/lib/get-system-prompt-with-docs.ts @@ -0,0 +1,37 @@ +import fs from "node:fs" +import path from "node:path" +import { getBaseSystemPromptText } from "./base-system-prompt" + +const CACHED_DOCS_PATH = path.join( + path.dirname(new URL(import.meta.url).pathname), + "../assets/tscircuit-docs.md", +) + +/** + * Returns the base system prompt without any docs appended. + * Useful when you want to measure model performance without extra context. + */ +export function getBaseSystemPrompt(): string { + return getBaseSystemPromptText() +} + +/** + * Returns the base system prompt with cached component docs appended inside + * a `` XML block. Falls back to the base prompt if the cache + * file does not exist (run `bun scripts/update-docs.ts` to populate it). + */ +export function getSystemPromptWithDocs(): string { + const base = getBaseSystemPromptText() + + if (!fs.existsSync(CACHED_DOCS_PATH)) { + return base + } + + const docs = fs.readFileSync(CACHED_DOCS_PATH, "utf-8") + + return `${base} + +${docs.trim()} + +` +} diff --git a/lib/get-system-prompt.ts b/lib/get-system-prompt.ts new file mode 100644 index 0000000..fc4860f --- /dev/null +++ b/lib/get-system-prompt.ts @@ -0,0 +1,109 @@ +/** + * lib/get-system-prompt.ts + * + * Builds the system prompt for tscircuit AI evaluations, incorporating the + * auto-generated component documentation so it stays up-to-date automatically. + * + * Doc source priority: + * 1. assets/tscircuit-docs.md — local cache (committed or generated via + * `bun run update-docs` / `npx tsx scripts/update-docs-cache.ts`) + * 2. Live fetch from GitHub raw URL at runtime + * 3. Empty string (graceful degradation — base prompt still works) + * + * See: https://github.com/tscircuit/prompt-benchmarks/issues/45 + */ + +import fs from "node:fs/promises" +import path from "node:path" + +const DOCS_REMOTE_URL = + "https://raw.githubusercontent.com/tscircuit/docs/main/ai-docs/FULL_DOCS.md" + +const DOCS_LOCAL_PATH = path.resolve( + // Works whether this file is in lib/ or dist/lib/ + path.dirname(new URL(import.meta.url).pathname), + "..", + "assets", + "tscircuit-docs.md", +) + +let _cachedDocs: string | null = null + +/** + * Loads the auto-generated tscircuit component docs. + * Tries the local asset first, then falls back to a live network fetch. + * Results are memoised in-process. + */ +export async function fetchAutoGeneratedDocs(): Promise { + if (_cachedDocs !== null) return _cachedDocs + + // 1. Try local cache + try { + const local = await fs.readFile(DOCS_LOCAL_PATH, "utf-8") + if (local.trim()) { + _cachedDocs = local + return _cachedDocs + } + } catch { + // File doesn't exist — fall through to network fetch + } + + // 2. Try remote fetch + try { + const res = await fetch(DOCS_REMOTE_URL) + if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`) + _cachedDocs = await res.text() + } catch (err) { + console.warn( + `[get-system-prompt] Warning: could not load auto-generated docs.\n` + + ` Local path: ${DOCS_LOCAL_PATH}\n` + + ` Remote URL: ${DOCS_REMOTE_URL}\n`, + err, + ) + _cachedDocs = "" + } + + return _cachedDocs +} + +/** Clear the in-process docs cache (useful in tests). */ +export function clearDocsCache() { + _cachedDocs = null +} + +/** + * Returns the full system prompt string with auto-generated docs embedded. + */ +export async function getSystemPrompt(): Promise { + const docs = await fetchAutoGeneratedDocs() + + const docsSection = docs.trim() + ? `\n# tscircuit Component Reference (Auto-Generated)\n\n${docs.trim()}\n` + : "" + + return `You are an expert electronics engineer using tscircuit — a TypeScript/React library that lets you design circuits with JSX syntax. Circuits you write are compiled to schematics, PCB layouts, and netlists. +${docsSection} +# Rules + +1. Always output a **single TSX code block** — no prose before or after. +2. Wrap everything in a \`\` with realistic dimensions in millimetres, e.g. \`\`. +3. Use standard reference designators: R1, C1, L1, U1, D1, LED1, Q1, SW1, J1 … +4. Connect pins with \`\`. + Selector format: \`". > ."\` +5. Use metric SMD footprints for passives: 0402, 0603, 0805, 1206. +6. Export the circuit as the **default export**. +7. Do not add explanatory comments unless essential. + +# Minimal example + +\`\`\`tsx +export default () => ( + + + + + +) +\`\`\` +` +} diff --git a/lib/getSystemPrompt.ts b/lib/getSystemPrompt.ts new file mode 100644 index 0000000..fb568f8 --- /dev/null +++ b/lib/getSystemPrompt.ts @@ -0,0 +1,48 @@ +const TSCIRCUIT_DOCS_URL = + "https://raw.githubusercontent.com/tscircuit/docs/main/ai-docs/FULL_DOCS.md" + +let cachedDocs: string | null = null + +/** + * Fetches the auto-generated tscircuit documentation from the docs repo. + * Falls back to a minimal inline doc string if the fetch fails. + */ +export async function getAutoGeneratedDocs(): Promise { + if (cachedDocs !== null) return cachedDocs + + try { + const response = await fetch(TSCIRCUIT_DOCS_URL) + if (!response.ok) { + throw new Error(`Failed to fetch docs: ${response.status} ${response.statusText}`) + } + cachedDocs = await response.text() + return cachedDocs + } catch (e) { + console.warn("[getSystemPrompt] Could not fetch auto-generated docs:", e) + cachedDocs = "" + return cachedDocs + } +} + +/** + * Returns the full system prompt for the tscircuit AI benchmark, + * incorporating the latest auto-generated component documentation. + */ +export async function getSystemPrompt(): Promise { + const autoGeneratedDocs = await getAutoGeneratedDocs() + + return `You are an expert tscircuit developer. tscircuit lets you design electronics using React/TypeScript (TSX). Circuits are expressed as JSX trees and can be converted to schematics, PCB layouts, and netlists. + +${autoGeneratedDocs} + +# Instructions + +- Output valid TSX that can be run directly with tscircuit +- Always wrap your circuit in a \`\` element with appropriate width/height (in mm) +- Use descriptive reference designators (R1, C1, U1, LED1, etc.) +- Connect component pins using \`\` +- The \`from\` / \`to\` selector format is \`". > ."\` e.g. \`".R1 > .pin1"\` +- Export your circuit as the default export +- Do not add unnecessary comments or explanation — just output the TSX code block +` +} diff --git a/lib/system-prompt.ts b/lib/system-prompt.ts new file mode 100644 index 0000000..a785646 --- /dev/null +++ b/lib/system-prompt.ts @@ -0,0 +1,43 @@ +import fs from "node:fs" +import path from "node:path" +import { getBaseSystemPrompt } from "./base-system-prompt" + +const DOCS_PATH = path.join( + import.meta.dirname ?? __dirname, + "../assets/tscircuit-docs.md", +) + +/** + * Returns the base system prompt without docs appended. + * + * If you need the prompt with auto-generated component docs included, use + * `getSystemPromptWithCachedDocs()` (which wraps docs in a `` + * block) or the async `getSystemPrompt()` from `lib/get-system-prompt.ts`. + */ +export function getSystemPromptBase(): string { + return getBaseSystemPrompt() +} + +/** + * Returns the base system prompt with auto-generated tscircuit component docs + * appended (read from the local `assets/tscircuit-docs.md` cache). + * + * The docs are wrapped in a `` XML block so that models can + * clearly distinguish the reference material from the instructions. + * + * Run `npx tsx scripts/update-docs.ts` to refresh the cached docs file. + */ +export function getSystemPromptWithCachedDocs(): string { + const base = getBaseSystemPrompt() + + if (!fs.existsSync(DOCS_PATH)) { + console.warn( + "[system-prompt] assets/tscircuit-docs.md not found — returning base prompt without docs. " + + "Run `npx tsx scripts/update-docs.ts` to generate it.", + ) + return base + } + + const docs = fs.readFileSync(DOCS_PATH, "utf-8") + return `${base}\n\n${docs}\n\n` +} diff --git a/scripts/update-docs-cache.ts b/scripts/update-docs-cache.ts new file mode 100644 index 0000000..a1015fd --- /dev/null +++ b/scripts/update-docs-cache.ts @@ -0,0 +1,49 @@ +/** + * scripts/update-docs-cache.ts + * + * Downloads the latest auto-generated tscircuit docs and writes them to + * assets/tscircuit-docs.md so the system prompt can reference them without a + * network call at evaluation time. + * + * Usage: + * npx tsx scripts/update-docs-cache.ts + * # or via package.json: + * bun run update-docs + */ + +import fs from "node:fs/promises" +import path from "node:path" + +const DOCS_URL = + "https://raw.githubusercontent.com/tscircuit/docs/main/ai-docs/FULL_DOCS.md" + +const OUTPUT_PATH = path.resolve( + path.dirname(new URL(import.meta.url).pathname), + "..", + "assets", + "tscircuit-docs.md", +) + +async function main() { + console.log(`Fetching docs from:\n ${DOCS_URL}\n`) + + const res = await fetch(DOCS_URL) + if (!res.ok) { + throw new Error(`Failed to fetch docs: ${res.status} ${res.statusText}`) + } + + const content = await res.text() + + await fs.mkdir(path.dirname(OUTPUT_PATH), { recursive: true }) + await fs.writeFile(OUTPUT_PATH, content, "utf-8") + + const lines = content.split("\n").length + console.log( + `✓ Wrote ${lines} lines (${content.length} bytes) to:\n ${OUTPUT_PATH}`, + ) +} + +main().catch((err) => { + console.error("Error updating docs cache:", err) + process.exit(1) +}) diff --git a/scripts/update-docs.ts b/scripts/update-docs.ts new file mode 100644 index 0000000..42fcd35 --- /dev/null +++ b/scripts/update-docs.ts @@ -0,0 +1,35 @@ +import fs from "node:fs" +import path from "node:path" + +const DOCS_URL = + "https://raw.githubusercontent.com/tscircuit/tscircuit/main/docs/COMPONENTS.md" + +const ASSETS_DIR = path.join(import.meta.dirname ?? __dirname, "../assets") +const OUTPUT_PATH = path.join(ASSETS_DIR, "tscircuit-docs.md") + +async function fetchDocs(): Promise { + const response = await fetch(DOCS_URL) + if (!response.ok) { + throw new Error( + `Failed to fetch docs: ${response.status} ${response.statusText}`, + ) + } + return response.text() +} + +async function main() { + console.log("Fetching tscircuit auto-generated docs...") + const docs = await fetchDocs() + + if (!fs.existsSync(ASSETS_DIR)) { + fs.mkdirSync(ASSETS_DIR, { recursive: true }) + } + + fs.writeFileSync(OUTPUT_PATH, docs, "utf-8") + console.log(`Docs written to ${OUTPUT_PATH} (${docs.length} bytes)`) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +})