feat: use auto-generated docs in the system prompt (#45)

assets/system-prompt.md

@@ -0,0 +1,4 @@
+# tscircuit AI System Prompt
+
+This file is auto-generated. Do not edit manually.
+See scripts/generate-system-prompt.ts to regenerate.

evals/eval-with-docs.eval.ts

@@ -0,0 +1,13 @@
+/**
+ * Eval entry-point that uses a docs-augmented system prompt.
+ *
+ * The prompt is built synchronously from the cached docs file
+ * (`assets/tscircuit-docs.md`). Run `bun scripts/update-docs.ts` first to
+ * populate the cache.
+ *
+ * Docs are wrapped in a `<tscircuit_docs>` XML block so the model can clearly
+ * identify the reference material.
+ */
+import { getSystemPromptWithDocs } from "../lib/get-system-prompt-with-docs"
+
+export const systemPrompt = getSystemPromptWithDocs()

lib/base-system-prompt.ts

@@ -0,0 +1,31 @@
+/**
+ * Shared base system prompt builder used by all tscircuit prompt benchmark
+ * variants. Each variant (with docs, without docs, async fetch, etc.) should
+ * call `getBaseSystemPrompt()` and only customise the docs-loading/wrapping
+ * strategy on top of it.
+ */
+export function getBaseSystemPrompt(): string {
+  return `You are an expert tscircuit developer. tscircuit is a TypeScript library for
+creating electronic circuit schematics and PCB layouts using a React-like syntax.
+
+Rules:
+- Only output a single tscircuit snippet
+- Use only tscircuit components (resistor, capacitor, chip, etc.)
+- Do NOT import anything — tscircuit components are available globally in snippets
+- Do NOT use \`ReactDOM.render\` or \`import React\`
+- The root component export must be named \`MyCircuit\` and use \`export default\`
+- Only output the code block, no explanation
+
+Example snippet:
+
+\`\`\`tsx
+export default function MyCircuit() {
+  return (
+    <board width="10mm" height="10mm">
+      <resistor resistance="10kohm" footprint="0402" name="R1" schX={3} schY={0} />
+    </board>
+  )
+}
+\`\`\`
+`
+}

lib/get-system-prompt-with-docs.ts

@@ -0,0 +1,37 @@
+import fs from "node:fs"
+import path from "node:path"
+import { getBaseSystemPromptText } from "./base-system-prompt"
+
+const CACHED_DOCS_PATH = path.join(
+  path.dirname(new URL(import.meta.url).pathname),
+  "../assets/tscircuit-docs.md",
+)
+
+/**
+ * Returns the base system prompt without any docs appended.
+ * Useful when you want to measure model performance without extra context.
+ */
+export function getBaseSystemPrompt(): string {
+  return getBaseSystemPromptText()
+}
+
+/**
+ * Returns the base system prompt with cached component docs appended inside
+ * a `<tscircuit_docs>` XML block. Falls back to the base prompt if the cache
+ * file does not exist (run `bun scripts/update-docs.ts` to populate it).
+ */
+export function getSystemPromptWithDocs(): string {
+  const base = getBaseSystemPromptText()
+
+  if (!fs.existsSync(CACHED_DOCS_PATH)) {
+    return base
+  }
+
+  const docs = fs.readFileSync(CACHED_DOCS_PATH, "utf-8")
+
+  return `${base}
+<tscircuit_docs>
+${docs.trim()}
+</tscircuit_docs>
+`
+}

lib/get-system-prompt.ts

@@ -0,0 +1,109 @@
+/**
+ * lib/get-system-prompt.ts
+ *
+ * Builds the system prompt for tscircuit AI evaluations, incorporating the
+ * auto-generated component documentation so it stays up-to-date automatically.
+ *
+ * Doc source priority:
+ *   1. assets/tscircuit-docs.md  — local cache (committed or generated via
+ *      `bun run update-docs` / `npx tsx scripts/update-docs-cache.ts`)
+ *   2. Live fetch from GitHub raw URL at runtime
+ *   3. Empty string (graceful degradation — base prompt still works)
+ *
+ * See: https://github.com/tscircuit/prompt-benchmarks/issues/45
+ */
+
+import fs from "node:fs/promises"
+import path from "node:path"
+
+const DOCS_REMOTE_URL =
+  "https://raw.githubusercontent.com/tscircuit/docs/main/ai-docs/FULL_DOCS.md"
+
+const DOCS_LOCAL_PATH = path.resolve(
+  // Works whether this file is in lib/ or dist/lib/
+  path.dirname(new URL(import.meta.url).pathname),
+  "..",
+  "assets",
+  "tscircuit-docs.md",
+)
+
+let _cachedDocs: string | null = null
+
+/**
+ * Loads the auto-generated tscircuit component docs.
+ * Tries the local asset first, then falls back to a live network fetch.
+ * Results are memoised in-process.
+ */
+export async function fetchAutoGeneratedDocs(): Promise<string> {
+  if (_cachedDocs !== null) return _cachedDocs
+
+  // 1. Try local cache
+  try {
+    const local = await fs.readFile(DOCS_LOCAL_PATH, "utf-8")
+    if (local.trim()) {
+      _cachedDocs = local
+      return _cachedDocs
+    }
+  } catch {
+    // File doesn't exist — fall through to network fetch
+  }
+
+  // 2. Try remote fetch
+  try {
+    const res = await fetch(DOCS_REMOTE_URL)
+    if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`)
+    _cachedDocs = await res.text()
+  } catch (err) {
+    console.warn(
+      `[get-system-prompt] Warning: could not load auto-generated docs.\n` +
+        `  Local path: ${DOCS_LOCAL_PATH}\n` +
+        `  Remote URL: ${DOCS_REMOTE_URL}\n`,
+      err,
+    )
+    _cachedDocs = ""
+  }
+
+  return _cachedDocs
+}
+
+/** Clear the in-process docs cache (useful in tests). */
+export function clearDocsCache() {
+  _cachedDocs = null
+}
+
+/**
+ * Returns the full system prompt string with auto-generated docs embedded.
+ */
+export async function getSystemPrompt(): Promise<string> {
+  const docs = await fetchAutoGeneratedDocs()
+
+  const docsSection = docs.trim()
+    ? `\n# tscircuit Component Reference (Auto-Generated)\n\n${docs.trim()}\n`
+    : ""
+
+  return `You are an expert electronics engineer using tscircuit — a TypeScript/React library that lets you design circuits with JSX syntax. Circuits you write are compiled to schematics, PCB layouts, and netlists.
+${docsSection}
+# Rules
+
+1. Always output a **single TSX code block** — no prose before or after.
+2. Wrap everything in a \`<board />\` with realistic dimensions in millimetres, e.g. \`<board width={20} height={20}>\`.
+3. Use standard reference designators: R1, C1, L1, U1, D1, LED1, Q1, SW1, J1 …
+4. Connect pins with \`<trace from=".R1 > .pin1" to=".C1 > .plus" />\`.
+   Selector format: \`".<refdes> > .<pinLabel>"\`
+5. Use metric SMD footprints for passives: 0402, 0603, 0805, 1206.
+6. Export the circuit as the **default export**.
+7. Do not add explanatory comments unless essential.
+
+# Minimal example
+
+\`\`\`tsx
+export default () => (
+  <board width={20} height={20}>
+    <resistor name="R1" resistance="10kohm" footprint="0402" schX={0} schY={0} />
+    <led name="LED1" color="red" footprint="0402" schX={3} schY={0} />
+    <trace from=".R1 > .pin2" to=".LED1 > .anode" />
+  </board>
+)
+\`\`\`
+`
+}

lib/getSystemPrompt.ts

@@ -0,0 +1,48 @@
+const TSCIRCUIT_DOCS_URL =
+  "https://raw.githubusercontent.com/tscircuit/docs/main/ai-docs/FULL_DOCS.md"
+
+let cachedDocs: string | null = null
+
+/**
+ * Fetches the auto-generated tscircuit documentation from the docs repo.
+ * Falls back to a minimal inline doc string if the fetch fails.
+ */
+export async function getAutoGeneratedDocs(): Promise<string> {
+  if (cachedDocs !== null) return cachedDocs
+
+  try {
+    const response = await fetch(TSCIRCUIT_DOCS_URL)
+    if (!response.ok) {
+      throw new Error(`Failed to fetch docs: ${response.status} ${response.statusText}`)
+    }
+    cachedDocs = await response.text()
+    return cachedDocs
+  } catch (e) {
+    console.warn("[getSystemPrompt] Could not fetch auto-generated docs:", e)
+    cachedDocs = ""
+    return cachedDocs
+  }
+}
+
+/**
+ * Returns the full system prompt for the tscircuit AI benchmark,
+ * incorporating the latest auto-generated component documentation.
+ */
+export async function getSystemPrompt(): Promise<string> {
+  const autoGeneratedDocs = await getAutoGeneratedDocs()
+
+  return `You are an expert tscircuit developer. tscircuit lets you design electronics using React/TypeScript (TSX). Circuits are expressed as JSX trees and can be converted to schematics, PCB layouts, and netlists.
+
+${autoGeneratedDocs}
+
+# Instructions
+
+- Output valid TSX that can be run directly with tscircuit
+- Always wrap your circuit in a \`<board />\` element with appropriate width/height (in mm)
+- Use descriptive reference designators (R1, C1, U1, LED1, etc.)
+- Connect component pins using \`<trace from="..." to="..." />\`
+- The \`from\` / \`to\` selector format is \`".<refdes> > .<pinLabel>"\` e.g. \`".R1 > .pin1"\`
+- Export your circuit as the default export
+- Do not add unnecessary comments or explanation — just output the TSX code block
+`
+}

lib/system-prompt.ts

@@ -0,0 +1,43 @@
+import fs from "node:fs"
+import path from "node:path"
+import { getBaseSystemPrompt } from "./base-system-prompt"
+
+const DOCS_PATH = path.join(
+  import.meta.dirname ?? __dirname,
+  "../assets/tscircuit-docs.md",
+)
+
+/**
+ * Returns the base system prompt without docs appended.
+ *
+ * If you need the prompt with auto-generated component docs included, use
+ * `getSystemPromptWithCachedDocs()` (which wraps docs in a `<tscircuit_docs>`
+ * block) or the async `getSystemPrompt()` from `lib/get-system-prompt.ts`.
+ */
+export function getSystemPromptBase(): string {
+  return getBaseSystemPrompt()
+}
+
+/**
+ * Returns the base system prompt with auto-generated tscircuit component docs
+ * appended (read from the local `assets/tscircuit-docs.md` cache).
+ *
+ * The docs are wrapped in a `<tscircuit_docs>` XML block so that models can
+ * clearly distinguish the reference material from the instructions.
+ *
+ * Run `npx tsx scripts/update-docs.ts` to refresh the cached docs file.
+ */
+export function getSystemPromptWithCachedDocs(): string {
+  const base = getBaseSystemPrompt()
+
+  if (!fs.existsSync(DOCS_PATH)) {
+    console.warn(
+      "[system-prompt] assets/tscircuit-docs.md not found — returning base prompt without docs. " +
+        "Run `npx tsx scripts/update-docs.ts` to generate it.",
+    )
+    return base
+  }
+
+  const docs = fs.readFileSync(DOCS_PATH, "utf-8")
+  return `${base}\n<tscircuit_docs>\n${docs}\n</tscircuit_docs>\n`
+}

scripts/update-docs-cache.ts

@@ -0,0 +1,49 @@
+/**
+ * scripts/update-docs-cache.ts
+ *
+ * Downloads the latest auto-generated tscircuit docs and writes them to
+ * assets/tscircuit-docs.md so the system prompt can reference them without a
+ * network call at evaluation time.
+ *
+ * Usage:
+ *   npx tsx scripts/update-docs-cache.ts
+ *   # or via package.json:
+ *   bun run update-docs
+ */
+
+import fs from "node:fs/promises"
+import path from "node:path"
+
+const DOCS_URL =
+  "https://raw.githubusercontent.com/tscircuit/docs/main/ai-docs/FULL_DOCS.md"
+
+const OUTPUT_PATH = path.resolve(
+  path.dirname(new URL(import.meta.url).pathname),
+  "..",
+  "assets",
+  "tscircuit-docs.md",
+)
+
+async function main() {
+  console.log(`Fetching docs from:\n  ${DOCS_URL}\n`)
+
+  const res = await fetch(DOCS_URL)
+  if (!res.ok) {
+    throw new Error(`Failed to fetch docs: ${res.status} ${res.statusText}`)
+  }
+
+  const content = await res.text()
+
+  await fs.mkdir(path.dirname(OUTPUT_PATH), { recursive: true })
+  await fs.writeFile(OUTPUT_PATH, content, "utf-8")
+
+  const lines = content.split("\n").length
+  console.log(
+    `✓ Wrote ${lines} lines (${content.length} bytes) to:\n  ${OUTPUT_PATH}`,
+  )
+}
+
+main().catch((err) => {
+  console.error("Error updating docs cache:", err)
+  process.exit(1)
+})

scripts/update-docs.ts

@@ -0,0 +1,35 @@
+import fs from "node:fs"
+import path from "node:path"
+
+const DOCS_URL =
+  "https://raw.githubusercontent.com/tscircuit/tscircuit/main/docs/COMPONENTS.md"
+
+const ASSETS_DIR = path.join(import.meta.dirname ?? __dirname, "../assets")
+const OUTPUT_PATH = path.join(ASSETS_DIR, "tscircuit-docs.md")
+
+async function fetchDocs(): Promise<string> {
+  const response = await fetch(DOCS_URL)
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch docs: ${response.status} ${response.statusText}`,
+    )
+  }
+  return response.text()
+}
+
+async function main() {
+  console.log("Fetching tscircuit auto-generated docs...")
+  const docs = await fetchDocs()
+
+  if (!fs.existsSync(ASSETS_DIR)) {
+    fs.mkdirSync(ASSETS_DIR, { recursive: true })
+  }
+
+  fs.writeFileSync(OUTPUT_PATH, docs, "utf-8")
+  console.log(`Docs written to ${OUTPUT_PATH} (${docs.length} bytes)`)
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})