diff --git a/.gitignore b/.gitignore index c49e8d7..59f96b3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist/ *.js.map .worktrees/ +memory/ diff --git a/src/tools/estimate-cost.ts b/src/tools/estimate-cost.ts index 327491a..f7477c7 100644 --- a/src/tools/estimate-cost.ts +++ b/src/tools/estimate-cost.ts @@ -31,11 +31,11 @@ const PREFLIGHT_TOOLS = new Set([ // ── Helpers ───────────────────────────────────────────────────────────────── -function estimateTokens(text: string): number { +export function estimateTokens(text: string): number { return Math.ceil(text.length / 4); } -function extractText(content: unknown): string { +export function extractText(content: unknown): string { if (typeof content === "string") return content; if (Array.isArray(content)) { return content @@ -46,25 +46,25 @@ function extractText(content: unknown): string { return ""; } -function extractToolNames(content: unknown): string[] { +export function extractToolNames(content: unknown): string[] { if (!Array.isArray(content)) return []; return content .filter((b: any) => b.type === "tool_use" && b.name) .map((b: any) => b.name as string); } -function formatTokens(n: number): string { +export function formatTokens(n: number): string { if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`; if (n >= 1_000) return `${(n / 1_000).toFixed(1)}k`; return String(n); } -function formatCost(dollars: number): string { +export function formatCost(dollars: number): string { if (dollars < 0.01) return `<$0.01`; return `$${dollars.toFixed(2)}`; } -function formatDuration(ms: number): string { +export function formatDuration(ms: number): string { const mins = Math.floor(ms / 60_000); if (mins < 60) return `${mins}m`; const hours = Math.floor(mins / 60); @@ -72,7 +72,7 @@ function formatDuration(ms: number): string { return `${hours}h ${rem}m`; } -interface SessionAnalysis { +export interface SessionAnalysis { inputTokens: number; outputTokens: number; promptCount: number; @@ -85,7 +85,7 @@ interface SessionAnalysis { lastTimestamp: string | null; } -function analyzeSessionFile(filePath: string): SessionAnalysis { +export function analyzeSessionFile(filePath: string): SessionAnalysis { const content = readFileSync(filePath, "utf-8"); const lines = content.trim().split("\n").filter(Boolean); diff --git a/src/tools/prompt-score.ts b/src/tools/prompt-score.ts index 1cecf01..9e91532 100644 --- a/src/tools/prompt-score.ts +++ b/src/tools/prompt-score.ts @@ -40,7 +40,7 @@ interface ScoreResult { feedback: string[]; } -function scorePrompt(text: string): ScoreResult { +export function scorePrompt(text: string): ScoreResult { const feedback: string[] = []; let specificity: number; let scope: number; diff --git a/tests/tools/estimate-cost.test.ts b/tests/tools/estimate-cost.test.ts new file mode 100644 index 0000000..ac27da4 --- /dev/null +++ b/tests/tools/estimate-cost.test.ts @@ -0,0 +1,211 @@ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { writeFileSync, mkdtempSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { + estimateTokens, + extractText, + extractToolNames, + formatTokens, + formatCost, + formatDuration, + analyzeSessionFile, +} from "../../src/tools/estimate-cost.js"; + +// ── Unit: estimateTokens ──────────────────────────────────────────────────── + +describe("estimateTokens", () => { + it("returns ~1 token per 4 chars", () => { + expect(estimateTokens("abcd")).toBe(1); + expect(estimateTokens("abcde")).toBe(2); // ceil(5/4) + }); + + it("returns 0 for empty string", () => { + expect(estimateTokens("")).toBe(0); + }); +}); + +// ── Unit: extractText ─────────────────────────────────────────────────────── + +describe("extractText", () => { + it("returns string content as-is", () => { + expect(extractText("hello world")).toBe("hello world"); + }); + + it("extracts text from content block arrays", () => { + const blocks = [ + { type: "text", text: "hello" }, + { type: "text", text: "world" }, + ]; + expect(extractText(blocks)).toBe("hello\nworld"); + }); + + it("filters out non-text blocks", () => { + const blocks = [ + { type: "text", text: "hello" }, + { type: "tool_use", name: "foo", input: {} }, + ]; + expect(extractText(blocks)).toBe("hello"); + }); + + it("returns empty string for unknown types", () => { + expect(extractText(42)).toBe(""); + expect(extractText(null)).toBe(""); + expect(extractText(undefined)).toBe(""); + }); +}); + +// ── Unit: extractToolNames ────────────────────────────────────────────────── + +describe("extractToolNames", () => { + it("extracts tool names from content blocks", () => { + const blocks = [ + { type: "text", text: "thinking..." }, + { type: "tool_use", name: "preflight_check", input: {} }, + { type: "tool_use", name: "scope_work", input: {} }, + ]; + expect(extractToolNames(blocks)).toEqual(["preflight_check", "scope_work"]); + }); + + it("returns empty array for non-array input", () => { + expect(extractToolNames("hello")).toEqual([]); + expect(extractToolNames(null)).toEqual([]); + }); + + it("skips tool_use blocks without name", () => { + const blocks = [{ type: "tool_use", input: {} }]; + expect(extractToolNames(blocks)).toEqual([]); + }); +}); + +// ── Unit: formatTokens ────────────────────────────────────────────────────── + +describe("formatTokens", () => { + it("formats millions", () => { + expect(formatTokens(1_500_000)).toBe("1.5M"); + }); + + it("formats thousands", () => { + expect(formatTokens(12_500)).toBe("12.5k"); + }); + + it("formats small numbers as-is", () => { + expect(formatTokens(500)).toBe("500"); + }); +}); + +// ── Unit: formatCost ──────────────────────────────────────────────────────── + +describe("formatCost", () => { + it("formats normal costs", () => { + expect(formatCost(1.5)).toBe("$1.50"); + }); + + it("shows <$0.01 for tiny costs", () => { + expect(formatCost(0.005)).toBe("<$0.01"); + }); +}); + +// ── Unit: formatDuration ──────────────────────────────────────────────────── + +describe("formatDuration", () => { + it("formats minutes", () => { + expect(formatDuration(300_000)).toBe("5m"); + }); + + it("formats hours and minutes", () => { + expect(formatDuration(5_400_000)).toBe("1h 30m"); + }); +}); + +// ── Integration: analyzeSessionFile ───────────────────────────────────────── + +describe("analyzeSessionFile", () => { + let tmpDir: string; + + beforeAll(() => { + tmpDir = mkdtempSync(join(tmpdir(), "pf-estimate-cost-")); + }); + + afterAll(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it("parses a basic session with user and assistant messages", () => { + const lines = [ + JSON.stringify({ type: "user", timestamp: "2026-01-01T00:00:00Z", message: { content: "Fix the bug in auth.ts" } }), + JSON.stringify({ type: "assistant", timestamp: "2026-01-01T00:01:00Z", message: { content: [{ type: "text", text: "I'll fix that for you." }] } }), + ]; + const file = join(tmpDir, "basic.jsonl"); + writeFileSync(file, lines.join("\n")); + + const result = analyzeSessionFile(file); + expect(result.promptCount).toBe(1); + expect(result.inputTokens).toBeGreaterThan(0); + expect(result.outputTokens).toBeGreaterThan(0); + expect(result.corrections).toBe(0); + expect(result.firstTimestamp).toBe("2026-01-01T00:00:00Z"); + expect(result.lastTimestamp).toBe("2026-01-01T00:01:00Z"); + }); + + it("detects corrections", () => { + const lines = [ + JSON.stringify({ type: "user", timestamp: "2026-01-01T00:00:00Z", message: { content: "Rename the variable" } }), + JSON.stringify({ type: "assistant", timestamp: "2026-01-01T00:01:00Z", message: { content: [{ type: "text", text: "Done, renamed foo to bar." }] } }), + JSON.stringify({ type: "user", timestamp: "2026-01-01T00:02:00Z", message: { content: "No, not that one. I meant the other variable." } }), + ]; + const file = join(tmpDir, "corrections.jsonl"); + writeFileSync(file, lines.join("\n")); + + const result = analyzeSessionFile(file); + expect(result.corrections).toBe(1); + expect(result.wastedOutputTokens).toBeGreaterThan(0); + }); + + it("counts tool calls and preflight calls", () => { + const lines = [ + JSON.stringify({ type: "user", timestamp: "2026-01-01T00:00:00Z", message: { content: "Check this prompt" } }), + JSON.stringify({ + type: "assistant", + timestamp: "2026-01-01T00:01:00Z", + message: { + content: [ + { type: "tool_use", name: "preflight_check", input: { prompt: "test" } }, + { type: "tool_use", name: "Read", input: { path: "foo.ts" } }, + { type: "text", text: "Here are the results." }, + ], + }, + }), + ]; + const file = join(tmpDir, "tools.jsonl"); + writeFileSync(file, lines.join("\n")); + + const result = analyzeSessionFile(file); + expect(result.toolCallCount).toBe(2); + expect(result.preflightCalls).toBe(1); + expect(result.preflightTokens).toBeGreaterThan(0); + }); + + it("handles empty files", () => { + const file = join(tmpDir, "empty.jsonl"); + writeFileSync(file, ""); + + const result = analyzeSessionFile(file); + expect(result.promptCount).toBe(0); + expect(result.inputTokens).toBe(0); + expect(result.outputTokens).toBe(0); + }); + + it("handles malformed JSON lines gracefully", () => { + const lines = [ + "not json", + JSON.stringify({ type: "user", timestamp: "2026-01-01T00:00:00Z", message: { content: "hello" } }), + "{broken", + ]; + const file = join(tmpDir, "malformed.jsonl"); + writeFileSync(file, lines.join("\n")); + + const result = analyzeSessionFile(file); + expect(result.promptCount).toBe(1); + }); +}); diff --git a/tests/tools/prompt-score.test.ts b/tests/tools/prompt-score.test.ts new file mode 100644 index 0000000..ca378ae --- /dev/null +++ b/tests/tools/prompt-score.test.ts @@ -0,0 +1,120 @@ +import { describe, it, expect } from "vitest"; + +// We need to extract scorePrompt for testing. Since it's not exported, +// we'll test the logic by reimporting the module internals. +// For now, let's replicate the scoring logic to verify behavior, +// then refactor the source to export scorePrompt. + +// Actually — let's just refactor the source to export scorePrompt first. +// This test file assumes the refactored version. + +import { scorePrompt } from "../../src/tools/prompt-score.js"; + +describe("scorePrompt", () => { + describe("specificity", () => { + it("scores 25 for prompts with file paths", () => { + const result = scorePrompt("Fix the bug in src/tools/prompt-score.ts"); + expect(result.specificity).toBe(25); + }); + + it("scores 25 for prompts with backtick identifiers", () => { + const result = scorePrompt("Rename `handleClick` to `onSubmit`"); + expect(result.specificity).toBe(25); + }); + + it("scores 15 for generic component mentions", () => { + const result = scorePrompt("Update the function"); + expect(result.specificity).toBe(15); + }); + + it("scores 5 for completely vague prompts", () => { + const result = scorePrompt("make it better"); + expect(result.specificity).toBe(5); + }); + }); + + describe("scope", () => { + it("scores 25 for bounded tasks", () => { + const result = scorePrompt("Only change this one line"); + expect(result.scope).toBe(25); + }); + + it("scores 10 for overly broad scope", () => { + const result = scorePrompt("Fix all bugs"); + expect(result.scope).toBe(10); + }); + }); + + describe("actionability", () => { + it("scores 25 for specific action verbs", () => { + const result = scorePrompt("Rename the variable to camelCase"); + expect(result.actionability).toBe(25); + }); + + it("scores 15 for vague verbs like 'make'", () => { + const result = scorePrompt("Make the code work"); + expect(result.actionability).toBe(15); + }); + + it("scores 5 for no verb at all", () => { + const result = scorePrompt("the button color"); + expect(result.actionability).toBe(5); + }); + }); + + describe("done condition", () => { + it("scores 25 for prompts with verifiable outcomes", () => { + const result = scorePrompt("Fix it so the test should pass"); + expect(result.doneCondition).toBe(25); + }); + + it("scores 20 for questions", () => { + const result = scorePrompt("Why is this breaking?"); + expect(result.doneCondition).toBe(20); + }); + + it("scores 5 for no done condition", () => { + const result = scorePrompt("Refactor the code"); + expect(result.doneCondition).toBe(5); + }); + }); + + describe("grading", () => { + it("gives A+ for perfect prompts", () => { + // File path (25) + bounded (25) + action verb (25) + outcome (25) = 100 + const result = scorePrompt( + "Fix the bug in `src/server.ts` — only the validation check should return a 400 error" + ); + expect(result.total).toBe(100); + expect(result.grade).toBe("A+"); + }); + + it("gives F for terrible prompts", () => { + const result = scorePrompt("stuff"); + expect(result.total).toBeLessThanOrEqual(45); + expect(result.grade).toBe("F"); + }); + + it("includes feedback for low-scoring dimensions", () => { + const result = scorePrompt("stuff"); + expect(result.feedback.length).toBeGreaterThan(0); + expect(result.feedback.some((f) => f.includes("📁"))).toBe(true); + }); + + it("gives praise for perfect scores", () => { + const result = scorePrompt( + "Fix the bug in `src/server.ts` — only the validation check should return a 400 error" + ); + expect(result.feedback[0]).toContain("🏆"); + }); + }); + + describe("total calculation", () => { + it("total equals sum of all dimensions", () => { + const result = scorePrompt("Add a test for the parser module"); + expect(result.total).toBe( + result.specificity + result.scope + result.actionability + result.doneCondition + ); + }); + }); +});