Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,13 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [18, 20]
node-version: [20, 22]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- run: npm ci
- run: npm ci || npm ci --ignore-optional
- run: npm run build
- run: npm run lint
- run: npm test
6 changes: 6 additions & 0 deletions memory/2026-03-04.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# 2026-03-04

## Dev Sprint

- Closed issues #7, #8, #9, #13, #14 — all were already implemented but never closed
- Shipped Ollama embedding support (PR #70, closes #6): new `OllamaEmbeddingProvider` with batch support, config via yml or env vars, 2 new tests (57 total passing)
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"preflight-dev": "./bin/cli.js"
},
"engines": {
"node": ">=18"
"node": ">=20"
},
"scripts": {
"build": "tsc",
Expand Down
16 changes: 8 additions & 8 deletions src/tools/estimate-cost.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ const PREFLIGHT_TOOLS = new Set([

// ── Helpers ─────────────────────────────────────────────────────────────────

function estimateTokens(text: string): number {
export function estimateTokens(text: string): number {
return Math.ceil(text.length / 4);
}

function extractText(content: unknown): string {
export function extractText(content: unknown): string {
if (typeof content === "string") return content;
if (Array.isArray(content)) {
return content
Expand All @@ -46,33 +46,33 @@ function extractText(content: unknown): string {
return "";
}

function extractToolNames(content: unknown): string[] {
export function extractToolNames(content: unknown): string[] {
if (!Array.isArray(content)) return [];
return content
.filter((b: any) => b.type === "tool_use" && b.name)
.map((b: any) => b.name as string);
}

function formatTokens(n: number): string {
export function formatTokens(n: number): string {
if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
if (n >= 1_000) return `${(n / 1_000).toFixed(1)}k`;
return String(n);
}

function formatCost(dollars: number): string {
export function formatCost(dollars: number): string {
if (dollars < 0.01) return `<$0.01`;
return `$${dollars.toFixed(2)}`;
}

function formatDuration(ms: number): string {
export function formatDuration(ms: number): string {
const mins = Math.floor(ms / 60_000);
if (mins < 60) return `${mins}m`;
const hours = Math.floor(mins / 60);
const rem = mins % 60;
return `${hours}h ${rem}m`;
}

interface SessionAnalysis {
export interface SessionAnalysis {
inputTokens: number;
outputTokens: number;
promptCount: number;
Expand All @@ -85,7 +85,7 @@ interface SessionAnalysis {
lastTimestamp: string | null;
}

function analyzeSessionFile(filePath: string): SessionAnalysis {
export function analyzeSessionFile(filePath: string): SessionAnalysis {
const content = readFileSync(filePath, "utf-8");
const lines = content.trim().split("\n").filter(Boolean);

Expand Down
2 changes: 1 addition & 1 deletion src/tools/prompt-score.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ interface ScoreResult {
feedback: string[];
}

function scorePrompt(text: string): ScoreResult {
export function scorePrompt(text: string): ScoreResult {
const feedback: string[] = [];
let specificity: number;
let scope: number;
Expand Down
217 changes: 217 additions & 0 deletions tests/estimate-cost.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,217 @@
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import { writeFileSync, mkdirSync, rmSync } from "node:fs";
import { join } from "node:path";
import {
estimateTokens,
extractText,
extractToolNames,
formatTokens,
formatCost,
formatDuration,
analyzeSessionFile,
} from "../src/tools/estimate-cost.js";

// ── Pure helpers ────────────────────────────────────────────────────────────

describe("estimateTokens", () => {
it("estimates ~1 token per 4 chars", () => {
expect(estimateTokens("abcd")).toBe(1);
expect(estimateTokens("abcde")).toBe(2); // ceil(5/4)
expect(estimateTokens("")).toBe(0);
});

it("handles long strings", () => {
const text = "x".repeat(4000);
expect(estimateTokens(text)).toBe(1000);
});
});

describe("extractText", () => {
it("returns string content directly", () => {
expect(extractText("hello")).toBe("hello");
});

it("extracts text from content block arrays", () => {
const blocks = [
{ type: "text", text: "line 1" },
{ type: "text", text: "line 2" },
];
expect(extractText(blocks)).toBe("line 1\nline 2");
});

it("skips non-text blocks", () => {
const blocks = [
{ type: "text", text: "hello" },
{ type: "tool_use", name: "foo", input: {} },
];
expect(extractText(blocks)).toBe("hello");
});

it("returns empty for null/undefined/numbers", () => {
expect(extractText(null)).toBe("");
expect(extractText(undefined)).toBe("");
expect(extractText(42)).toBe("");
});

it("returns empty for empty array", () => {
expect(extractText([])).toBe("");
});
});

describe("extractToolNames", () => {
it("extracts tool names from content blocks", () => {
const blocks = [
{ type: "text", text: "thinking..." },
{ type: "tool_use", name: "preflight_check", input: {} },
{ type: "tool_use", name: "scope_work", input: {} },
];
expect(extractToolNames(blocks)).toEqual(["preflight_check", "scope_work"]);
});

it("returns empty for non-array input", () => {
expect(extractToolNames("hello")).toEqual([]);
expect(extractToolNames(null)).toEqual([]);
});

it("skips blocks without name", () => {
const blocks = [{ type: "tool_use" }];
expect(extractToolNames(blocks)).toEqual([]);
});
});

describe("formatTokens", () => {
it("formats millions", () => {
expect(formatTokens(1_500_000)).toBe("1.5M");
});

it("formats thousands", () => {
expect(formatTokens(42_000)).toBe("42.0k");
});

it("formats small numbers as-is", () => {
expect(formatTokens(500)).toBe("500");
expect(formatTokens(0)).toBe("0");
});
});

describe("formatCost", () => {
it("formats dollars with 2 decimals", () => {
expect(formatCost(1.5)).toBe("$1.50");
expect(formatCost(0.05)).toBe("$0.05");
});

it("shows <$0.01 for tiny amounts", () => {
expect(formatCost(0.001)).toBe("<$0.01");
expect(formatCost(0)).toBe("<$0.01");
});
});

describe("formatDuration", () => {
it("formats minutes", () => {
expect(formatDuration(5 * 60_000)).toBe("5m");
expect(formatDuration(0)).toBe("0m");
});

it("formats hours and minutes", () => {
expect(formatDuration(90 * 60_000)).toBe("1h 30m");
expect(formatDuration(120 * 60_000)).toBe("2h 0m");
});
});

// ── analyzeSessionFile ──────────────────────────────────────────────────────

describe("analyzeSessionFile", () => {
const tmpDir = join(import.meta.dirname ?? ".", ".tmp-test-sessions");

beforeAll(() => {
mkdirSync(tmpDir, { recursive: true });
});

afterAll(() => {
rmSync(tmpDir, { recursive: true, force: true });
});

function writeSession(name: string, lines: object[]): string {
const path = join(tmpDir, name);
writeFileSync(path, lines.map((l) => JSON.stringify(l)).join("\n"));
return path;
}

it("counts user prompts and input tokens", () => {
const path = writeSession("basic.jsonl", [
{ type: "user", message: { content: "Hello world" }, timestamp: "2025-01-01T00:00:00Z" },
{ type: "assistant", message: { content: "Hi there!" }, timestamp: "2025-01-01T00:01:00Z" },
]);
const result = analyzeSessionFile(path);
expect(result.promptCount).toBe(1);
expect(result.inputTokens).toBeGreaterThan(0);
expect(result.outputTokens).toBeGreaterThan(0);
});

it("detects corrections", () => {
const path = writeSession("corrections.jsonl", [
{ type: "user", message: { content: "Fix the login" }, timestamp: "2025-01-01T00:00:00Z" },
{ type: "assistant", message: { content: "I updated the login page with new styles" }, timestamp: "2025-01-01T00:01:00Z" },
{ type: "user", message: { content: "No, that's not what I meant, revert that" }, timestamp: "2025-01-01T00:02:00Z" },
]);
const result = analyzeSessionFile(path);
expect(result.corrections).toBe(1);
expect(result.wastedOutputTokens).toBeGreaterThan(0);
});

it("counts tool calls", () => {
const path = writeSession("tools.jsonl", [
{ type: "user", message: { content: "Check the code" }, timestamp: "2025-01-01T00:00:00Z" },
{
type: "assistant",
message: {
content: [
{ type: "text", text: "Let me check..." },
{ type: "tool_use", name: "preflight_check", input: { task: "review" } },
],
},
timestamp: "2025-01-01T00:01:00Z",
},
]);
const result = analyzeSessionFile(path);
expect(result.toolCallCount).toBe(1);
expect(result.preflightCalls).toBe(1);
expect(result.preflightTokens).toBeGreaterThan(0);
});

it("tracks timestamps for duration", () => {
const path = writeSession("duration.jsonl", [
{ type: "user", message: { content: "start" }, timestamp: "2025-01-01T10:00:00Z" },
{ type: "assistant", message: { content: "done" }, timestamp: "2025-01-01T11:30:00Z" },
]);
const result = analyzeSessionFile(path);
expect(result.firstTimestamp).toBe("2025-01-01T10:00:00Z");
expect(result.lastTimestamp).toBe("2025-01-01T11:30:00Z");
});

it("handles empty file gracefully", () => {
const path = writeSession("empty.jsonl", []);
const result = analyzeSessionFile(path);
expect(result.promptCount).toBe(0);
expect(result.inputTokens).toBe(0);
expect(result.outputTokens).toBe(0);
expect(result.firstTimestamp).toBeNull();
});

it("skips malformed JSON lines", () => {
const path = join(tmpDir, "malformed.jsonl");
writeFileSync(path, 'not json\n{"type":"user","message":{"content":"hi"},"timestamp":"2025-01-01T00:00:00Z"}\n');
const result = analyzeSessionFile(path);
expect(result.promptCount).toBe(1);
});

it("handles tool_result messages", () => {
const path = writeSession("tool-result.jsonl", [
{ type: "user", message: { content: "check" }, timestamp: "2025-01-01T00:00:00Z" },
{ type: "tool_result", content: "Tool output here", tool_use_id: "abc123" },
]);
const result = analyzeSessionFile(path);
// tool_result content counts as input tokens
expect(result.inputTokens).toBeGreaterThan(0);
});
});
Loading
Loading