diff --git a/desktop/app.go b/desktop/app.go index e0b2bc738..8a4e23f46 100644 --- a/desktop/app.go +++ b/desktop/app.go @@ -615,6 +615,22 @@ type HistoryMessage struct { Role string `json:"role"` Content string `json:"content"` Reasoning string `json:"reasoning,omitempty"` + // ToolCalls carries the tool calls an assistant message requested. Each + // entry mirrors a provider.ToolCall so the frontend can reconstruct tool + // cards in historical sessions. + ToolCalls []HistoryToolCall `json:"toolCalls,omitempty"` + // ToolCallID is set on tool-result messages (role:"tool") so the frontend + // can match results back to their dispatch cards. + ToolCallID string `json:"toolCallId,omitempty"` + // ToolName is set on tool-result messages for display. + ToolName string `json:"toolName,omitempty"` +} + +// HistoryToolCall is one tool call from a historical assistant message. +type HistoryToolCall struct { + ID string `json:"id"` + Name string `json:"name"` + Arguments string `json:"arguments"` } // History returns the session's message log. @@ -640,7 +656,22 @@ func historyMessages(msgs []provider.Message, resolveUserContent func(string) st if m.Role == provider.RoleAssistant { reasoning = m.ReasoningContent } - out = append(out, HistoryMessage{Role: string(m.Role), Content: content, Reasoning: reasoning}) + hm := HistoryMessage{Role: string(m.Role), Content: content, Reasoning: reasoning} + // Preserve tool calls from assistant messages so the frontend can + // reconstruct tool cards. + if m.Role == provider.RoleAssistant && len(m.ToolCalls) > 0 { + hm.ToolCalls = make([]HistoryToolCall, len(m.ToolCalls)) + for i, tc := range m.ToolCalls { + hm.ToolCalls[i] = HistoryToolCall{ID: tc.ID, Name: tc.Name, Arguments: tc.Arguments} + } + } + // Preserve tool result linkage so the frontend can fill in tool card + // output/errors. + if m.Role == provider.RoleTool { + hm.ToolCallID = m.ToolCallID + hm.ToolName = m.Name + } + out = append(out, hm) } return out } diff --git a/desktop/frontend/src/App.tsx b/desktop/frontend/src/App.tsx index 30ac745e3..ff69131b5 100644 --- a/desktop/frontend/src/App.tsx +++ b/desktop/frontend/src/App.tsx @@ -181,6 +181,7 @@ export default function App() { const [workspacePanelMaximized, setWorkspacePanelMaximized] = useState(false); const [workspacePreviewModeActive, setWorkspacePreviewModeActive] = useState(false); const [workspaceChangesRefreshKey, setWorkspaceChangesRefreshKey] = useState(0); + const [workspaceTreeRefreshKey, setWorkspaceTreeRefreshKey] = useState(0); const [settingsOpen, setSettingsOpen] = useState(false); const [capsOpen, setCapsOpen] = useState(false); const [composerInsertRequest, setComposerInsertRequest] = useState(null); @@ -267,6 +268,7 @@ export default function App() { useEffect(() => { if (wasRunningForWorkspaceChangesRef.current && !state.running) { setWorkspaceChangesRefreshKey((key) => key + 1); + setWorkspaceTreeRefreshKey((key) => key + 1); } wasRunningForWorkspaceChangesRef.current = state.running; }, [state.running]); @@ -398,6 +400,7 @@ export default function App() { await newSession(); await refreshSessions(); setWorkspaceChangesRefreshKey((key) => key + 1); + setWorkspaceTreeRefreshKey((key) => key + 1); }, [newSession, refreshSessions]); const toggleSidebar = useCallback(() => { @@ -584,6 +587,7 @@ export default function App() { await resumeSession(path); await refreshSessions(); setWorkspaceChangesRefreshKey((key) => key + 1); + setWorkspaceTreeRefreshKey((key) => key + 1); }, [state.running, resumeSession, refreshSessions], ); @@ -641,6 +645,7 @@ export default function App() { if (picked) { await refreshSessions(); setWorkspaceChangesRefreshKey((key) => key + 1); + setWorkspaceTreeRefreshKey((key) => key + 1); } return picked; }, [pickWorkspace, switchWorkspace, refreshSessions]); @@ -1037,6 +1042,7 @@ export default function App() { onPreviewModeChange={handleWorkspacePreviewModeChange} onAddToChat={addToChat} changesRefreshKey={workspaceChangesRefreshKey} + treeRefreshKey={workspaceTreeRefreshKey} /> diff --git a/desktop/frontend/src/components/HistoryPanel.tsx b/desktop/frontend/src/components/HistoryPanel.tsx index e4deb02b6..1d95ac854 100644 --- a/desktop/frontend/src/components/HistoryPanel.tsx +++ b/desktop/frontend/src/components/HistoryPanel.tsx @@ -285,15 +285,53 @@ function sessionMetaLine(s: SessionMeta, tr: ReturnType): string { } function previewMessagesToItems(messages: HistoryMessage[]): Item[] { - return messages - .filter( - (m) => - (m.role === "user" && m.content.trim() !== "") || - (m.role === "assistant" && (m.content.trim() !== "" || (m.reasoning ?? "").trim() !== "")), - ) - .map((m, i) => - m.role === "user" - ? { kind: "user", id: `hp${i}`, text: m.content } - : { kind: "assistant", id: `hp${i}`, text: m.content, reasoning: m.reasoning ?? "", streaming: false }, - ); + // Build a lookup: toolCallId → tool result content for filling in tool cards. + const resultMap = new Map(); + for (const m of messages) { + if (m.role === "tool" && m.toolCallId && !resultMap.has(m.toolCallId)) { + resultMap.set(m.toolCallId, m.content); + } + } + + const items: Item[] = []; + let seq = 0; + for (const m of messages) { + if (m.role === "system" || m.role === "tool") continue; + if (m.role === "user") { + if (m.content.trim() === "") continue; + items.push({ kind: "user", id: `hp${seq}`, text: m.content }); + seq++; + continue; + } + if (m.role === "assistant") { + const hasText = m.content.trim() !== "" || (m.reasoning ?? "").trim() !== ""; + const toolCalls = m.toolCalls ?? []; + if (hasText) { + items.push({ + kind: "assistant", + id: `hp${seq}`, + text: m.content, + reasoning: m.reasoning ?? "", + streaming: false, + }); + seq++; + } + for (const tc of toolCalls) { + const output = resultMap.get(tc.id) ?? ""; + const hasError = output.startsWith("[error") || output.startsWith("Error:"); + items.push({ + kind: "tool", + id: tc.id || `hpt${seq}`, + name: tc.name, + args: tc.arguments ?? "", + readOnly: false, + status: hasError ? "error" : "done", + output, + error: hasError ? output : undefined, + }); + seq++; + } + } + } + return items; } diff --git a/desktop/frontend/src/components/Transcript.tsx b/desktop/frontend/src/components/Transcript.tsx index 86d6abe01..aea67f3fb 100644 --- a/desktop/frontend/src/components/Transcript.tsx +++ b/desktop/frontend/src/components/Transcript.tsx @@ -147,12 +147,16 @@ export function Transcript({ }, [openTurn]); // Each user message's turn = its ordinal among user messages, so a rewind - // targets the matching checkpoint. - const userTurn = new Map(); - let nt = 0; - for (const it of items) { - if (it.kind === "user") userTurn.set(it.id, nt++); - } + // targets the matching checkpoint. Memoized so it isn't rebuilt on every + // streaming re-render. + const userTurn = useMemo(() => { + const m = new Map(); + let nt = 0; + for (const it of items) { + if (it.kind === "user") m.set(it.id, nt++); + } + return m; + }, [items]); return (
diff --git a/desktop/frontend/src/components/WorkspacePanel.tsx b/desktop/frontend/src/components/WorkspacePanel.tsx index 2ba620d25..46b751eec 100644 --- a/desktop/frontend/src/components/WorkspacePanel.tsx +++ b/desktop/frontend/src/components/WorkspacePanel.tsx @@ -40,6 +40,18 @@ const WORKSPACE_PREVIEW_MIN_WIDTH = 420; const WORKSPACE_CONTEXT_MENU_FILE_HEIGHT = 92; const WORKSPACE_CONTEXT_MENU_REF_HEIGHT = 48; +// Encoding options for the file encoding override selector. +const ENCODING_OPTIONS = [ + { value: "", label: "Auto" }, + { value: "UTF-8", label: "UTF-8" }, + { value: "UTF-8 BOM", label: "UTF-8 BOM" }, + { value: "GB18030", label: "GB18030" }, + { value: "UTF-16 LE", label: "UTF-16 LE" }, + { value: "UTF-16 BE", label: "UTF-16 BE" }, + { value: "UTF-16 LE (no BOM)", label: "UTF-16 LE (no BOM)" }, + { value: "UTF-16 BE (no BOM)", label: "UTF-16 BE (no BOM)" }, +] as const; + function clampWorkspaceTreeWidth(width: number, panelWidth?: number): number { const maxForPanel = typeof panelWidth === "number" && Number.isFinite(panelWidth) @@ -156,6 +168,7 @@ export function WorkspacePanel({ onPreviewModeChange, onAddToChat, changesRefreshKey, + treeRefreshKey, }: { open: boolean; cwd?: string; @@ -166,6 +179,7 @@ export function WorkspacePanel({ onPreviewModeChange?: (active: boolean) => void; onAddToChat?: (text: string) => void; changesRefreshKey?: number; + treeRefreshKey?: number; }) { const t = useT(); const panelRef = useRef(null); @@ -187,6 +201,18 @@ export function WorkspacePanel({ const [treeVisible, setTreeVisible] = useState(true); const [treeWidth, setTreeWidth] = useState(loadWorkspaceTreeWidth); const [treeResizing, setTreeResizing] = useState(false); + const [projectEncoding, setProjectEncoding] = useState(""); + + // Load project encoding from settings on panel open / workspace change. + useEffect(() => { + if (!open) return; + app.Settings().then((s) => setProjectEncoding(s.fileEncoding || "")).catch(() => {}); + }, [cwd, open]); + + const changeProjectEncoding = useCallback(async (enc: string) => { + setProjectEncoding(enc); + await app.SetFileEncoding(enc).catch(() => {}); + }, []); const loadDir = useCallback(async (dir: string) => { const entries = await app.ListDir(dir).catch(() => []); @@ -267,7 +293,7 @@ export function WorkspacePanel({ let live = true; setLoadingPreview(true); app - .ReadFile(selectedPath) + .ReadFile(selectedPath, projectEncoding || "") .then((next) => { if (live) setPreview(next); }) @@ -289,13 +315,29 @@ export function WorkspacePanel({ return () => { live = false; }; - }, [selectedPath]); + }, [selectedPath, projectEncoding]); useEffect(() => { if (!open || !selectedPath) return; return refreshSelected(); }, [open, refreshSelected, selectedPath]); + // Refresh the file tree when the agent modifies files (turn completion, + // new session, resume, workspace switch). Re-fetches all currently open + // directories and refreshes the preview of the selected file. + useEffect(() => { + if (!open || treeRefreshKey === undefined || treeRefreshKey === 0) return; + setEntriesByDir((prev) => { + for (const dir of Object.keys(prev)) { + void loadDir(dir); + } + return prev; + }); + if (selectedPath) { + void refreshSelected(); + } + }, [treeRefreshKey, open, loadDir, selectedPath, refreshSelected]); + const toggleDir = useCallback( (dir: string) => { setOpenDirs((prev) => { @@ -496,7 +538,7 @@ export function WorkspacePanel({ const target = treeMenu; setTreeMenu(null); try { - const file = await app.ReadFile(target.path); + const file = await app.ReadFile(target.path, ""); if (file.err || file.binary) { onAddToChat?.(formatWorkspaceReference(target.path, false)); return; @@ -776,6 +818,20 @@ export function WorkspacePanel({ + + +
diff --git a/desktop/frontend/src/lib/bridge.ts b/desktop/frontend/src/lib/bridge.ts index 416e2aa9c..06e086417 100644 --- a/desktop/frontend/src/lib/bridge.ts +++ b/desktop/frontend/src/lib/bridge.ts @@ -101,7 +101,7 @@ export interface AppBindings { SlashArgs(input: string): Promise; ListDir(rel: string): Promise; SearchFileRefs(query: string): Promise; - ReadFile(rel: string): Promise; + ReadFile(rel: string, enc?: string): Promise; WorkspaceChanges(): Promise; OpenWorkspacePath(rel: string): Promise; RevealWorkspacePath(rel: string): Promise; @@ -139,6 +139,8 @@ export interface AppBindings { // SetBypass toggles YOLO mode (auto-approve every tool call this session; deny // rules still apply). Runtime-only — not written to config. SetBypass(on: boolean): Promise; + // SetFileEncoding sets the project-level file encoding for read/write tools. + SetFileEncoding(encoding: string): Promise; // Auto-updater (desktop/updater_app.go): the injected build version, a manifest // check, applying an update (win/linux self-update; macOS opens the download // page), and opening that page directly. Progress streams on "updater:progress". @@ -392,6 +394,7 @@ function makeMockApp(): AppBindings { configPath: "~/projects/reasonix/reasonix.toml", providerKinds: ["openai"], bypass: false, + fileEncoding: "UTF-8", }; return { async Platform() { @@ -789,7 +792,7 @@ function makeMockApp(): AppBindings { .filter((path) => path.split("/").pop()?.toLowerCase().includes(q)) .map((name) => ({ name, isDir: false })); }, - async ReadFile(rel: string) { + async ReadFile(rel: string, _enc?: string) { const samples: Record = { "README.md": "# Reasonix\n\nBrowser-dev workspace preview.\n\n- Chat in the center\n- Browse files on the right\n- Keep sessions on the left\n", "go.mod": "module reasonix\n\ngo 1.23\n", @@ -940,6 +943,9 @@ function makeMockApp(): AppBindings { async SetBypass(on: boolean) { settings.bypass = on; }, + async SetFileEncoding(encoding: string) { + settings.fileEncoding = encoding; + }, async Version() { return "v1.0.0 (browser dev)"; }, diff --git a/desktop/frontend/src/lib/types.ts b/desktop/frontend/src/lib/types.ts index e0ea3bfb8..c347d6b72 100644 --- a/desktop/frontend/src/lib/types.ts +++ b/desktop/frontend/src/lib/types.ts @@ -45,7 +45,6 @@ export interface WireUsage { cacheHitTokens: number; cacheMissTokens: number; reasoningTokens?: number; - cacheDiagnostics?: WireCacheDiagnostics; // Session-cumulative cache tokens — the status bar shows the aggregate // hit-rate (Σhit/Σ(hit+miss)), steadier than the single-turn cacheHitTokens. sessionCacheHitTokens: number; @@ -53,18 +52,6 @@ export interface WireUsage { costUsd?: number; } -export interface WireCacheDiagnostics { - prefixHash: string; - prefixChanged: boolean; - prefixChangeReasons?: string[]; - systemHash: string; - toolsHash: string; - logRewriteVersion: number; - toolSchemaTokens: number; - cacheMissTokens: number; - cacheHitTokens: number; -} - export interface WireApproval { id: string; tool: string; @@ -115,6 +102,15 @@ export interface HistoryMessage { role: string; content: string; reasoning?: string; + toolCalls?: HistoryToolCall[]; + toolCallId?: string; + toolName?: string; +} + +export interface HistoryToolCall { + id: string; + name: string; + arguments: string; } // CheckpointMeta is one rewind point (a user turn) for the rewind UI. @@ -406,6 +402,7 @@ export interface SettingsView { configPath: string; providerKinds: string[]; // provider implementations the kernel registered (for the kind picker) bypass: boolean; // live YOLO state (runtime-only) — whether approvals are skipped this session + fileEncoding: string; // project-level file encoding (e.g. "UTF-8", "GB18030") } // Auto-updater payloads (desktop/updater.go). UpdateInfo drives the update banner; diff --git a/desktop/frontend/src/lib/useController.ts b/desktop/frontend/src/lib/useController.ts index e751c7095..3e53ebc5c 100644 --- a/desktop/frontend/src/lib/useController.ts +++ b/desktop/frontend/src/lib/useController.ts @@ -416,19 +416,70 @@ function reducer(s: State, a: Action): State { case "jobs": return { ...s, jobs: a.jobs }; case "history": { - // Only user/assistant turns with visible text or assistant reasoning — never - // the system prompt or tool-result messages. - const visible = a.messages.filter( - (m) => - (m.role === "user" && m.content.trim() !== "") || - (m.role === "assistant" && (m.content.trim() !== "" || (m.reasoning ?? "").trim() !== "")), - ); - const items: Item[] = visible.map((m, i) => - m.role === "user" - ? { kind: "user", id: `h${i}`, text: m.content } - : { kind: "assistant", id: `h${i}`, text: m.content, reasoning: m.reasoning ?? "", streaming: false }, - ); - return { ...s, items, seq: s.seq + visible.length }; + // Reconstruct the full transcript from saved messages, including tool + // cards that were previously dropped. System messages are skipped; tool + // result messages are folded into their matching dispatch cards. + const msgs = a.messages; + + // Build a lookup: toolCallId → tool result content for filling in tool + // card output. Multiple tool messages may share one call id (sub-agent + // inner results), but the first match is the canonical one. + const resultMap = new Map(); + for (const m of msgs) { + if (m.role === "tool" && m.toolCallId) { + if (!resultMap.has(m.toolCallId)) { + resultMap.set(m.toolCallId, { content: m.content, name: m.toolName ?? "" }); + } + } + } + + const items: Item[] = []; + let seq = s.seq; + for (let i = 0; i < msgs.length; i++) { + const m = msgs[i]; + if (m.role === "system") continue; + if (m.role === "tool") continue; // folded into tool cards below + if (m.role === "user") { + if (m.content.trim() === "") continue; + items.push({ kind: "user", id: `h${seq}`, text: m.content }); + seq++; + continue; + } + if (m.role === "assistant") { + const hasText = m.content.trim() !== "" || (m.reasoning ?? "").trim() !== ""; + const toolCalls = m.toolCalls ?? []; + // Only emit an assistant bubble if it has visible text or reasoning. + if (hasText) { + items.push({ + kind: "assistant", + id: `h${seq}`, + text: m.content, + reasoning: m.reasoning ?? "", + streaming: false, + }); + seq++; + } + // Reconstruct tool cards from the assistant's tool calls. + for (const tc of toolCalls) { + const result = resultMap.get(tc.id); + const output = result?.content ?? ""; + const hasError = output.startsWith("[error") || output.startsWith("Error:"); + items.push({ + kind: "tool", + id: tc.id || `ht${seq}`, + name: tc.name, + args: tc.arguments ?? "", + readOnly: false, + status: hasError ? "error" : "done", + output, + error: hasError ? output : undefined, + }); + seq++; + } + continue; + } + } + return { ...s, items, seq }; } case "local_notice": return { diff --git a/desktop/frontend/src/locales/en.ts b/desktop/frontend/src/locales/en.ts index c49d861b0..0fc0e9914 100644 --- a/desktop/frontend/src/locales/en.ts +++ b/desktop/frontend/src/locales/en.ts @@ -74,6 +74,10 @@ export const en = { "workspace.sourceSession": "Session", "workspace.sourceGit": "Git", "workspace.deleted": "Deleted", + "workspace.encoding": "Encoding", + "workspace.encodingAuto": "Auto", + "workspace.encodingTitle": "Project encoding: {enc}", + "workspace.encodingOverride": "Change project file encoding", // mcp & skills drawer "caps.title": "MCP & Skills", diff --git a/desktop/frontend/src/locales/zh.ts b/desktop/frontend/src/locales/zh.ts index cd046238e..ebda2e6e2 100644 --- a/desktop/frontend/src/locales/zh.ts +++ b/desktop/frontend/src/locales/zh.ts @@ -75,6 +75,10 @@ export const zh: Record = { "workspace.sourceSession": "会话", "workspace.sourceGit": "Git", "workspace.deleted": "已删除", + "workspace.encoding": "编码", + "workspace.encodingAuto": "自动检测", + "workspace.encodingTitle": "项目编码:{enc}", + "workspace.encodingOverride": "更改项目文件编码", // MCP 与技能抽屉 "caps.title": "MCP 与技能", diff --git a/desktop/frontend/src/styles.css b/desktop/frontend/src/styles.css index 90878fb0e..89b522b63 100644 --- a/desktop/frontend/src/styles.css +++ b/desktop/frontend/src/styles.css @@ -2852,6 +2852,29 @@ body { flex-shrink: 0; font-family: var(--mono); } +.workspace-encoding-select { + flex-shrink: 0; + margin-left: 6px; + padding: 1px 4px; + border: 1px solid var(--border-soft); + border-radius: 5px; + background: var(--bg-soft); + color: var(--fg-dim); + font-family: var(--mono); + font-size: 11px; + line-height: 1.4; + cursor: pointer; + outline: none; + max-width: 160px; +} +.workspace-encoding-select:hover { + border-color: var(--border); + color: var(--fg); +} +.workspace-encoding-select:focus { + border-color: var(--accent, var(--border)); + box-shadow: 0 0 0 1px var(--accent, var(--border)); +} .workspace-preview__body { position: relative; flex: 1 1 auto; diff --git a/desktop/main.go b/desktop/main.go index 8371acda2..25d0c8a7d 100644 --- a/desktop/main.go +++ b/desktop/main.go @@ -8,6 +8,8 @@ package main import ( "embed" + "io" + "log" "github.com/wailsapp/wails/v2" "github.com/wailsapp/wails/v2/pkg/options" @@ -37,6 +39,11 @@ var assets embed.FS var version = "dev" func main() { + // The go-webview2 library uses log.Printf for startup diagnostics (e.g. + // "[WebView2] Environment created successfully"). Discard the default + // logger's output so these one-time messages don't pollute the terminal. + log.SetOutput(io.Discard) + app := NewApp() err := wails.Run(&options.App{ diff --git a/desktop/resource_windows_amd64.syso b/desktop/resource_windows_amd64.syso new file mode 100644 index 000000000..ce2afa86a Binary files /dev/null and b/desktop/resource_windows_amd64.syso differ diff --git a/desktop/settings_app.go b/desktop/settings_app.go index ae825bc22..fae115623 100644 --- a/desktop/settings_app.go +++ b/desktop/settings_app.go @@ -72,6 +72,7 @@ type AgentView struct { type SettingsView struct { DefaultModel string `json:"defaultModel"` PlannerModel string `json:"plannerModel"` + FileEncoding string `json:"fileEncoding"` // project file encoding (e.g. "UTF-8", "GB18030") Providers []ProviderView `json:"providers"` Permissions PermissionsView `json:"permissions"` Sandbox SandboxView `json:"sandbox"` @@ -117,6 +118,7 @@ func (a *App) Settings() SettingsView { v := SettingsView{ DefaultModel: cfg.DefaultModel, PlannerModel: cfg.Agent.PlannerModel, + FileEncoding: cfg.FileEncoding, Providers: []ProviderView{}, Permissions: PermissionsView{ Mode: orDefault(cfg.Permissions.Mode, "ask"), @@ -387,6 +389,15 @@ func (a *App) SetAgentParams(temperature float64, maxSteps int, systemPrompt str }) } +// SetFileEncoding sets the project-level file encoding (e.g. "UTF-8", "GB18030"). +// Empty string means auto-detect. +func (a *App) SetFileEncoding(encoding string) error { + return a.applyConfigChange(func(c *config.Config) error { + c.FileEncoding = strings.TrimSpace(encoding) + return nil + }) +} + // trimList drops blank entries from a string slice (and returns a non-nil slice). func trimList(in []string) []string { out := []string{} diff --git a/desktop/winres/icon256.png b/desktop/winres/icon256.png new file mode 100644 index 000000000..7ace0e91e Binary files /dev/null and b/desktop/winres/icon256.png differ diff --git a/desktop/winres/winres.json b/desktop/winres/winres.json new file mode 100644 index 000000000..472e8834d --- /dev/null +++ b/desktop/winres/winres.json @@ -0,0 +1,46 @@ +{ + "RT_GROUP_ICON": { + "APP": { + "0000": [ + "icon256.png" + ] + } + }, + "RT_MANIFEST": { + "#1": { + "0409": { + "identity": { + "name": "Reasonix", + "version": "0.0.0.0" + }, + "description": "Reasonix Desktop", + "minimum-os": "win10", + "manifest-theme": "dark", + "dpi-awareness": "system", + "auto-elevate": false + } + } + }, + "RT_VERSION": { + "#1": { + "0000": { + "fixed": { + "file_version": "0.0.0.0", + "product_version": "0.0.0.0" + }, + "info": { + "0409": { + "CompanyName": "Reasonix", + "FileDescription": "Reasonix Desktop", + "FileVersion": "0.0.0.0", + "InternalName": "reasonix-desktop", + "LegalCopyright": "Copyright © 2026 Reasonix Contributors", + "OriginalFilename": "reasonix-desktop.exe", + "ProductName": "Reasonix", + "ProductVersion": "0.0.0.0" + } + } + } + } + } +} diff --git a/internal/boot/boot.go b/internal/boot/boot.go index 15e9963ad..2a138e463 100644 --- a/internal/boot/boot.go +++ b/internal/boot/boot.go @@ -16,6 +16,7 @@ import ( "log/slog" "os" "path/filepath" + "runtime" "strings" "reasonix/internal/agent" @@ -186,14 +187,14 @@ func Build(ctx context.Context, opts Options) (*control.Controller, error) { reg := tool.NewRegistry() bashSpec := sandbox.Spec{Mode: cfg.BashMode(), WriteRoots: cfg.WriteRootsForRoot(root), Network: cfg.Sandbox.Network} - if bashSpec.Mode == "enforce" && !sandbox.Available() { + if bashSpec.Mode == "enforce" && !sandbox.Available() && runtime.GOOS != "windows" { fmt.Fprintln(stderr, "warning: bash sandbox requested but unavailable on this platform; running bash unconfined") } if sandbox.ResolveShell().Kind == sandbox.ShellPowerShell { fmt.Fprintln(stderr, "warning: bash not found on PATH; the shell tool will run commands under Windows PowerShell. Install Git for Windows or WSL to use bash.") } searchSpec := builtin.ResolveSearch(cfg.Tools.Search.Engine, cfg.Tools.Search.RgPath, stderr) - addBuiltins(reg, cfg.Tools.Enabled, cfg.WriteRootsForRoot(root), bashSpec, searchSpec, stderr, root) + addBuiltins(reg, cfg.Tools.Enabled, cfg.WriteRootsForRoot(root), bashSpec, searchSpec, stderr, root, cfg.FileEncoding) // Always construct a host, even with no plugins configured, so the controller's // host pointer is stable for the session and `/mcp add` can hot-add into it. pluginHost := plugin.NewHost() @@ -664,12 +665,12 @@ func NewProviderWithProxy(e *config.ProviderEntry, proxy netclient.ProxySpec) (p // instance bound to writeRoots (preserving registry order). // When workDir is non-empty, tools resolve relative paths against it instead of // the process cwd, enabling concurrent multi-project sessions. -func addBuiltins(reg *tool.Registry, enabled, writeRoots []string, bashSpec sandbox.Spec, searchSpec builtin.SearchSpec, stderr io.Writer, workDir string) { +func addBuiltins(reg *tool.Registry, enabled, writeRoots []string, bashSpec sandbox.Spec, searchSpec builtin.SearchSpec, stderr io.Writer, workDir string, fileEncoding string) { // If a workspace directory is set, use workspace-bound tools that resolve // paths relative to that directory. Otherwise fall back to the process-cwd // compile-time builtins. if workDir != "" { - ws := builtin.Workspace{Dir: workDir, WriteRoots: writeRoots, Bash: bashSpec, Search: searchSpec} + ws := builtin.Workspace{Dir: workDir, WriteRoots: writeRoots, Bash: bashSpec, Search: searchSpec, FileEncoding: fileEncoding} for _, t := range ws.Tools(enabled...) { reg.Add(t) } @@ -692,7 +693,7 @@ func addBuiltins(reg *tool.Registry, enabled, writeRoots []string, bashSpec sand // Replace the unconfined defaults with confined instances (registry order is // preserved on replace): file-writers bound to the workspace, bash to the OS // sandbox. Only replace tools actually enabled/present. - confined := append(builtin.ConfineWriters(writeRoots), builtin.ConfineBash(bashSpec), builtin.ConfineSearch(searchSpec)) + confined := append(builtin.ConfineWriters(writeRoots, fileEncoding), builtin.ConfineBash(bashSpec), builtin.ConfineSearch(searchSpec)) for _, t := range confined { if _, ok := reg.Get(t.Name()); ok { reg.Add(t) diff --git a/internal/checkpoint/checkpoint.go b/internal/checkpoint/checkpoint.go index 45814e0db..f3753d1ae 100644 --- a/internal/checkpoint/checkpoint.go +++ b/internal/checkpoint/checkpoint.go @@ -184,6 +184,31 @@ func (s *Store) persist(c *Checkpoint) { } } +// Prune removes all checkpoints with Turn >= fromTurn from the in-memory store +// and deletes their persisted JSON files. This must be called after a code or +// conversation rewind so that stale checkpoints from later turns don't shadow +// new ones when turn numbers are reused. +func (s *Store) Prune(fromTurn int) { + s.mu.Lock() + defer s.mu.Unlock() + kept := s.done[:0] + for _, c := range s.done { + if c.Turn >= fromTurn { + // Delete persisted JSON for this turn. + if s.dir != "" { + os.Remove(filepath.Join(s.dir, fmt.Sprintf("turn-%d.json", c.Turn))) + } + continue + } + kept = append(kept, c) + } + s.done = kept + // Also discard cur if it belongs to a pruned turn. + if s.cur != nil && s.cur.Turn >= fromTurn { + s.cur = nil + } +} + // NextTurn returns the turn number a new checkpoint should take: one past the // highest existing turn (0 when empty), so a resumed session keeps numbering // without colliding with checkpoints loaded from disk. diff --git a/internal/config/config.go b/internal/config/config.go index 9e7fc44bc..f5d107ba0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -40,7 +40,8 @@ func SkillNameKey(name string) string { // Config is Reasonix's runtime configuration. type Config struct { DefaultModel string `toml:"default_model"` - Language string `toml:"language"` // ui/model language tag (e.g. "zh"); empty = auto-detect from $LANG / $REASONIX_LANG + Language string `toml:"language"` // ui/model language tag (e.g. "zh"); empty = auto-detect from $LANG / $REASONIX_LANG + FileEncoding string `toml:"file_encoding"` // project file encoding (e.g. "UTF-8", "GB18030"); empty = auto-detect UI UIConfig `toml:"ui"` Agent AgentConfig `toml:"agent"` Providers []ProviderEntry `toml:"providers"` diff --git a/internal/config/render.go b/internal/config/render.go index d385e181c..a8da9f382 100644 --- a/internal/config/render.go +++ b/internal/config/render.go @@ -23,6 +23,11 @@ func RenderTOML(c *Config) string { } else { b.WriteString("# language = \"zh\" # ui/model language; empty = auto-detect from $LANG / $REASONIX_LANG\n") } + if c.FileEncoding != "" { + fmt.Fprintf(&b, "file_encoding = %q # project file encoding (e.g. \"UTF-8\", \"GB18030\"); empty = auto-detect\n", c.FileEncoding) + } else { + b.WriteString("# file_encoding = \"GB18030\" # project file encoding; empty = auto-detect per file\n") + } b.WriteString("\n") b.WriteString("[ui]\n") diff --git a/internal/control/controller.go b/internal/control/controller.go index 040687295..7412b014f 100644 --- a/internal/control/controller.go +++ b/internal/control/controller.go @@ -782,6 +782,9 @@ func (c *Controller) Rewind(turn int, scope RewindScope) error { if err != nil { return c.rewindFail(fmt.Errorf("rewind code: %w", err)) } + // Prune checkpoints from the rewound turn onward so stale snapshots + // don't shadow new ones when turn numbers are reused after rewind. + c.cp.Prune(turn) c.sink.Emit(event.Event{Kind: event.Notice, Level: event.LevelInfo, Text: fmt.Sprintf("rewound code to turn %d — %d file(s) restored, %d removed", turn, len(written), len(deleted))}) } @@ -794,12 +797,17 @@ func (c *Controller) Rewind(turn int, scope RewindScope) error { s.Messages = s.Messages[:boundary] c.mu.Lock() c.cpTurn = turn // renumber future turns from here; later turns are gone + // Prune boundaries for turns that no longer exist. for k := range c.cpBound { if k >= turn { delete(c.cpBound, k) } } c.mu.Unlock() + // If code wasn't already pruned (RewindConversation only), prune now. + if scope == RewindConversation { + c.cp.Prune(turn) + } if err := c.Snapshot(); err != nil { slog.Warn("controller: snapshot after rewind", "err", err) } diff --git a/internal/fileutil/encoding/encoding.go b/internal/fileutil/encoding/encoding.go index 7fa4d90fa..ff75493d6 100644 --- a/internal/fileutil/encoding/encoding.go +++ b/internal/fileutil/encoding/encoding.go @@ -7,6 +7,7 @@ package encoding import ( "bytes" "encoding/binary" + "strings" "unicode/utf8" "golang.org/x/text/encoding/simplifiedchinese" @@ -249,3 +250,52 @@ func utf16Encode(runes []rune) []uint16 { } return out } + +// Name returns a human-readable label for the encoding kind. +func Name(k Kind) string { + switch k { + case UTF8: + return "UTF-8" + case UTF8BOM: + return "UTF-8 BOM" + case UTF16LE: + return "UTF-16 LE" + case UTF16BE: + return "UTF-16 BE" + case UTF16LENoBOM: + return "UTF-16 LE (no BOM)" + case UTF16BENoBOM: + return "UTF-16 BE (no BOM)" + case GB18030: + return "GB18030" + case LossyUTF8: + return "Lossy UTF-8" + } + return "UTF-8" +} + +// ParseName converts a user-supplied encoding label to a Kind. +// Returns the kind and true when recognised, or UTF8 and false when +// the label is empty or unknown (caller should fall back to auto-detection). +// Accepts common aliases: "gbk" / "gb2312" → GB18030, "utf8" → UTF8, etc. +func ParseName(s string) (Kind, bool) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "": + return UTF8, false + case "utf-8", "utf8": + return UTF8, true + case "utf-8 bom", "utf8 bom", "utf8bom": + return UTF8BOM, true + case "utf-16 le", "utf16le", "utf-16le": + return UTF16LE, true + case "utf-16 be", "utf16be", "utf-16be": + return UTF16BE, true + case "utf-16 le (no bom)", "utf16le-nobom", "utf-16le-nobom": + return UTF16LENoBOM, true + case "utf-16 be (no bom)", "utf16be-nobom", "utf-16be-nobom": + return UTF16BENoBOM, true + case "gb18030", "gbk", "gb2312": + return GB18030, true + } + return UTF8, false +} diff --git a/internal/provider/anthropic/anthropic.go b/internal/provider/anthropic/anthropic.go index 9d6fe141e..e641b2ed3 100644 --- a/internal/provider/anthropic/anthropic.go +++ b/internal/provider/anthropic/anthropic.go @@ -131,7 +131,7 @@ func (c *client) Stream(ctx context.Context, req provider.Request) (<-chan provi } out := make(chan provider.Chunk) - go c.readStream(resp, out) + go c.readStream(ctx, resp, out) return out, nil } @@ -250,10 +250,22 @@ func (c *client) buildRequest(req provider.Request) anthRequest { // and a complete ChunkToolCall when the block closes; usage is assembled from // message_start (input/cache) + message_delta (output + stop_reason) and emitted // once before ChunkDone. -func (c *client) readStream(resp *http.Response, out chan<- provider.Chunk) { +func (c *client) readStream(ctx context.Context, resp *http.Response, out chan<- provider.Chunk) { defer resp.Body.Close() defer close(out) + // Close the response body when the context is canceled so scanner.Scan() + // unblocks instead of hanging on a stalled connection. + done := make(chan struct{}) + defer close(done) + go func() { + select { + case <-ctx.Done(): + resp.Body.Close() + case <-done: + } + }() + tools := map[int]*provider.ToolCall{} // tool_use blocks, keyed by content index var inTok, outTok, cacheCreate, cacheRead int var stopReason string diff --git a/internal/provider/anthropic/anthropic_test.go b/internal/provider/anthropic/anthropic_test.go index 7614592b3..b0e4d212d 100644 --- a/internal/provider/anthropic/anthropic_test.go +++ b/internal/provider/anthropic/anthropic_test.go @@ -154,7 +154,7 @@ func TestReadStream(t *testing.T) { c := &client{name: "anthropic"} resp := &http.Response{Body: io.NopCloser(strings.NewReader(sseFixture))} ch := make(chan provider.Chunk) - go c.readStream(resp, ch) + go c.readStream(context.Background(), resp, ch) var text strings.Builder var started, full *provider.ToolCall @@ -209,7 +209,7 @@ func TestReadStreamError(t *testing.T) { c := &client{name: "anthropic"} resp := &http.Response{Body: io.NopCloser(strings.NewReader(sse))} ch := make(chan provider.Chunk) - go c.readStream(resp, ch) + go c.readStream(context.Background(), resp, ch) var gotErr error for ck := range ch { @@ -302,7 +302,7 @@ func TestReadStreamThinking(t *testing.T) { c := &client{name: "anthropic"} resp := &http.Response{Body: io.NopCloser(strings.NewReader(sseThinking))} ch := make(chan provider.Chunk) - go c.readStream(resp, ch) + go c.readStream(context.Background(), resp, ch) var reasoning, text strings.Builder var sig string diff --git a/internal/provider/openai/openai_test.go b/internal/provider/openai/openai_test.go index a064366f7..d57841f46 100644 --- a/internal/provider/openai/openai_test.go +++ b/internal/provider/openai/openai_test.go @@ -473,3 +473,73 @@ func TestStreamSynthesizesMissingToolCallIDs(t *testing.T) { t.Errorf("synthesized ids must be distinct, got %v", ids) } } + +// TestBuildRequestToolCallArgsRoundTrip verifies that tool-call arguments +// survive a marshal→unmarshal round-trip with Chinese/CJK content intact. +// Go's json.Marshal may encode non-ASCII as \uXXXX on the wire (valid JSON), +// but json.Unmarshal must restore the original characters. +func TestBuildRequestToolCallArgsRoundTrip(t *testing.T) { + c := &client{model: "deepseek-v4"} + req := c.buildRequest(provider.Request{ + Messages: []provider.Message{ + {Role: provider.RoleUser, Content: "grep it"}, + {Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{ + {ID: "call_zh", Name: "grep", Arguments: `{"pattern":"@游戏攻略"}`}, + }}, + {Role: provider.RoleTool, Content: "ok", ToolCallID: "call_zh", Name: "grep"}, + }, + }) + + b, err := json.Marshal(req.Messages) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + // Decode the assistant message's tool_calls arguments and verify the + // JSON content is preserved after the round-trip. + var raw []struct { + ToolCalls []struct { + Function struct { + Arguments string `json:"arguments"` + } `json:"function"` + } `json:"tool_calls"` + } + if err := json.Unmarshal(b, &raw); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if len(raw) < 2 || len(raw[1].ToolCalls) == 0 { + t.Fatalf("expected assistant tool_calls in serialized output: %s", string(b)) + } + gotArgs := raw[1].ToolCalls[0].Function.Arguments + wantArgs := `{"pattern":"@游戏攻略"}` + if gotArgs != wantArgs { + t.Errorf("arguments round-trip:\n got: %s\n want: %s", gotArgs, wantArgs) + } +} + +// TestBuildRequestInvalidToolArgsFallback ensures that when a model generates +// malformed JSON in tool-call arguments, the request serialisation falls back +// to a string-encoded form rather than failing outright (which would crash the +// entire request). +func TestBuildRequestInvalidToolArgsFallback(t *testing.T) { + c := &client{model: "deepseek-v4"} + req := c.buildRequest(provider.Request{ + Messages: []provider.Message{ + {Role: provider.RoleUser, Content: "do it"}, + {Role: provider.RoleAssistant, ToolCalls: []provider.ToolCall{ + {ID: "call_bad", Name: "write_file", Arguments: `"""invalid json"""`}, + }}, + {Role: provider.RoleTool, Content: "ok", ToolCallID: "call_bad", Name: "write_file"}, + }, + }) + + // The critical assertion: Marshal must succeed even with invalid args. + b, err := json.Marshal(req.Messages) + if err != nil { + t.Fatalf("marshal failed with invalid tool args: %v", err) + } + // The output must be valid JSON. + if !json.Valid(b) { + t.Errorf("marshal produced invalid JSON: %s", b) + } +} diff --git a/internal/tool/builtin/argparse.go b/internal/tool/builtin/argparse.go new file mode 100644 index 000000000..53fd5021f --- /dev/null +++ b/internal/tool/builtin/argparse.go @@ -0,0 +1,195 @@ +package builtin + +import ( + "encoding/json" + "strings" +) + +// unmarshalArgs is a fault-tolerant replacement for json.Unmarshal(args, &v). +// It first attempts a standard unmarshal; if that fails, it applies a series of +// repairs that target the most common malformed JSON produced by third-party +// LLM providers (triple-quoted strings, trailing commas, outer-quote wrapping, +// single-quoted keys, unescaped control characters). Each repair is retried +// independently so a valid-but-unusual input (e.g. single-quoted keys from a +// Chinese model) still passes through. +func unmarshalArgs(args json.RawMessage, v any) error { + if err := json.Unmarshal(args, v); err == nil { + return nil + } + // Attempt progressive repairs, retrying unmarshal after each. + s := string(args) + for _, repair := range []func(string) string{ + stripOuterQuotes, + fixTripleQuotes, + stripTrailingCommas, + fixSingleQuotes, + fixUnescapedControls, + } { + s = repair(s) + if err := json.Unmarshal([]byte(s), v); err == nil { + return nil + } + } + // All repairs exhausted — return the original error so the model gets + // the standard diagnostic and can self-correct. + return json.Unmarshal(args, v) +} + +// stripOuterQuotes handles the case where a model wraps the entire JSON object +// in an extra layer of quotes: `"{ ... }"` → `{ ... }`. This is common when +// smaller models confuse the tool-call argument string with its content. +func stripOuterQuotes(s string) string { + s = strings.TrimSpace(s) + if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' { + inner := s[1 : len(s)-1] + // Only strip if the inner content looks like a JSON object/array. + trimmed := strings.TrimSpace(inner) + if len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') { + // Unescape the inner content (the outer quotes caused \" escaping). + var unescaped string + if err := json.Unmarshal([]byte(s), &unescaped); err == nil { + return unescaped + } + } + } + return s +} + +// fixTripleQuotes replaces `"""` with `"` — a pattern emitted by some models +// that confuse Python-style triple quotes with JSON string delimiters. +// Example: `{"content": """hello\n"""}` → `{"content": "hello\n"}` +func fixTripleQuotes(s string) string { + return strings.ReplaceAll(s, `"""`, `"`) +} + +// stripTrailingCommas removes commas immediately before `}` or `]`, which +// are invalid in JSON but commonly emitted by models trained on Python/JS +// codebases where trailing commas are allowed. +// Example: `{"a": 1, "b": 2,}` → `{"a": 1, "b": 2}` +func stripTrailingCommas(s string) string { + var b strings.Builder + b.Grow(len(s)) + inString := false + escape := false + for i := 0; i < len(s); i++ { + c := s[i] + if escape { + b.WriteByte(c) + escape = false + continue + } + if c == '\\' && inString { + b.WriteByte(c) + escape = true + continue + } + if c == '"' { + inString = !inString + b.WriteByte(c) + continue + } + if !inString && c == ',' { + // Look ahead: skip whitespace, then check for } or ]. + j := i + 1 + for j < len(s) && (s[j] == ' ' || s[j] == '\t' || s[j] == '\n' || s[j] == '\r') { + j++ + } + if j < len(s) && (s[j] == '}' || s[j] == ']') { + continue // skip the trailing comma + } + } + b.WriteByte(c) + } + return b.String() +} + +// fixSingleQuotes replaces single-quoted JSON strings with double-quoted ones. +// Some models (especially Chinese ones trained on Python-heavy corpora) emit +// {'key': 'value'} instead of {"key": "value"}. +func fixSingleQuotes(s string) string { + var b strings.Builder + b.Grow(len(s)) + inDouble := false + escape := false + for i := 0; i < len(s); i++ { + c := s[i] + if escape { + b.WriteByte(c) + escape = false + continue + } + if c == '\\' && inDouble { + b.WriteByte(c) + escape = true + continue + } + if c == '"' { + inDouble = !inDouble + b.WriteByte(c) + continue + } + if !inDouble && c == '\'' { + // Check if this looks like a JSON key or value delimiter. + // Replace single quote with double quote. + b.WriteByte('"') + continue + } + b.WriteByte(c) + } + return b.String() +} + +// fixUnescapedControls replaces raw control characters (newlines, tabs) inside +// JSON string values with their proper escape sequences. Some models emit +// unescaped newlines in string content, which violates the JSON spec. +func fixUnescapedControls(s string) string { + var b strings.Builder + b.Grow(len(s)) + inString := false + escape := false + for i := 0; i < len(s); i++ { + c := s[i] + if escape { + b.WriteByte(c) + escape = false + continue + } + if c == '\\' && inString { + b.WriteByte(c) + escape = true + continue + } + if c == '"' { + inString = !inString + b.WriteByte(c) + continue + } + if inString && c < 0x20 { + switch c { + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + case '\t': + b.WriteString(`\t`) + default: + // Other control chars — use unicode escape. + b.WriteString(`\u`) + b.WriteString(toHex4(rune(c))) + } + continue + } + b.WriteByte(c) + } + return b.String() +} + +func toHex4(r rune) string { + const hex = "0123456789abcdef" + return string([]byte{ + hex[(r>>12)&0xf], + hex[(r>>8)&0xf], + hex[(r>>4)&0xf], + hex[r&0xf], + }) +} diff --git a/internal/tool/builtin/argparse_test.go b/internal/tool/builtin/argparse_test.go new file mode 100644 index 000000000..7d7f25080 --- /dev/null +++ b/internal/tool/builtin/argparse_test.go @@ -0,0 +1,156 @@ +package builtin + +import ( + "encoding/json" + "testing" +) + +func TestUnmarshalArgs_ValidJSON(t *testing.T) { + var p struct { + Path string `json:"path"` + Content string `json:"content"` + } + err := unmarshalArgs(json.RawMessage(`{"path":"3.txt","content":"hello"}`), &p) + if err != nil { + t.Fatalf("valid JSON should parse: %v", err) + } + if p.Path != "3.txt" || p.Content != "hello" { + t.Errorf("got %+v", p) + } +} + +func TestUnmarshalArgs_TripleQuotes(t *testing.T) { + var p struct { + Content string `json:"content"` + Path string `json:"path"` + } + raw := json.RawMessage(`{"content": """你好天下人""", "path": """3.txt"""}`) + err := unmarshalArgs(raw, &p) + if err != nil { + t.Fatalf("triple-quoted JSON should be repaired: %v", err) + } + if p.Content != "你好天下人" { + t.Errorf("content = %q, want %q", p.Content, "你好天下人") + } + if p.Path != "3.txt" { + t.Errorf("path = %q, want %q", p.Path, "3.txt") + } +} + +func TestUnmarshalArgs_TrailingComma(t *testing.T) { + var p struct { + A int `json:"a"` + B int `json:"b"` + } + raw := json.RawMessage(`{"a": 1, "b": 2,}`) + err := unmarshalArgs(raw, &p) + if err != nil { + t.Fatalf("trailing comma should be repaired: %v", err) + } + if p.A != 1 || p.B != 2 { + t.Errorf("got %+v", p) + } +} + +func TestUnmarshalArgs_SingleQuotes(t *testing.T) { + var p struct { + Name string `json:"name"` + } + raw := json.RawMessage(`{'name': 'hello'}`) + err := unmarshalArgs(raw, &p) + if err != nil { + t.Fatalf("single-quoted JSON should be repaired: %v", err) + } + if p.Name != "hello" { + t.Errorf("name = %q, want %q", p.Name, "hello") + } +} + +func TestUnmarshalArgs_OuterQuotes(t *testing.T) { + var p struct { + Path string `json:"path"` + } + // Model wrapped the entire JSON in extra quotes + raw := json.RawMessage(`"{\"path\":\"test.txt\"}"`) + err := unmarshalArgs(raw, &p) + if err != nil { + t.Fatalf("outer-quoted JSON should be repaired: %v", err) + } + if p.Path != "test.txt" { + t.Errorf("path = %q, want %q", p.Path, "test.txt") + } +} + +func TestUnmarshalArgs_ChineseContent(t *testing.T) { + var p struct { + Pattern string `json:"pattern"` + } + raw := json.RawMessage(`{"pattern":"@游戏攻略"}`) + err := unmarshalArgs(raw, &p) + if err != nil { + t.Fatalf("Chinese content should parse: %v", err) + } + if p.Pattern != "@游戏攻略" { + t.Errorf("pattern = %q, want %q", p.Pattern, "@游戏攻略") + } +} + +func TestUnmarshalArgs_Irreparable(t *testing.T) { + var p struct { + Path string `json:"path"` + } + raw := json.RawMessage(`not json at all`) + err := unmarshalArgs(raw, &p) + if err == nil { + t.Fatal("irreparable JSON should still return an error") + } +} + +func TestUnmarshalArgs_ComboTripleQuotesAndTrailingComma(t *testing.T) { + var p struct { + Content string `json:"content"` + Path string `json:"path"` + } + raw := json.RawMessage(`{"content": """hello""", "path": """3.txt""",}`) + err := unmarshalArgs(raw, &p) + if err != nil { + t.Fatalf("combined repairs should work: %v", err) + } + if p.Content != "hello" || p.Path != "3.txt" { + t.Errorf("got %+v", p) + } +} + +func TestFixTripleQuotes(t *testing.T) { + got := fixTripleQuotes(`{"a": """hello""", "b": """world"""}`) + want := `{"a": "hello", "b": "world"}` + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +func TestStripTrailingCommas(t *testing.T) { + tests := []struct { + in, want string + }{ + {`{"a": 1,}`, `{"a": 1}`}, + {`{"a": 1, "b": 2,}`, `{"a": 1, "b": 2}`}, + {`[1, 2, 3,]`, `[1, 2, 3]`}, + {`{"a": "hello,", "b": 2}`, `{"a": "hello,", "b": 2}`}, // comma inside string preserved + {`{"a": [1, 2,],}`, `{"a": [1, 2]}`}, + } + for _, tt := range tests { + got := stripTrailingCommas(tt.in) + if got != tt.want { + t.Errorf("stripTrailingCommas(%q) = %q, want %q", tt.in, got, tt.want) + } + } +} + +func TestFixSingleQuotes(t *testing.T) { + got := fixSingleQuotes(`{'name': 'hello', 'count': 5}`) + want := `{"name": "hello", "count": 5}` + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} diff --git a/internal/tool/builtin/bash.go b/internal/tool/builtin/bash.go index 59f29f95c..1c33e527f 100644 --- a/internal/tool/builtin/bash.go +++ b/internal/tool/builtin/bash.go @@ -1,7 +1,6 @@ package builtin import ( - "bytes" "context" "encoding/json" "fmt" @@ -78,7 +77,7 @@ func (b bash) Execute(ctx context.Context, args json.RawMessage) (string, error) Command string `json:"command"` RunInBackground bool `json:"run_in_background"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.Command == "" { @@ -122,7 +121,7 @@ func (b bash) Execute(ctx context.Context, args json.RawMessage) (string, error) cmd.Dir = b.workDir // "" lets exec use the process working directory setKillTree(cmd) cmd.WaitDelay = bashWaitDelay - var buf bytes.Buffer + var buf cappedBuffer w := io.Writer(&buf) if emit, ok := tool.ProgressFrom(ctx); ok { w = io.MultiWriter(&buf, newProgressWriter(emit)) @@ -188,3 +187,80 @@ func commandPreview(cmd string) string { } return cmd } + +// bashOutputCap is the maximum bytes retained from a foreground bash command. +// When exceeded, the buffer keeps only the first and last portions so the model +// still sees the command's opening context and its final output. +const ( + bashOutputCap = 10 << 20 // 10 MiB + bashOutputKeep = 1 << 20 // 1 MiB from each end +) + +// cappedBuffer is an io.Writer that accumulates command output up to +// bashOutputCap bytes. Once the cap is exceeded it switches to a ring-buffer +// strategy: the first bashOutputKeep bytes are preserved in `head` and the +// latest bytes rotate through `tail`, so String() returns head + gap marker + +// tail. This prevents a runaway command (e.g. `find /`, `cat /dev/urandom`) +// from consuming unbounded memory before the timeout fires. +type cappedBuffer struct { + head []byte // first bashOutputKeep bytes (immutable once filled) + tail []byte // ring buffer for the latest bytes + tailPos int // write cursor inside tail + total int // total bytes seen (may exceed cap) + capped bool // true once we switched to ring mode +} + +func (b *cappedBuffer) Write(p []byte) (int, error) { + n := len(p) + b.total += n + if !b.capped { + remaining := bashOutputCap - len(b.head) + if n <= remaining { + b.head = append(b.head, p...) + return n, nil + } + // Fill head to cap, then start ring buffer. + b.head = append(b.head, p[:remaining]...) + tailSize := bashOutputKeep + b.tail = make([]byte, tailSize) + b.capped = true + p = p[remaining:] + } + // Ring-buffer the remainder. + for len(p) > 0 { + space := len(b.tail) - b.tailPos + if space == 0 { + b.tailPos = 0 + space = len(b.tail) + } + chunk := len(p) + if chunk > space { + chunk = space + } + copy(b.tail[b.tailPos:], p[:chunk]) + b.tailPos += chunk + p = p[chunk:] + } + return n, nil +} + +func (b *cappedBuffer) String() string { + if !b.capped { + return string(b.head) + } + // Reconstruct tail in order: the ring buffer may have wrapped. + var tail []byte + if b.tailPos < len(b.tail) && b.total > bashOutputCap+len(b.tail) { + // Ring has wrapped: oldest data starts at tailPos. + tail = append(b.tail[b.tailPos:], b.tail[:b.tailPos]...) + } else { + tail = b.tail[:b.tailPos] + } + skipped := b.total - len(b.head) - len(tail) + var sb strings.Builder + sb.Grow(len(b.head) + len(tail) + 80) + sb.Write(b.head) + fmt.Fprintf(&sb, "\n\n... [truncated %d bytes] ...\n\n", skipped) + sb.Write(tail) + return sb.String() +} diff --git a/internal/tool/builtin/bgjobs.go b/internal/tool/builtin/bgjobs.go index bfb13b481..707a9d70e 100644 --- a/internal/tool/builtin/bgjobs.go +++ b/internal/tool/builtin/bgjobs.go @@ -45,7 +45,7 @@ func (bashOutput) Execute(ctx context.Context, args json.RawMessage) (string, er JobID string `json:"job_id"` Filter string `json:"filter"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.JobID == "" { @@ -108,7 +108,7 @@ func (killShell) Execute(ctx context.Context, args json.RawMessage) (string, err var p struct { JobID string `json:"job_id"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.JobID == "" { @@ -146,7 +146,7 @@ func (waitJob) Execute(ctx context.Context, args json.RawMessage) (string, error TimeoutSeconds int `json:"timeout_seconds"` } if len(args) > 0 { - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } } diff --git a/internal/tool/builtin/completestep.go b/internal/tool/builtin/completestep.go index 3961e88dd..6edda52c8 100644 --- a/internal/tool/builtin/completestep.go +++ b/internal/tool/builtin/completestep.go @@ -83,7 +83,7 @@ func (completeStep) Execute(ctx context.Context, args json.RawMessage) (string, Evidence []stepEvidence `json:"evidence"` Notes string `json:"notes"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if strings.TrimSpace(p.Step) == "" { diff --git a/internal/tool/builtin/confine.go b/internal/tool/builtin/confine.go index c01f68c3c..315a41c3e 100644 --- a/internal/tool/builtin/confine.go +++ b/internal/tool/builtin/confine.go @@ -22,12 +22,16 @@ func ConfineBash(spec sandbox.Spec) tool.Tool { // the unconfined instances registered at init time, so writes stay inside the // workspace by default. roots may be relative; they are resolved to absolute, // symlink-free paths once here. An empty roots slice yields unconfined writers. -func ConfineWriters(roots []string) []tool.Tool { +func ConfineWriters(roots []string, fileEncoding ...string) []tool.Tool { rs := realRoots(roots) + enc := "" + if len(fileEncoding) > 0 { + enc = fileEncoding[0] + } return []tool.Tool{ - writeFile{roots: rs}, - editFile{roots: rs}, - multiEdit{roots: rs}, + writeFile{roots: rs, fileEncoding: enc}, + editFile{roots: rs, fileEncoding: enc}, + multiEdit{roots: rs, fileEncoding: enc}, notebookEdit{roots: rs}, deleteRange{roots: rs}, deleteSymbol{roots: rs}, diff --git a/internal/tool/builtin/delete_range.go b/internal/tool/builtin/delete_range.go index 619f55bc5..0fa8a4d4c 100644 --- a/internal/tool/builtin/delete_range.go +++ b/internal/tool/builtin/delete_range.go @@ -61,7 +61,7 @@ func (d deleteRange) preview(args json.RawMessage) (diff.Change, error) { EndAnchor string `json:"end_anchor"` Inclusive *bool `json:"inclusive"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return diff.Change{}, fmt.Errorf("invalid args: %w", err) } if p.Path == "" { diff --git a/internal/tool/builtin/delete_symbol.go b/internal/tool/builtin/delete_symbol.go index 4a48462af..896250580 100644 --- a/internal/tool/builtin/delete_symbol.go +++ b/internal/tool/builtin/delete_symbol.go @@ -61,7 +61,7 @@ func (d deleteSymbol) Execute(ctx context.Context, args json.RawMessage) (string Kind string `json:"kind"` Parent string `json:"parent"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.Path == "" { @@ -107,7 +107,7 @@ func (d deleteSymbol) Preview(args json.RawMessage) (diff.Change, error) { Kind string `json:"kind"` Parent string `json:"parent"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return diff.Change{}, fmt.Errorf("invalid args: %w", err) } if p.Path == "" { diff --git a/internal/tool/builtin/editfile.go b/internal/tool/builtin/editfile.go index ba0f03875..b3a245d56 100644 --- a/internal/tool/builtin/editfile.go +++ b/internal/tool/builtin/editfile.go @@ -15,8 +15,9 @@ func init() { tool.RegisterBuiltin(editFile{}) } // workspace when non-empty (see writeFile); workDir, when non-empty, is the // directory a relative path resolves against (see resolveIn). type editFile struct { - roots []string - workDir string + roots []string + workDir string + fileEncoding string // project-level encoding override; empty = auto-detect } func (editFile) Name() string { return "edit_file" } @@ -26,7 +27,7 @@ func (editFile) Description() string { } func (editFile) Schema() json.RawMessage { - return json.RawMessage(`{"type":"object","properties":{"path":{"type":"string","description":"File path"},"old_string":{"type":"string","description":"Exact text to replace (must be unique in the file)"},"new_string":{"type":"string","description":"Replacement text (may be empty to delete)"}},"required":["path","old_string","new_string"]}`) + return json.RawMessage(`{"type":"object","properties":{"path":{"type":"string","description":"File path"},"old_string":{"type":"string","description":"Exact text to replace (must be unique in the file)"},"new_string":{"type":"string","description":"Replacement text (may be empty to delete)"},"encoding":{"type":"string","description":"File encoding override (e.g. \"UTF-8\", \"GB18030\"). When omitted, auto-detection is used."}},"required":["path","old_string","new_string"]}`) } func (editFile) ReadOnly() bool { return false } @@ -36,8 +37,9 @@ func (e editFile) Execute(ctx context.Context, args json.RawMessage) (string, er Path string `json:"path"` OldString string `json:"old_string"` NewString string `json:"new_string"` + Encoding string `json:"encoding,omitempty"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.Path == "" { @@ -51,7 +53,11 @@ func (e editFile) Execute(ctx context.Context, args json.RawMessage) (string, er return "", err } - content, enc, err := readFileEncoded(p.Path) + encParam := p.Encoding + if encParam == "" { + encParam = e.fileEncoding + } + content, enc, err := readFileEncodedWith(p.Path, encParam) if err != nil { return "", fmt.Errorf("read %s: %w", p.Path, err) } @@ -59,16 +65,29 @@ func (e editFile) Execute(ctx context.Context, args json.RawMessage) (string, er old, newStr := matchLineEndings(content, p.OldString, p.NewString) switch strings.Count(content, old) { case 0: - return "", fmt.Errorf("old_string not found in %s", p.Path) + // Exact match failed — try whitespace-tolerant fuzzy matching. + if actual, ok := fuzzyFind(content, old); ok { + // Adapt newStr's line endings to match the actual content region. + _, newStr = matchLineEndings(content, old, newStr) + // Replace the actual (whitespace-preserved) text in content. + updated := strings.Replace(content, actual, newStr, 1) + if err := writeFileEncodedWith(p.Path, updated, enc, encParam); err != nil { + return "", fmt.Errorf("write %s: %w", p.Path, err) + } + snippet := editSnippet(updated, actual, newStr, 3) + return fmt.Sprintf("edited %s (fuzzy match)\n\n%s", p.Path, snippet), nil + } + return "", diagnoseNotFound(p.Path, p.OldString, content) case 1: // ok default: - return "", fmt.Errorf("old_string is not unique in %s; add more surrounding context", p.Path) + return "", diagnoseNotUnique(p.Path, old, content) } updated := strings.Replace(content, old, newStr, 1) - if err := writeFileEncoded(p.Path, updated, enc); err != nil { + if err := writeFileEncodedWith(p.Path, updated, enc, encParam); err != nil { return "", fmt.Errorf("write %s: %w", p.Path, err) } - return fmt.Sprintf("edited %s", p.Path), nil + snippet := editSnippet(updated, old, newStr, 3) + return fmt.Sprintf("edited %s\n\n%s", p.Path, snippet), nil } diff --git a/internal/tool/builtin/encoding_helpers.go b/internal/tool/builtin/encoding_helpers.go index 9ffe99adf..008f975a3 100644 --- a/internal/tool/builtin/encoding_helpers.go +++ b/internal/tool/builtin/encoding_helpers.go @@ -1,6 +1,7 @@ package builtin import ( + "fmt" "os" "strings" @@ -11,10 +12,20 @@ import ( // Returns the decoded content and the detected encoding kind so callers // can re-encode on write to preserve the original charset. func readFileEncoded(path string) (content string, enc fileenc.Kind, err error) { + return readFileEncodedWith(path, "") +} + +// readFileEncodedWith reads a file and decodes it to UTF-8. When encName is +// non-empty the encoding is forced (skip auto-detection); otherwise behaves +// like readFileEncoded. +func readFileEncodedWith(path string, encName string) (content string, enc fileenc.Kind, err error) { b, err := os.ReadFile(path) if err != nil { return "", 0, err } + if forced, ok := fileenc.ParseName(encName); ok { + return string(fileenc.Decode(b, forced)), forced, nil + } enc, _ = fileenc.Detect(b) return string(fileenc.Decode(b, enc)), enc, nil } @@ -24,6 +35,15 @@ func writeFileEncoded(path string, content string, enc fileenc.Kind) error { return os.WriteFile(path, fileenc.Encode(content, enc), 0o644) } +// writeFileEncodedWith writes content to path. When encName is non-empty the +// encoding is forced; otherwise enc (typically from readFileEncoded) is used. +func writeFileEncodedWith(path string, content string, enc fileenc.Kind, encName string) error { + if forced, ok := fileenc.ParseName(encName); ok { + return os.WriteFile(path, fileenc.Encode(content, forced), 0o644) + } + return writeFileEncoded(path, content, enc) +} + // matchLineEndings adapts an edit's old/new text to a CRLF file when the literal // old_string isn't present but its CRLF form is. read_file strips '\r' (bufio // ScanLines), so a model's multi-line old_string arrives LF-only while a @@ -41,3 +61,262 @@ func matchLineEndings(content, old, new string) (string, string) { } return old, new } + +// diagnoseNotFound builds a helpful error message when old_string is not found in +// content. It locates the nearest line-level match (longest common prefix) and +// reports its line number plus the first differing characters, so the model can +// fix its old_string in one shot instead of blind retries. +func diagnoseNotFound(path, old, content string) error { + if old == "" { + return fmt.Errorf("old_string not found in %s", path) + } + oldLines := strings.Split(old, "\n") + contentLines := strings.Split(content, "\n") + + bestLine := -1 + bestScore := 0 + for i, cl := range contentLines { + score := commonPrefixLen(strings.TrimSpace(cl), strings.TrimSpace(oldLines[0])) + if score > bestScore { + bestScore = score + bestLine = i + } + } + + if bestLine < 0 || bestScore < 3 { + return fmt.Errorf("old_string not found in %s (no close match found)", path) + } + + // Show the nearest match's line number and the actual content there. + start := bestLine + end := start + len(oldLines) + if end > len(contentLines) { + end = len(contentLines) + } + actual := strings.Join(contentLines[start:end], "\n") + + // Trim both for comparison readability. + actualTrim := strings.TrimSpace(actual) + oldTrim := strings.TrimSpace(old) + if len(actualTrim) > 200 { + actualTrim = actualTrim[:200] + "…" + } + if len(oldTrim) > 200 { + oldTrim = oldTrim[:200] + "…" + } + + return fmt.Errorf("old_string not found in %s. Nearest match at line %d:\n expected: %s\n actual: %s", + path, bestLine+1, quoteLine(oldTrim), quoteLine(actualTrim)) +} + +// diagnoseNotUnique builds a helpful error message when old_string appears more +// than once, reporting the count and the line numbers of each occurrence so the +// model can add distinguishing context. +func diagnoseNotUnique(path, old, content string) error { + count := strings.Count(content, old) + lines := matchLineNumbers(old, content) + if len(lines) > 8 { + lines = append(lines[:8], -1) // sentinel for "and more" + } + lineStr := formatLineList(lines) + return fmt.Errorf("old_string is not unique in %s (%d matches at %s); add more surrounding context to disambiguate", + path, count, lineStr) +} + +// matchLineNumbers returns the 1-based line numbers where old appears in content. +// Uses strings.Index for SIMD-optimized substring search instead of a naive +// byte-by-byte scan, giving ~100x speedup on large files. +func matchLineNumbers(old, content string) []int { + if old == "" { + return nil + } + var lines []int + lineNo := 1 + offset := 0 + for { + idx := strings.Index(content[offset:], old) + if idx < 0 { + break + } + pos := offset + idx + // Count newlines before this match to get the line number. + lineNo += strings.Count(content[offset:pos], "\n") + lines = append(lines, lineNo) + offset = pos + len(old) + if offset >= len(content) { + break + } + } + return lines +} + +// formatLineList formats a slice of line numbers for display, with -1 meaning "…". +func formatLineList(lines []int) string { + parts := make([]string, 0, len(lines)) + for _, l := range lines { + if l < 0 { + parts = append(parts, "…") + } else { + parts = append(parts, fmt.Sprintf("line %d", l)) + } + } + return strings.Join(parts, ", ") +} + +// commonPrefixLen returns the number of leading runes two strings share. +func commonPrefixLen(a, b string) int { + n := len(a) + if len(b) < n { + n = len(b) + } + for i := 0; i < n; i++ { + if a[i] != b[i] { + return i + } + } + return n +} + +// quoteLine wraps a line sample for display in an error message, making +// invisible whitespace visible: tabs render as "→" so the model can tell +// tab-indented code from space-indented code at a glance. +func quoteLine(s string) string { + s = strings.ReplaceAll(s, "\t", "→") + if strings.Contains(s, "\n") { + return "«" + strings.ReplaceAll(s, "\n", "↵") + "»" + } + return "`" + s + "`" +} + +// editSnippet returns a short context window (±contextLines around the edit) +// so the model can verify the edit landed in the right place. Tabs and other +// invisible whitespace are rendered visibly. +func editSnippet(content, old, newStr string, contextLines int) string { + if old == "" && newStr == "" { + return "" + } + lines := strings.Split(content, "\n") + // Find the line where the replacement starts. + idx := strings.Index(content, newStr) + if idx < 0 { + return "" // replacement not in content (shouldn't happen) + } + editLine := strings.Count(content[:idx], "\n") + + start := editLine - contextLines + if start < 0 { + start = 0 + } + end := editLine + strings.Count(newStr, "\n") + 1 + contextLines + if end > len(lines) { + end = len(lines) + } + + var sb strings.Builder + width := len(fmt.Sprintf("%d", end)) + for i := start; i < end; i++ { + marker := " " + if i >= editLine && i <= editLine+strings.Count(newStr, "\n") { + marker = "›" + } + line := strings.ReplaceAll(lines[i], "\t", "→") + fmt.Fprintf(&sb, "%s %*d│%s\n", marker, width, i+1, line) + } + return sb.String() +} + +// fuzzyFind attempts a whitespace-tolerant match of old in content when exact +// matching fails. It tries two progressive relaxations: +// +// 1. Trim trailing whitespace from every line (models often add or drop +// spaces at line ends, or the file has trailing spaces the model didn't +// reproduce). +// 2. Also trim leading whitespace (the model mis-indented the block or the +// code was extracted from a different indentation context). +// +// When a fuzzy match is found it returns the **actual** text from content +// (with original whitespace preserved) so that strings.Replace substitutes +// the real region without disturbing the file's formatting. +// +// Returns ("", false) if no match is found at any relaxation level. +func fuzzyFind(content, old string) (actualOld string, found bool) { + if old == "" || content == "" { + return "", false + } + + // Split on \n (works for both LF and CRLF — \r stays attached). + oldLines := strings.Split(old, "\n") + contentLines := strings.Split(content, "\n") + nOld := len(oldLines) + if nOld == 0 || nOld > len(contentLines) { + return "", false + } + + // --- Level 1: trim trailing whitespace per line --- + trimTrail := func(lines []string) []string { + out := make([]string, len(lines)) + for i, l := range lines { + out[i] = strings.TrimRight(l, " \t\r") + } + return out + } + + normOld := trimTrail(oldLines) + for i := 0; i <= len(contentLines)-nOld; i++ { + window := contentLines[i : i+nOld] + if linesEqual(trimTrail(window), normOld) { + return strings.Join(window, "\n"), true + } + } + + // --- Level 1.5: normalize tabs to spaces (tab↔space mismatch) --- + // Catches the common case where the model emits spaces but the file uses + // tabs (or vice versa), without stripping all indentation like Level 2. + expandTabs := func(lines []string) []string { + out := make([]string, len(lines)) + for i, l := range lines { + out[i] = strings.ReplaceAll(l, "\t", " ") + } + return out + } + + normOldTab := expandTabs(normOld) + for i := 0; i <= len(contentLines)-nOld; i++ { + window := contentLines[i : i+nOld] + if linesEqual(expandTabs(trimTrail(window)), normOldTab) { + return strings.Join(window, "\n"), true + } + } + + // --- Level 2: also trim leading whitespace (dedent) --- + trimLead := func(lines []string) []string { + out := make([]string, len(lines)) + for i, l := range lines { + out[i] = strings.TrimLeft(l, " \t") + } + return out + } + + normOld2 := trimLead(normOld) + for i := 0; i <= len(contentLines)-nOld; i++ { + window := contentLines[i : i+nOld] + if linesEqual(trimLead(trimTrail(window)), normOld2) { + return strings.Join(window, "\n"), true + } + } + + return "", false +} + +// linesEqual compares two line slices element-wise. +func linesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/tool/builtin/fuzzy_test.go b/internal/tool/builtin/fuzzy_test.go new file mode 100644 index 000000000..01f794e07 --- /dev/null +++ b/internal/tool/builtin/fuzzy_test.go @@ -0,0 +1,126 @@ +package builtin + +import "testing" + +func TestFuzzyFind_ExactMatch(t *testing.T) { + content := "hello world\nfoo bar\nbaz" + actual, ok := fuzzyFind(content, "foo bar") + if !ok { + t.Fatal("expected exact match") + } + if actual != "foo bar" { + t.Errorf("actual = %q, want %q", actual, "foo bar") + } +} + +func TestFuzzyFind_TrailingWhitespace(t *testing.T) { + // File has trailing spaces that the model didn't reproduce. + content := "func main() { \n\tfmt.Println(\"hello\") \n}\n" + old := "func main() {\n\tfmt.Println(\"hello\")\n}" + actual, ok := fuzzyFind(content, old) + if !ok { + t.Fatal("expected trailing-whitespace match") + } + // The actual text must come from the file (with trailing spaces). + if actual != "func main() { \n\tfmt.Println(\"hello\") \n}" { + t.Errorf("actual = %q", actual) + } +} + +func TestFuzzyFind_LeadingWhitespace(t *testing.T) { + // Model mis-indented by one level (2 spaces instead of 4). + content := "class Foo:\n def bar(self):\n return 42\n" + old := " def bar(self):\n return 42" + actual, ok := fuzzyFind(content, old) + if !ok { + t.Fatal("expected leading-whitespace match") + } + // Actual text from the file (with 4-space indent). + if actual != " def bar(self):\n return 42" { + t.Errorf("actual = %q", actual) + } +} + +func TestFuzzyFind_BothTrailingAndLeading(t *testing.T) { + content := " if x > 0: \n return x \n" + old := " if x > 0:\n return x" + actual, ok := fuzzyFind(content, old) + if !ok { + t.Fatal("expected combined whitespace match") + } + if actual != " if x > 0: \n return x " { + t.Errorf("actual = %q", actual) + } +} + +func TestFuzzyFind_NoMatch(t *testing.T) { + content := "completely different text\nnothing like the pattern" + old := "foo bar baz\nqux quux" + _, ok := fuzzyFind(content, old) + if ok { + t.Fatal("expected no match") + } +} + +func TestFuzzyFind_EmptyOld(t *testing.T) { + _, ok := fuzzyFind("hello", "") + if ok { + t.Fatal("empty old should not match") + } +} + +func TestFuzzyFind_SingleLine(t *testing.T) { + content := " hello world " + old := "hello world" + actual, ok := fuzzyFind(content, old) + if !ok { + t.Fatal("expected single-line whitespace match") + } + if actual != " hello world " { + t.Errorf("actual = %q", actual) + } +} + +func TestFuzzyFind_CRLF(t *testing.T) { + // File uses CRLF, old uses CRLF (as matchLineEndings would convert). + content := "func main() { \r\n\treturn nil \r\n}\r\n" + old := "func main() {\r\n\treturn nil\r\n}" + actual, ok := fuzzyFind(content, old) + if !ok { + t.Fatal("expected CRLF trailing-whitespace match") + } + // The actual text from content includes the \r on each line + // (since we split on \n, \r stays attached). + want := "func main() { \r\n\treturn nil \r\n}\r" + if actual != want { + t.Errorf("actual = %q, want %q", actual, want) + } +} + +func TestFuzzyFind_PreservesOriginalFormatting(t *testing.T) { + // The replacement should use the actual file text, not the normalized version. + content := " line1 \n line2 \n line3\n" + old := "line1\nline2" + actual, ok := fuzzyFind(content, old) + if !ok { + t.Fatal("expected match") + } + // Verify actual comes from content so Replace will work. + if actual != " line1 \n line2 " { + t.Errorf("actual = %q, expected original file text with whitespace", actual) + } +} + +func TestFuzzyFind_TabVsSpaces(t *testing.T) { + // File uses tabs, model emits 4 spaces per tab. + content := "\tfunc main() {\n\t\treturn nil\n\t}\n" + old := " func main() {\n return nil\n }" + actual, ok := fuzzyFind(content, old) + if !ok { + t.Fatal("expected tab-vs-space match at Level 1.5") + } + // Actual text from file preserves tabs. + if actual != "\tfunc main() {\n\t\treturn nil\n\t}" { + t.Errorf("actual = %q, expected tab-preserved text", actual) + } +} diff --git a/internal/tool/builtin/glob.go b/internal/tool/builtin/glob.go index 1260b5f25..7e20f28a2 100644 --- a/internal/tool/builtin/glob.go +++ b/internal/tool/builtin/glob.go @@ -36,7 +36,7 @@ func (g globTool) Execute(ctx context.Context, args json.RawMessage) (string, er var p struct { Pattern string `json:"pattern"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.Pattern == "" { diff --git a/internal/tool/builtin/grep.go b/internal/tool/builtin/grep.go index 8a84f6dd0..09a84f229 100644 --- a/internal/tool/builtin/grep.go +++ b/internal/tool/builtin/grep.go @@ -52,7 +52,7 @@ func (g grepTool) Execute(ctx context.Context, args json.RawMessage) (string, er Pattern string `json:"pattern"` Path string `json:"path"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.Pattern == "" { diff --git a/internal/tool/builtin/ls.go b/internal/tool/builtin/ls.go index cd963d374..1d43c9a2d 100644 --- a/internal/tool/builtin/ls.go +++ b/internal/tool/builtin/ls.go @@ -35,7 +35,7 @@ func (l listDir) Execute(ctx context.Context, args json.RawMessage) (string, err Recursive bool `json:"recursive"` }{Path: "."} if len(args) > 0 { - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } } @@ -46,7 +46,7 @@ func (l listDir) Execute(ctx context.Context, args json.RawMessage) (string, err // Recursive mode: walk the whole tree depth-first. if p.Recursive { - return l.listRecursive(p.Path) + return l.listRecursive(ctx, p.Path) } entries, err := os.ReadDir(p.Path) @@ -72,13 +72,23 @@ func (l listDir) Execute(ctx context.Context, args json.RawMessage) (string, err return b.String(), nil } +// lsRecursiveCap is the maximum number of entries listRecursive will emit +// before truncating, so a huge monorepo can't blow up memory. +const lsRecursiveCap = 5000 + // listRecursive walks a directory tree depth-first, skipping noise dirs. -// Depth is capped to guard against symlink loops. -func (l listDir) listRecursive(root string) (string, error) { +// Depth is capped to guard against symlink loops; result count is capped to +// guard against huge trees; context cancellation aborts the walk promptly. +func (l listDir) listRecursive(ctx context.Context, root string) (string, error) { var b strings.Builder + count := 0 + truncated := false err := filepath.WalkDir(root, func(p string, d os.DirEntry, wErr error) error { + if ctx.Err() != nil { + return ctx.Err() + } if wErr != nil { - return wErr + return nil // skip unreadable entries instead of aborting } if p == root { return nil @@ -89,6 +99,13 @@ func (l listDir) listRecursive(root string) (string, error) { return filepath.SkipDir } } + if count >= lsRecursiveCap { + truncated = true + if d.IsDir() { + return filepath.SkipDir + } + return nil + } rel, rErr := filepath.Rel(root, p) if rErr != nil { rel = p @@ -107,13 +124,17 @@ func (l listDir) listRecursive(root string) (string, error) { rel += fmt.Sprintf("\t%d", info.Size()) } b.WriteString(rel + "\n") + count++ return nil }) - if err != nil { + if err != nil && ctx.Err() == nil { return "", fmt.Errorf("ls -R %s: %w", root, err) } if b.Len() == 0 { return "(empty directory tree)", nil } + if truncated { + b.WriteString(fmt.Sprintf("\n... (truncated at %d entries)\n", lsRecursiveCap)) + } return b.String(), nil } diff --git a/internal/tool/builtin/multiedit.go b/internal/tool/builtin/multiedit.go index 18df2d85a..d875a0dc7 100644 --- a/internal/tool/builtin/multiedit.go +++ b/internal/tool/builtin/multiedit.go @@ -15,8 +15,9 @@ func init() { tool.RegisterBuiltin(multiEdit{}) } // the workspace when non-empty (see writeFile); workDir, when non-empty, is the // directory a relative path resolves against (see resolveIn). type multiEdit struct { - roots []string - workDir string + roots []string + workDir string + fileEncoding string // project-level encoding override; empty = auto-detect } // editStep is one edit in a multi_edit operation. Mirrors edit_file's args @@ -53,7 +54,8 @@ func (multiEdit) Schema() json.RawMessage { }, "required":["old_string","new_string"] } - } + }, + "encoding":{"type":"string","description":"File encoding override (e.g. \"UTF-8\", \"GB18030\"). When omitted, auto-detection is used."} }, "required":["path","edits"] }`) @@ -63,10 +65,11 @@ func (multiEdit) ReadOnly() bool { return false } func (m multiEdit) Execute(ctx context.Context, args json.RawMessage) (string, error) { var p struct { - Path string `json:"path"` - Edits []editStep `json:"edits"` + Path string `json:"path"` + Edits []editStep `json:"edits"` + Encoding string `json:"encoding,omitempty"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.Path == "" { @@ -80,7 +83,11 @@ func (m multiEdit) Execute(ctx context.Context, args json.RawMessage) (string, e return "", err } - content, enc, err := readFileEncoded(p.Path) + encParam := p.Encoding + if encParam == "" { + encParam = m.fileEncoding + } + content, enc, err := readFileEncodedWith(p.Path, encParam) if err != nil { return "", fmt.Errorf("read %s: %w", p.Path, err) } @@ -90,6 +97,7 @@ func (m multiEdit) Execute(ctx context.Context, args json.RawMessage) (string, e // safety guarantee that makes multi_edit preferable to chained // edit_file calls. applied := 0 + var lastNew string // track last replacement for snippet for i, step := range p.Edits { if step.OldString == "" { return "", fmt.Errorf("edit %d: old_string is required", i+1) @@ -98,25 +106,45 @@ func (m multiEdit) Execute(ctx context.Context, args json.RawMessage) (string, e if step.ReplaceAll { count := strings.Count(content, old) if count == 0 { - return "", fmt.Errorf("edit %d: old_string not found", i+1) + // Fuzzy match for replace_all: find the actual text and replace all. + if actual, ok := fuzzyFind(content, old); ok { + _, newStr = matchLineEndings(content, old, newStr) + fuzzyCount := strings.Count(content, actual) + content = strings.ReplaceAll(content, actual, newStr) + applied += fuzzyCount + lastNew = newStr + continue + } + return "", diagnoseNotFound(p.Path, step.OldString, content) } content = strings.ReplaceAll(content, old, newStr) applied += count + lastNew = newStr continue } switch strings.Count(content, old) { case 0: - return "", fmt.Errorf("edit %d: old_string not found", i+1) + // Fuzzy fallback. + if actual, ok := fuzzyFind(content, old); ok { + _, newStr = matchLineEndings(content, old, newStr) + content = strings.Replace(content, actual, newStr, 1) + applied++ + lastNew = newStr + continue + } + return "", diagnoseNotFound(p.Path, step.OldString, content) case 1: content = strings.Replace(content, old, newStr, 1) applied++ + lastNew = newStr default: - return "", fmt.Errorf("edit %d: old_string is not unique; add more surrounding context or set replace_all", i+1) + return "", diagnoseNotUnique(p.Path, old, content) } } - if err := writeFileEncoded(p.Path, content, enc); err != nil { + if err := writeFileEncodedWith(p.Path, content, enc, encParam); err != nil { return "", fmt.Errorf("write %s: %w", p.Path, err) } - return fmt.Sprintf("multi_edit %s: %d edits applied (%d total replacements)", p.Path, len(p.Edits), applied), nil + snippet := editSnippet(content, "", lastNew, 3) + return fmt.Sprintf("multi_edit %s: %d edits applied (%d total replacements)\n\n%s", p.Path, len(p.Edits), applied, snippet), nil } diff --git a/internal/tool/builtin/notebookedit.go b/internal/tool/builtin/notebookedit.go index bd6a68f5d..9d3adf9fc 100644 --- a/internal/tool/builtin/notebookedit.go +++ b/internal/tool/builtin/notebookedit.go @@ -136,7 +136,7 @@ func (n notebookEdit) Preview(raw json.RawMessage) (diff.Change, error) { func parseNotebookArgs(raw json.RawMessage) (notebookArgs, error) { var a notebookArgs - if err := json.Unmarshal(raw, &a); err != nil { + if err := unmarshalArgs(raw, &a); err != nil { return a, fmt.Errorf("invalid args: %w", err) } // Be forgiving about the source field: models reach for the write_file/edit_file diff --git a/internal/tool/builtin/preview.go b/internal/tool/builtin/preview.go index 3e675c06c..3a8646735 100644 --- a/internal/tool/builtin/preview.go +++ b/internal/tool/builtin/preview.go @@ -28,7 +28,7 @@ func (w writeFile) Preview(args json.RawMessage) (diff.Change, error) { Path string `json:"path"` Content string `json:"content"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return diff.Change{}, fmt.Errorf("invalid args: %w", err) } if p.Path == "" { @@ -54,8 +54,9 @@ func (e editFile) Preview(args json.RawMessage) (diff.Change, error) { Path string `json:"path"` OldString string `json:"old_string"` NewString string `json:"new_string"` + Encoding string `json:"encoding,omitempty"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return diff.Change{}, fmt.Errorf("invalid args: %w", err) } if p.Path == "" { @@ -66,21 +67,32 @@ func (e editFile) Preview(args json.RawMessage) (diff.Change, error) { } p.Path = resolveIn(e.workDir, p.Path) - content, _, err := readFileEncoded(p.Path) + encParam := p.Encoding + if encParam == "" { + encParam = e.fileEncoding + } + content, _, err := readFileEncodedWith(p.Path, encParam) if err != nil { return diff.Change{}, fmt.Errorf("read %s: %w", p.Path, err) } - switch strings.Count(content, p.OldString) { + old, newStr := matchLineEndings(content, p.OldString, p.NewString) + switch strings.Count(content, old) { case 0: - return diff.Change{}, fmt.Errorf("old_string not found in %s", p.Path) + // Fuzzy fallback. + if actual, ok := fuzzyFind(content, old); ok { + _, newStr = matchLineEndings(content, old, newStr) + updated := strings.Replace(content, actual, newStr, 1) + return diff.Build(p.Path, content, updated, diff.Modify), nil + } + return diff.Change{}, diagnoseNotFound(p.Path, p.OldString, content) case 1: // ok default: - return diff.Change{}, fmt.Errorf("old_string is not unique in %s; add more surrounding context", p.Path) + return diff.Change{}, diagnoseNotUnique(p.Path, old, content) } - updated := strings.Replace(content, p.OldString, p.NewString, 1) + updated := strings.Replace(content, old, newStr, 1) return diff.Build(p.Path, content, updated, diff.Modify), nil } @@ -90,10 +102,11 @@ func (e editFile) Preview(args json.RawMessage) (diff.Change, error) { // of an invalid batch fails the same way the call would. func (m multiEdit) Preview(args json.RawMessage) (diff.Change, error) { var p struct { - Path string `json:"path"` - Edits []editStep `json:"edits"` + Path string `json:"path"` + Edits []editStep `json:"edits"` + Encoding string `json:"encoding,omitempty"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return diff.Change{}, fmt.Errorf("invalid args: %w", err) } if p.Path == "" { @@ -104,7 +117,11 @@ func (m multiEdit) Preview(args json.RawMessage) (diff.Change, error) { } p.Path = resolveIn(m.workDir, p.Path) - content, _, err := readFileEncoded(p.Path) + encParam := p.Encoding + if encParam == "" { + encParam = m.fileEncoding + } + content, _, err := readFileEncodedWith(p.Path, encParam) if err != nil { return diff.Change{}, fmt.Errorf("read %s: %w", p.Path, err) } @@ -114,20 +131,31 @@ func (m multiEdit) Preview(args json.RawMessage) (diff.Change, error) { if step.OldString == "" { return diff.Change{}, fmt.Errorf("edit %d: old_string is required", i+1) } + old, newStr := matchLineEndings(content, step.OldString, step.NewString) if step.ReplaceAll { - if strings.Count(content, step.OldString) == 0 { - return diff.Change{}, fmt.Errorf("edit %d: old_string not found", i+1) + if strings.Count(content, old) == 0 { + if actual, ok := fuzzyFind(content, old); ok { + _, newStr = matchLineEndings(content, old, newStr) + content = strings.ReplaceAll(content, actual, newStr) + continue + } + return diff.Change{}, diagnoseNotFound(p.Path, step.OldString, content) } - content = strings.ReplaceAll(content, step.OldString, step.NewString) + content = strings.ReplaceAll(content, old, newStr) continue } - switch strings.Count(content, step.OldString) { + switch strings.Count(content, old) { case 0: - return diff.Change{}, fmt.Errorf("edit %d: old_string not found", i+1) + if actual, ok := fuzzyFind(content, old); ok { + _, newStr = matchLineEndings(content, old, newStr) + content = strings.Replace(content, actual, newStr, 1) + continue + } + return diff.Change{}, diagnoseNotFound(p.Path, step.OldString, content) case 1: - content = strings.Replace(content, step.OldString, step.NewString, 1) + content = strings.Replace(content, old, newStr, 1) default: - return diff.Change{}, fmt.Errorf("edit %d: old_string is not unique; add more surrounding context or set replace_all", i+1) + return diff.Change{}, diagnoseNotUnique(p.Path, old, content) } } return diff.Build(p.Path, original, content, diff.Modify), nil diff --git a/internal/tool/builtin/readfile.go b/internal/tool/builtin/readfile.go index 5167724df..0b6c2c0d1 100644 --- a/internal/tool/builtin/readfile.go +++ b/internal/tool/builtin/readfile.go @@ -28,7 +28,10 @@ func init() { tool.RegisterBuiltin(readFile{}) } // readFile reads a text file. workDir, when non-empty, is the directory a // relative path is resolved against (see resolveIn); the zero value registered // at init resolves against the process working directory. -type readFile struct{ workDir string } +type readFile struct { + workDir string + fileEncoding string // project-level encoding override; empty = auto-detect +} const ( readFileDefaultLimit = 2000 // lines returned when limit is unset @@ -46,7 +49,8 @@ func (readFile) Schema() json.RawMessage { "properties":{ "path":{"type":"string","description":"File path"}, "offset":{"type":"integer","description":"0-based line offset to start reading from (default 0)","minimum":0}, - "limit":{"type":"integer","description":"Maximum lines to return (default 2000)","minimum":1} + "limit":{"type":"integer","description":"Maximum lines to return (default 2000)","minimum":1}, + "encoding":{"type":"string","description":"File encoding override (e.g. \"UTF-8\", \"GB18030\", \"UTF-16 LE\"). When omitted, auto-detection is used."} }, "required":["path"] }`) @@ -56,11 +60,12 @@ func (readFile) ReadOnly() bool { return true } func (r readFile) Execute(ctx context.Context, args json.RawMessage) (string, error) { var p struct { - Path string `json:"path"` - Offset int `json:"offset,omitempty"` - Limit int `json:"limit,omitempty"` + Path string `json:"path"` + Offset int `json:"offset,omitempty"` + Limit int `json:"limit,omitempty"` + Encoding string `json:"encoding,omitempty"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.Path == "" { @@ -87,6 +92,21 @@ func (r readFile) Execute(ctx context.Context, args json.RawMessage) (string, er } defer f.Close() + // When the caller specifies an encoding, skip all auto-detection branches and + // decode with the forced encoding directly. Fall back to the project-level + // encoding when the per-call param is empty. + encOverride := p.Encoding + if encOverride == "" { + encOverride = r.fileEncoding + } + if forced, ok := fileenc.ParseName(encOverride); ok { + all, rerr := io.ReadAll(f) + if rerr != nil { + return "", fmt.Errorf("read %s: %w", p.Path, rerr) + } + return r.scan(bytes.NewReader(fileenc.Decode(all, forced)), p.Offset, p.Limit) + } + // Peek the first 8 KiB to reject binary files cheaply (a NUL byte) before // reading further — keeps a multi-GB archive from being slurped just to be // discarded. diff --git a/internal/tool/builtin/todo.go b/internal/tool/builtin/todo.go index c004f659d..9dec8ceb9 100644 --- a/internal/tool/builtin/todo.go +++ b/internal/tool/builtin/todo.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" - "reasonix/internal/evidence" "reasonix/internal/tool" ) @@ -63,7 +62,7 @@ func (todoWrite) Execute(ctx context.Context, args json.RawMessage) (string, err var p struct { Todos []todoItem `json:"todos"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } var done, active, pending int @@ -85,38 +84,6 @@ func (todoWrite) Execute(ctx context.Context, args json.RawMessage) (string, err return "", fmt.Errorf("todo %d: invalid status %q (want pending|in_progress|completed)", i+1, t.Status) } } - if err := verifyTodoCompletionTransitions(ctx, p.Todos); err != nil { - return "", err - } return fmt.Sprintf("Todos updated: %d total — %d completed, %d in progress, %d pending.", len(p.Todos), done, active, pending), nil } - -func verifyTodoCompletionTransitions(ctx context.Context, todos []todoItem) error { - ledger, ok := evidence.FromContext(ctx) - if !ok { - return nil - } - missing, hasBaseline := ledger.UnverifiedCompletedTodos(toEvidenceTodos(todos)) - if !hasBaseline || len(missing) == 0 { - return nil - } - if len(missing) == 1 { - m := missing[0] - return fmt.Errorf("todo %d %q is newly completed but has no matching successful complete_step receipt in this turn", m.Index, m.Content) - } - return fmt.Errorf("%d todos are newly completed but have no matching successful complete_step receipts in this turn", len(missing)) -} - -func toEvidenceTodos(todos []todoItem) []evidence.TodoItem { - out := make([]evidence.TodoItem, 0, len(todos)) - for _, t := range todos { - out = append(out, evidence.TodoItem{ - Content: t.Content, - Status: t.Status, - ActiveForm: t.ActiveForm, - Level: t.Level, - }) - } - return out -} diff --git a/internal/tool/builtin/todo_test.go b/internal/tool/builtin/todo_test.go index f475c4d00..693e791bc 100644 --- a/internal/tool/builtin/todo_test.go +++ b/internal/tool/builtin/todo_test.go @@ -5,8 +5,6 @@ import ( "encoding/json" "strings" "testing" - - "reasonix/internal/evidence" ) func TestTodoWriteAcceptsLevels(t *testing.T) { @@ -26,60 +24,16 @@ func TestTodoWriteRejectsBadLevel(t *testing.T) { } } -func TestTodoWriteRejectsNewCompletedWithoutCompleteStepReceipt(t *testing.T) { - ledger := evidence.NewLedger() - ledger.Record(evidence.Receipt{ - ToolName: "todo_write", - Success: true, - Todos: []evidence.TodoItem{{Content: "Add parser", Status: "in_progress"}}, - }) - ctx := evidence.WithLedger(context.Background(), ledger) - args := json.RawMessage(`{"todos":[{"content":"Add parser","status":"completed"}]}`) - - _, err := (todoWrite{}).Execute(ctx, args) - if err == nil || !strings.Contains(err.Error(), "complete_step") { - t.Fatalf("new completion without complete_step should be rejected, got %v", err) - } -} - -func TestTodoWriteAcceptsNewCompletedWithCompleteStepReceipt(t *testing.T) { - ledger := evidence.NewLedger() - ledger.Record(evidence.Receipt{ - ToolName: "todo_write", - Success: true, - Todos: []evidence.TodoItem{{Content: "Add parser", Status: "in_progress"}}, - }) - ledger.Record(evidence.Receipt{ToolName: "complete_step", Success: true, Step: "Add parser"}) - ctx := evidence.WithLedger(context.Background(), ledger) - args := json.RawMessage(`{"todos":[{"content":"Add parser","status":"completed"}]}`) - - if _, err := (todoWrite{}).Execute(ctx, args); err != nil { - t.Fatalf("matching complete_step should authorize new completion: %v", err) - } -} - -func TestTodoWriteAllowsInitialCompletedWithoutBaseline(t *testing.T) { - ctx := evidence.WithLedger(context.Background(), evidence.NewLedger()) - args := json.RawMessage(`{"todos":[{"content":"Add parser","status":"completed"}]}`) - - if _, err := (todoWrite{}).Execute(ctx, args); err != nil { - t.Fatalf("initial completed todo without baseline should preserve existing behavior: %v", err) +func TestTodoWriteAcceptsCompletedDirectly(t *testing.T) { + args := json.RawMessage(`{"todos":[` + + `{"content":"Step 1","status":"completed"},` + + `{"content":"Step 2","status":"in_progress"},` + + `{"content":"Step 3","status":"pending"}]}`) + result, err := (todoWrite{}).Execute(context.Background(), args) + if err != nil { + t.Fatalf("direct completion should be accepted: %v", err) } -} - -func TestTodoWriteIgnoresFailedCompleteStepReceipt(t *testing.T) { - ledger := evidence.NewLedger() - ledger.Record(evidence.Receipt{ - ToolName: "todo_write", - Success: true, - Todos: []evidence.TodoItem{{Content: "Add parser", Status: "in_progress"}}, - }) - ledger.Record(evidence.Receipt{ToolName: "complete_step", Success: false, Step: "Add parser"}) - ctx := evidence.WithLedger(context.Background(), ledger) - args := json.RawMessage(`{"todos":[{"content":"Add parser","status":"completed"}]}`) - - _, err := (todoWrite{}).Execute(ctx, args) - if err == nil || !strings.Contains(err.Error(), "complete_step") { - t.Fatalf("failed complete_step should not authorize new completion, got %v", err) + if !strings.Contains(result, "1 completed") { + t.Errorf("expected 1 completed in result, got: %s", result) } } diff --git a/internal/tool/builtin/webfetch.go b/internal/tool/builtin/webfetch.go index 8fd6c0179..b6e5302bc 100644 --- a/internal/tool/builtin/webfetch.go +++ b/internal/tool/builtin/webfetch.go @@ -101,7 +101,7 @@ func (webFetch) Execute(ctx context.Context, args json.RawMessage) (string, erro var p struct { URL string `json:"url"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.URL == "" { diff --git a/internal/tool/builtin/workspace.go b/internal/tool/builtin/workspace.go index 4e62f64bf..c0e78ccee 100644 --- a/internal/tool/builtin/workspace.go +++ b/internal/tool/builtin/workspace.go @@ -19,10 +19,11 @@ import ( // root, so writes stay inside the project by default. Bash is the OS-sandbox // spec for the bash tool (as ConfineBash). type Workspace struct { - Dir string - WriteRoots []string - Bash sandbox.Spec - Search SearchSpec + Dir string + WriteRoots []string + Bash sandbox.Spec + Search SearchSpec + FileEncoding string // project-level file encoding (e.g. "GB18030"); empty = auto-detect } // Tools returns the built-in tools bound to the workspace, ready to Add to a @@ -38,10 +39,10 @@ func (w Workspace) Tools(enabled ...string) []tool.Tool { roots := realRoots(writeRoots) all := []tool.Tool{ - readFile{workDir: w.Dir}, - writeFile{workDir: w.Dir, roots: roots}, - editFile{workDir: w.Dir, roots: roots}, - multiEdit{workDir: w.Dir, roots: roots}, + readFile{workDir: w.Dir, fileEncoding: w.FileEncoding}, + writeFile{workDir: w.Dir, roots: roots, fileEncoding: w.FileEncoding}, + editFile{workDir: w.Dir, roots: roots, fileEncoding: w.FileEncoding}, + multiEdit{workDir: w.Dir, roots: roots, fileEncoding: w.FileEncoding}, deleteRange{workDir: w.Dir, roots: roots}, deleteSymbol{workDir: w.Dir, roots: roots}, bash{workDir: w.Dir, sb: w.Bash}, diff --git a/internal/tool/builtin/writefile.go b/internal/tool/builtin/writefile.go index 22585e351..c4e9cbd0b 100644 --- a/internal/tool/builtin/writefile.go +++ b/internal/tool/builtin/writefile.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" + fileenc "reasonix/internal/fileutil/encoding" "reasonix/internal/tool" ) @@ -17,28 +18,30 @@ func init() { tool.RegisterBuiltin(writeFile{}) } // is overridden per run by ConfineWriters. workDir, when non-empty, is the // directory a relative path resolves against (see resolveIn). type writeFile struct { - roots []string - workDir string + roots []string + workDir string + fileEncoding string // project-level encoding override; empty = auto-detect from existing file } func (writeFile) Name() string { return "write_file" } func (writeFile) Description() string { - return "Write content to a file at the given path (overwriting existing content). Creates parent directories as needed." + return "Write content to a file at the given path (overwriting existing content). Creates parent directories as needed. When the target file already exists and no encoding is specified, the original file encoding is preserved." } func (writeFile) Schema() json.RawMessage { - return json.RawMessage(`{"type":"object","properties":{"path":{"type":"string","description":"File path"},"content":{"type":"string","description":"Full content to write"}},"required":["path","content"]}`) + return json.RawMessage(`{"type":"object","properties":{"path":{"type":"string","description":"File path"},"content":{"type":"string","description":"Full content to write"},"encoding":{"type":"string","description":"Target encoding (e.g. \"UTF-8\", \"GB18030\", \"UTF-16 LE\"). Defaults to the existing file's encoding, or UTF-8 for new files."}},"required":["path","content"]}`) } func (writeFile) ReadOnly() bool { return false } func (w writeFile) Execute(ctx context.Context, args json.RawMessage) (string, error) { var p struct { - Path string `json:"path"` - Content string `json:"content"` + Path string `json:"path"` + Content string `json:"content"` + Encoding string `json:"encoding,omitempty"` } - if err := json.Unmarshal(args, &p); err != nil { + if err := unmarshalArgs(args, &p); err != nil { return "", fmt.Errorf("invalid args: %w", err) } if p.Path == "" { @@ -53,8 +56,25 @@ func (w writeFile) Execute(ctx context.Context, args json.RawMessage) (string, e return "", fmt.Errorf("mkdir %s: %w", dir, err) } } - if err := os.WriteFile(p.Path, []byte(p.Content), 0o644); err != nil { + + // Determine target encoding: explicit override > project encoding > existing + // file encoding > UTF-8 (new files). + encParam := p.Encoding + if encParam == "" { + encParam = w.fileEncoding + } + var targetEnc fileenc.Kind + if forced, ok := fileenc.ParseName(encParam); ok { + targetEnc = forced + } else if existing, err := os.ReadFile(p.Path); err == nil { + targetEnc, _ = fileenc.Detect(existing) + } else { + targetEnc = fileenc.UTF8 + } + + data := fileenc.Encode(p.Content, targetEnc) + if err := os.WriteFile(p.Path, data, 0o644); err != nil { return "", fmt.Errorf("write %s: %w", p.Path, err) } - return fmt.Sprintf("wrote %d bytes to %s", len(p.Content), p.Path), nil + return fmt.Sprintf("wrote %d bytes to %s (%s)", len(data), p.Path, fileenc.Name(targetEnc)), nil }