From a1fad6142c83cbef612aab26c9466c38923ec88f Mon Sep 17 00:00:00 2001 From: Max Stepanov Date: Wed, 27 May 2026 14:37:20 +0300 Subject: [PATCH 01/10] tool for artifacts generation --- anton/core/session.py | 2 + .../core/tools/generate_artifact/__init__.py | 10 + .../tools/generate_artifact/data_resolver.py | 209 +++++++++ anton/core/tools/generate_artifact/engine.py | 298 +++++++++++++ anton/core/tools/generate_artifact/prompts.py | 412 ++++++++++++++++++ .../core/tools/generate_artifact/sub_tools.py | 152 +++++++ anton/core/tools/tool_defs.py | 182 ++++++++ anton/core/tools/tool_handlers.py | 78 ++++ 8 files changed, 1343 insertions(+) create mode 100644 anton/core/tools/generate_artifact/__init__.py create mode 100644 anton/core/tools/generate_artifact/data_resolver.py create mode 100644 anton/core/tools/generate_artifact/engine.py create mode 100644 anton/core/tools/generate_artifact/prompts.py create mode 100644 anton/core/tools/generate_artifact/sub_tools.py diff --git a/anton/core/session.py b/anton/core/session.py index b8e5536a..a295619b 100644 --- a/anton/core/session.py +++ b/anton/core/session.py @@ -36,6 +36,7 @@ from anton.core.tools.registry import ToolRegistry from anton.core.tools.tool_defs import ( CREATE_ARTIFACT_TOOL, + GENERATE_ARTIFACT_TOOL, LAUNCH_BACKEND_TOOL, LIST_ARTIFACTS_TOOL, MEMORIZE_TOOL, @@ -637,6 +638,7 @@ def _build_core_tools(self) -> None: self.tool_registry.register_tool(OPEN_ARTIFACT_TOOL) self.tool_registry.register_tool(UPDATE_ARTIFACT_METADATA_TOOL) self.tool_registry.register_tool(LAUNCH_BACKEND_TOOL) + self.tool_registry.register_tool(GENERATE_ARTIFACT_TOOL) async def close(self) -> None: """Clean up scratchpads and other resources.""" diff --git a/anton/core/tools/generate_artifact/__init__.py b/anton/core/tools/generate_artifact/__init__.py new file mode 100644 index 00000000..6dd2c1ff --- /dev/null +++ b/anton/core/tools/generate_artifact/__init__.py @@ -0,0 +1,10 @@ +"""Inner generator used by the experimental ``generate_artifact`` tool. + +Public entry point: ``generate(session, artifact_type, artifact_path, +context, data_refs)``. The outer tool handler validates input and reads +artifact metadata; everything below this surface is provider-agnostic. +""" + +from .engine import generate, MAX_ROUNDS + +__all__ = ["generate", "MAX_ROUNDS"] diff --git a/anton/core/tools/generate_artifact/data_resolver.py b/anton/core/tools/generate_artifact/data_resolver.py new file mode 100644 index 00000000..55221cd7 --- /dev/null +++ b/anton/core/tools/generate_artifact/data_resolver.py @@ -0,0 +1,209 @@ +"""Resolve ``data_refs`` — variable lookups against running scratchpads. + +Each ``{scratchpad, variable}`` pair is materialised into: + - a sidecar file under ``/_gen_data/.{pkl,json}`` + that the sub-agent can ``open()`` to access the raw value; + - a short textual summary embedded into the sub-agent's prompt so it + can reason about shape and types without re-loading the object. + +Extraction runs inside the source scratchpad subprocess via ``pad.execute`` +and goes through a ``__OK_DILL__`` marker (with a ``__OK_JSON__`` fallback +for non-picklable objects). The variable name is validated upfront as a +Python identifier so we never inject untrusted strings into the exec body. +""" + +from __future__ import annotations + +import base64 +import re +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from anton.chat_session import ChatSession + + +_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +_EXTRACT_TEMPLATE = """\ +import sys, json, traceback, base64 +try: + import dill as _pickle + _SCHEME = '__OK_DILL__' +except ImportError: + import pickle as _pickle + _SCHEME = '__OK_PICKLE__' +try: + _v = {var} +except NameError: + print('__MISSING__') + sys.exit(0) +try: + print(_SCHEME + base64.b64encode(_pickle.dumps(_v)).decode()) +except Exception: + try: + print('__OK_JSON__' + json.dumps(_v, default=str)) + except Exception: + print('__FAIL__' + traceback.format_exc()[-800:]) +""" + + +def _summarize(value: Any, max_chars: int = 800) -> str: + """Build a short, type-aware text summary for the sub-agent prompt.""" + try: + import pandas as pd # type: ignore + + if isinstance(value, pd.DataFrame): + head = value.head(5).to_string() + return ( + f"pandas.DataFrame shape={value.shape}\n" + f"columns={list(value.columns)}\n" + f"dtypes={ {c: str(t) for c, t in value.dtypes.items()} }\n" + f"head(5):\n{head}" + )[:max_chars] + if isinstance(value, pd.Series): + return ( + f"pandas.Series shape={value.shape} dtype={value.dtype}\n" + f"head(5):\n{value.head(5).to_string()}" + )[:max_chars] + except ImportError: + pass + + if isinstance(value, dict): + keys_preview = list(value.keys())[:20] + return ( + f"dict({len(value)} keys)\n" + f"sample_keys={keys_preview}\n" + f"repr={repr(value)[:max_chars]}" + )[:max_chars] + if isinstance(value, list): + sample = repr(value[:5]) + return ( + f"list({len(value)} items)\nsample={sample[:max_chars]}" + )[:max_chars] + return f"{type(value).__name__}: {repr(value)[:max_chars]}" + + +async def resolve_refs( + session: "ChatSession", + refs: list[dict], + sidecar_dir: Path, +) -> list[dict] | str: + """Extract each ref into a sidecar file + summary. + + Returns the list of resolved dicts on success or a single error + string on the first failure (no partial results — keeps the + contract simple for the main LLM). + """ + if not refs: + return [] + + sidecar_dir.mkdir(parents=True, exist_ok=True) + resolved: list[dict] = [] + + for ref in refs: + scratchpad = ref["scratchpad"] + variable = ref["variable"] + if not _IDENT_RE.match(variable): + return ( + f"Error: variable name `{variable}` is not a valid Python " + "identifier (must match [A-Za-z_][A-Za-z0-9_]*)." + ) + + pad = await session._scratchpads.get_or_create(scratchpad) + code = _EXTRACT_TEMPLATE.format(var=variable) + cell = await pad.execute( + code, + description=f"generate_artifact: extract `{variable}`", + estimated_seconds=15, + ) + + stdout = (cell.stdout or "").strip() + err = (cell.error or "").strip() + if err: + return ( + f"Error: extraction failed for `{variable}` in scratchpad " + f"`{scratchpad}`: {err[:300]}" + ) + + marker_line: str | None = None + for line in stdout.splitlines(): + if ( + line.startswith("__OK_DILL__") + or line.startswith("__OK_PICKLE__") + or line.startswith("__OK_JSON__") + or line.startswith("__MISSING__") + or line.startswith("__FAIL__") + ): + marker_line = line + break + + if marker_line is None: + return ( + f"Error: extractor produced no marker for `{variable}` in " + f"scratchpad `{scratchpad}`. stdout head: {stdout[:300]!r}" + ) + + if marker_line.startswith("__MISSING__"): + return ( + f"Error: variable `{variable}` not found in scratchpad " + f"`{scratchpad}`." + ) + if marker_line.startswith("__FAIL__"): + return ( + f"Error: extraction raised inside scratchpad `{scratchpad}` " + f"for `{variable}`: {marker_line[len('__FAIL__'):][:400]}" + ) + + if marker_line.startswith("__OK_DILL__") or marker_line.startswith( + "__OK_PICKLE__" + ): + scheme = "dill" if marker_line.startswith("__OK_DILL__") else "pickle" + tag_len = len("__OK_DILL__") if scheme == "dill" else len("__OK_PICKLE__") + payload_b64 = marker_line[tag_len:] + try: + raw_bytes = base64.b64decode(payload_b64) + except Exception as exc: + return f"Error: base64 decode failed for `{variable}`: {exc}" + + value: Any + try: + if scheme == "dill": + import dill as _pickle # type: ignore + else: + import pickle as _pickle + value = _pickle.loads(raw_bytes) + except Exception as exc: + return f"Error: {scheme} decode failed for `{variable}`: {exc}" + + sidecar_path = sidecar_dir / f"{variable}.pkl" + sidecar_path.write_bytes(raw_bytes) + resolved.append( + { + "scratchpad": scratchpad, + "variable": variable, + "format": scheme, + "sidecar_path": str(sidecar_path), + "summary": _summarize(value), + } + ) + continue + + # __OK_JSON__ — value isn't picklable but is JSON-serialisable. + payload_json = marker_line[len("__OK_JSON__"):] + sidecar_path = sidecar_dir / f"{variable}.json" + sidecar_path.write_text(payload_json, encoding="utf-8") + resolved.append( + { + "scratchpad": scratchpad, + "variable": variable, + "format": "json", + "sidecar_path": str(sidecar_path), + "summary": ( + "(json-only; not picklable) " + + payload_json[:800] + ), + } + ) + + return resolved diff --git a/anton/core/tools/generate_artifact/engine.py b/anton/core/tools/generate_artifact/engine.py new file mode 100644 index 00000000..cf70648c --- /dev/null +++ b/anton/core/tools/generate_artifact/engine.py @@ -0,0 +1,298 @@ +"""Async tool-call loop(s) that drive the inner generation LLM. + +For html-app: single loop. +For fullstack-stateless-app and fullstack-stateful-app: + 1. One-shot planning call → REST API specification (saved to _api_spec.md). + 2. asyncio.gather → backend loop + frontend loop in parallel. + +The caller is responsible for providing real data context: a `### Sample` +subsection inside `## Data` in the brief and/or `data_refs` pointing at +scratchpad variables. The engine no longer fabricates test data. + +The loop protocol is Anthropic tool-use / tool-result blocks, which both +providers Anton ships (AnthropicProvider, OpenAIProvider) accept on input. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import TYPE_CHECKING + +from . import sub_tools +from .data_resolver import resolve_refs +from .prompts import ( + build_api_spec_prompt, + build_backend_kickoff, + build_backend_system_prompt, + build_frontend_kickoff, + build_frontend_system_prompt, + build_subagent_system_prompt, + build_user_kickoff, +) + +if TYPE_CHECKING: + from anton.chat_session import ChatSession + + +MAX_ROUNDS = 12 + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +async def generate( + *, + session: "ChatSession", + artifact_type: str, + artifact_path: Path, + context: str, + data_refs: list[dict], +) -> dict | str: + """Drive the inner LLM(s) to populate ``artifact_path``. + + Returns either a result dict (on success) or a single error string. + """ + sidecar_dir = artifact_path / "_gen_data" + sidecar_dir.mkdir(parents=True, exist_ok=True) + + resolved = await resolve_refs(session, data_refs, sidecar_dir) + if isinstance(resolved, str): + return resolved + + # --- html-app: single generation loop, no changes -------------------- + if artifact_type == "html-app": + return await _run_loop( + session=session, + system=build_subagent_system_prompt("html-app", artifact_path), + kickoff=build_user_kickoff(context, resolved), + artifact_path=artifact_path, + ) + + # --- fullstack types: spec → parallel backend + frontend -------------- + if artifact_type not in ("fullstack-stateless-app", "fullstack-stateful-app"): + return f"Error: unsupported artifact type: {artifact_type!r}" + + api_spec_or_err = await _generate_api_spec(session, context, resolved) + if api_spec_or_err.startswith("Error:"): + return api_spec_or_err + api_spec = api_spec_or_err + (artifact_path / "_api_spec.md").write_text(api_spec, encoding="utf-8") + + stateless = artifact_type == "fullstack-stateless-app" + + backend_result, frontend_result = await asyncio.gather( + _run_loop( + session=session, + system=build_backend_system_prompt(artifact_path, stateless=stateless), + kickoff=build_backend_kickoff(context, resolved, api_spec), + artifact_path=artifact_path, + # Two-step backend generation: write backend.py first so that + # requirements.txt can be based on its actual imports. + step_injections=[ + ( + "backend.py", + "backend.py written. Now write requirements.txt listing " + "EVERY package imported in backend.py (one per line, no " + "extras). Then call finish.", + ), + ], + ), + _run_loop( + session=session, + system=build_frontend_system_prompt(artifact_path), + kickoff=build_frontend_kickoff(context, resolved, api_spec), + artifact_path=artifact_path, + ), + ) + + if isinstance(backend_result, str): + return f"Backend generation failed: {backend_result}" + if isinstance(frontend_result, str): + return f"Frontend generation failed: {frontend_result}" + + return { + "files_written": ( + backend_result["files_written"] + frontend_result["files_written"] + ), + "rounds_used": max( + backend_result["rounds_used"], frontend_result["rounds_used"] + ), + "summary": ( + f"backend: {backend_result['summary']} | " + f"frontend: {frontend_result['summary']}" + ), + } + + +# --------------------------------------------------------------------------- +# Pre-generation steps +# --------------------------------------------------------------------------- + + +async def _generate_api_spec( + session: "ChatSession", + context: str, + data_summaries: list[dict], +) -> str: + """One-shot planning call → REST API specification markdown.""" + system, user = build_api_spec_prompt(context, data_summaries) + response = await session._llm.plan( + system=system, + messages=[{"role": "user", "content": user}], + ) + spec = (response.content or "").strip() + if not spec: + return "Error: API spec generation returned empty response." + return spec + + +# --------------------------------------------------------------------------- +# Generic bounded tool-call loop +# --------------------------------------------------------------------------- + + +async def _run_loop( + *, + session: "ChatSession", + system: str, + kickoff: str, + artifact_path: Path, + step_injections: list[tuple[str, str]] | None = None, +) -> dict | str: + """Run one bounded sub-agent tool-call loop. + + ``step_injections`` is an optional list of ``(trigger_filename, message)`` + pairs. When a ``write_file`` call successfully writes ``trigger_filename``, + ``message`` is appended to the tool-result content so the model receives + the next-step instruction in the same turn. Each trigger fires at most once. + + Returns a result dict ``{files_written, rounds_used, summary}`` on + success, or a plain error string on failure. + """ + tools = sub_tools.tool_schemas() + messages: list[dict] = [{"role": "user", "content": kickoff}] + + files_written: list[str] = [] + finished_summary: str | None = None + injected: set[str] = set() + + for round_idx in range(MAX_ROUNDS): + # First round: use the planning model for highest-quality initial generation. + # Subsequent rounds (retries, read_file refinements) use the coding model. + llm_call = session._llm.plan if round_idx == 0 else session._llm.code + response = await llm_call( + system=system, + messages=messages, + tools=tools, + ) + + if not response.tool_calls: + tail = (response.content or "").strip() + return ( + f"generator stopped without writing files " + f"(round {round_idx + 1}/{MAX_ROUNDS}). " + f"Last output: {tail[:300]!r}" + ) + + assistant_blocks: list[dict] = [] + if response.content: + assistant_blocks.append({"type": "text", "text": response.content}) + for tc in response.tool_calls: + assistant_blocks.append( + { + "type": "tool_use", + "id": tc.id, + "name": tc.name, + "input": tc.input, + } + ) + messages.append({"role": "assistant", "content": assistant_blocks}) + + result_blocks: list[dict] = [] + for tc in response.tool_calls: + if tc.parse_error: + result_blocks.append( + { + "type": "tool_result", + "tool_use_id": tc.id, + "content": ( + "Error: malformed tool input — re-emit with valid " + f"JSON. ({tc.parse_error})" + ), + } + ) + continue + + name = tc.name + inp = tc.input or {} + + if name == "finish": + summary = str(inp.get("summary") or "").strip() + finished_summary = summary or "(no summary)" + result_blocks.append( + {"type": "tool_result", "tool_use_id": tc.id, "content": "ok"} + ) + elif name == "write_file": + res = sub_tools.write_file( + artifact_path, + inp.get("path", ""), + inp.get("content", ""), + ) + msg = res["message"] + if res.get("ok"): + written = res["written"] + if written not in files_written: + files_written.append(written) + for trigger, inject_msg in (step_injections or []): + if written == trigger and inject_msg not in injected: + injected.add(inject_msg) + msg = f"{msg}\n\n{inject_msg}" + result_blocks.append( + { + "type": "tool_result", + "tool_use_id": tc.id, + "content": msg, + } + ) + elif name == "read_file": + res = sub_tools.read_file(artifact_path, inp.get("path", "")) + result_blocks.append( + { + "type": "tool_result", + "tool_use_id": tc.id, + "content": res["message"], + } + ) + else: + result_blocks.append( + { + "type": "tool_result", + "tool_use_id": tc.id, + "content": ( + f"Error: unknown sub-tool `{name}`. " + "Use write_file, read_file, or finish." + ), + } + ) + + messages.append({"role": "user", "content": result_blocks}) + + if finished_summary is not None: + break + else: + return ( + f"generator exceeded round budget ({MAX_ROUNDS}) after writing " + f"{len(files_written)} file(s): {files_written}." + ) + + if not files_written: + return "generator finished without writing any files." + + return { + "files_written": files_written, + "rounds_used": round_idx + 1, + "summary": finished_summary, + } diff --git a/anton/core/tools/generate_artifact/prompts.py b/anton/core/tools/generate_artifact/prompts.py new file mode 100644 index 00000000..c488aad0 --- /dev/null +++ b/anton/core/tools/generate_artifact/prompts.py @@ -0,0 +1,412 @@ +"""System + kickoff prompts for the inner generation LLM. + +All prompt text lives here as dedicated constants — we do NOT re-use the +main-agent's ``BACKEND_GENERATION_PROMPT`` or ``VISUALIZATIONS_HTML_OUTPUT_FORMAT_PROMPT`` +verbatim because those are written for the *outer* agent's workflow (they include +artifact registration, scratchpad-cell discipline, ``launch_backend``, etc. +that are irrelevant and confusing to the sub-agent). + +Instead we extract only the technical rules relevant to the sub-agent's job: +write the files, nothing else. +""" + +from __future__ import annotations + +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Role / tool contract (shared across all artifact types) +# --------------------------------------------------------------------------- + +_ROLE = """\ +You are a focused code-generator. Your ONLY job is to produce the files for +one artifact by calling `write_file`, then call `finish` when done. + +HARD RULES: +- Call `write_file` exactly once per file with the COMPLETE contents. + Do NOT split a single file across multiple calls. +- All `path` values are RELATIVE to the artifact folder — never write outside it. +- Do not access the network. Use only what is in the brief and the pre-fetched + data sidecar files. +- Call `finish(summary="")` exactly once when all files are written. + +AVAILABLE TOOLS: +- `write_file(path, content)` — write a UTF-8 text file at `/`. +- `read_file(path)` — read a file you already wrote (for iterative refinement). +- `finish(summary)` — terminate generation with a one-line summary.\ +""" + + +# --------------------------------------------------------------------------- +# Visual design rules (used in every type that has a frontend) +# --------------------------------------------------------------------------- + +_VISUAL_RULES = """\ +VISUAL DESIGN (for every HTML file you produce): +- Dark theme: background #0d1117, text #e6edf3. + System sans-serif font stack, generous padding, responsive layout. +- ALWAYS use Apache ECharts for interactive charts via CDN: + `` + Initialise with `echarts.init(dom, 'dark')` and customise background to #0d1117. + NEVER use Plotly, matplotlib, or other chart libraries unless explicitly asked. +- Line smoothing: `smooth: false` on ALL line series by default. + Use `smooth: true` ONLY for cumulative / monotonic series (running totals, growth curves). + Line widths: 2.5 for primary, 1.5 for comparisons, 1 for reference lines. +- Chart readability: + - `axisLabel: { rotate: -45 }` on crowded axes. + - `grid: { containLabel: true }` so labels never clip. + - `legend: { type: 'scroll', bottom: 0 }` for many series. + - Pie/donut: `label: { show: true, position: 'outside' }` + `labelLayout: { hideOverlap: true }`. + - Rich `tooltip` with `formatter` functions for precise hover values. + - `dataZoom` on time series so users can zoom. +- Multi-tab dashboards: NEVER call `echarts.init()` on a hidden container. + Use lazy init — initialise charts only on first tab visibility. + Pattern: `const _rendered = new Set(['overview']); function showPage(name) { if (!_rendered.has(name)) { _rendered.add(name); initChartsFor(name); } }` +- Layout composition: + - Hero KPI cards at the top (large numbers, colour-coded ±, delta arrows). + - Main narrative chart immediately below KPIs. + - Supporting charts below, each with a subtitle explaining what it reveals. + - Use ECharts `markLine` for thresholds, `markPoint` for outliers, + `markArea` for highlighted regions. +- Responsive: + - `` + - Multi-card grid: `grid-template-columns: repeat(auto-fit, minmax(360px, 1fr))` + - Chart containers: `width: 100%; height: min(420px, 60vh)` + - Register `window.addEventListener('resize', () => chart.resize())` on every ECharts instance. + - Tables wrapped in `
` — never fixed widths. +- SECURITY: NEVER embed API keys, tokens, passwords, or connection strings in HTML/JS. + Credentials were already used server-side; serialise only the resulting data.\ +""" + + +# --------------------------------------------------------------------------- +# Backend rules (fullstack types only) +# --------------------------------------------------------------------------- + +_BACKEND_RULES = """\ +BACKEND — `backend.py` (FastAPI, runs locally AND as AWS Lambda): + +Use this canonical skeleton verbatim, add routes inside `# === API routes ===`: + +```python +import argparse +from pathlib import Path +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.staticfiles import StaticFiles +from mangum import Mangum + +app = FastAPI() + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + +# === API routes === +@app.get("/api/hello") +async def hello(): + return {"hello": "world"} + +STATIC_DIR = Path(__file__).parent / "static" +if STATIC_DIR.exists(): + app.mount("/", StaticFiles(directory=str(STATIC_DIR), html=True), name="static") + +handler = Mangum(app, lifespan="off") + +if __name__ == "__main__": + import uvicorn + parser = argparse.ArgumentParser() + parser.add_argument("--port", type=int, required=True) + args = parser.parse_args() + uvicorn.run(app, host="127.0.0.1", port=args.port) +``` + +CRITICAL RULES: +- File MUST be named `backend.py`. The `handler` attribute MUST stay `handler`. +- ALL API endpoints MUST use the `/api/*` prefix (e.g. `/api/items`, `/api/search`). + Never expose routes at the root — they collide with the `StaticFiles` mount. +- API routes MUST be registered BEFORE `app.mount("/", StaticFiles(...))`. +- The backend MUST accept `--port` via argparse. NEVER hardcode a port. +- Keep `Mangum(app, lifespan="off")`. Required for Lambda cold-start. +- Use `async def` for I/O-bound routes (DB queries, external HTTP). +- STATELESS: no module-level mutable caches. Lambda globals are unreliable. +- FILESYSTEM: assume read-only at runtime (Lambda). Only `/tmp` is writable. +- LOGGING: `print()` and `logging.getLogger(__name__).info(...)` work everywhere. + +`requirements.txt` — always include at minimum: +``` +fastapi +mangum +uvicorn +``` +Add any other packages the backend imports, one per line (`pkg` or `pkg==1.2`).\ +""" + + +# --------------------------------------------------------------------------- +# Frontend rules for fullstack types +# --------------------------------------------------------------------------- + +_FRONTEND_RULES = """\ +FRONTEND — `static/index.html`: + +- Single self-contained HTML file. Inline all CSS in `