diff --git a/.github/workflows/openwiki-update.yml b/.github/workflows/openwiki-update.yml new file mode 100644 index 0000000..74bebc2 --- /dev/null +++ b/.github/workflows/openwiki-update.yml @@ -0,0 +1,62 @@ +name: OpenWiki Update + +on: + workflow_dispatch: + schedule: + - cron: "0 8 * * *" + +permissions: + contents: write + pull-requests: write + +jobs: + update: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + # Full history so `openwiki code --update` can diff HEAD against the + # commit it last documented; a shallow clone hides that commit and the + # update runs against an empty change summary. + fetch-depth: 0 + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "22" + + - name: Install OpenWiki + # mermaid + jsdom are optional; they add high-fidelity validation of Mermaid diagrams. Remove if your wiki has none. + run: npm install --global openwiki@0.3.3 mermaid@11.16.0 jsdom@29.1.1 + + - name: Run OpenWiki + run: openwiki code --update --print + env: + OPENWIKI_PROVIDER: baseten + BASETEN_API_KEY: ${{ secrets.BASETEN_API_KEY }} + OPENWIKI_MODEL_ID: "moonshotai/Kimi-K3" + # Required for the LangSmith connector's code-mode pull to authenticate. + # For extra workspaces, add OPENWIKI_LANGSMITH_API_KEY_2, _3, ... as repo + # secrets and env entries here. + OPENWIKI_LANGSMITH_API_KEY: ${{ secrets.OPENWIKI_LANGSMITH_API_KEY }} + # Optional: also trace this workflow's own OpenWiki run to LangSmith. + LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} + LANGCHAIN_PROJECT: openwiki + LANGCHAIN_TRACING_V2: "true" + + - name: Create OpenWiki update pull request + uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 # v7 + with: + add-paths: | + openwiki + AGENTS.md + CLAUDE.md + .github/workflows/openwiki-update.yml + branch: openwiki/update + commit-message: "docs: update OpenWiki" + title: "docs: update OpenWiki" + body: | + Automated OpenWiki documentation update. + + This PR was generated by the scheduled OpenWiki workflow. diff --git a/AGENTS.md b/AGENTS.md index 5486fc6..5f922e4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -103,3 +103,16 @@ Before ending a work session: 5. Hand off remaining context clearly. Do not commit or push unrelated uncommitted user changes. + + + +## OpenWiki + +This repository has a generated `openwiki/` evidence index. It is optional just-in-time context, not required startup reading. + +- Treat source code and tests as authoritative. A brief's unknowns and review items are verification gaps, not automatic requirements. +- Prefer the narrowest quiet validation that proves the changed behavior. Preserve complete failure output. + +The scheduled OpenWiki GitHub Actions workflow refreshes the repository wiki. Do not hand-edit generated OpenWiki pages unless explicitly asked; prefer updating source code/docs and letting OpenWiki regenerate. + + diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..4ed2390 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,7 @@ + + +## OpenWiki + +See [AGENTS.md](AGENTS.md) for OpenWiki agent instructions. + + diff --git a/agentic_internet/tools/code_execution.py b/agentic_internet/tools/code_execution.py index 39ad002..850e013 100644 --- a/agentic_internet/tools/code_execution.py +++ b/agentic_internet/tools/code_execution.py @@ -1,6 +1,7 @@ """Code execution tool for agents.""" import ast +import builtins import io import json import logging @@ -20,7 +21,30 @@ logger = logging.getLogger(__name__) +_BUILTIN_IMPORT = builtins.__import__ + + +def _guarded_import( + name: str, + globals_: dict[str, Any] | None = None, + locals_: dict[str, Any] | None = None, + fromlist: tuple[str, ...] = (), + level: int = 0, +) -> Any: + """Restricted ``__import__`` for executed code. + + C extensions (e.g. numpy reductions) import submodules at runtime through + the frame's ``__builtins__``; without this entry they crash with + ``KeyError: '__import__'``. The guard enforces the same module blocklist + as the AST validator, so aliased indirect imports stay blocked. + """ + if name.split(".")[0] in _ASTSafetyValidator.BLOCKED_MODULES: + raise ImportError(f"import of blocked module '{name}' is not allowed") + return _BUILTIN_IMPORT(name, globals_, locals_, fromlist, level) + + ALLOWED_BUILTINS = { + "__import__": _guarded_import, "print": print, "len": len, "range": range, diff --git a/docs/exec-plans/active/.gitkeep b/docs/exec-plans/active/.gitkeep new file mode 100644 index 0000000..61efb03 --- /dev/null +++ b/docs/exec-plans/active/.gitkeep @@ -0,0 +1 @@ +In-flight exec-plans live here (see docs/PLANS.md). Completed plans move to docs/exec-plans/completed/. diff --git a/openwiki/.last-update.json b/openwiki/.last-update.json new file mode 100644 index 0000000..31057ca --- /dev/null +++ b/openwiki/.last-update.json @@ -0,0 +1,8 @@ +{ + "updatedAt": "2026-08-15T16:23:09.917Z", + "command": "init", + "gitHead": "bea3d8316ab249a8ae63910e4a1186ec147b51f4", + "model": "gpt-5.6-sol", + "status": "complete", + "language": "en" +} diff --git a/openwiki/INSTRUCTIONS.md b/openwiki/INSTRUCTIONS.md new file mode 100644 index 0000000..29fc498 --- /dev/null +++ b/openwiki/INSTRUCTIONS.md @@ -0,0 +1 @@ +A code wiki for this repository. diff --git a/openwiki/agents/index.md b/openwiki/agents/index.md new file mode 100644 index 0000000..14d036c --- /dev/null +++ b/openwiki/agents/index.md @@ -0,0 +1,4 @@ +# Files + +- [Internet and Research Agents](internet-and-research.md) - Construction, tool registration, execution, chat, and research-history behavior for the core single-agent runtime. +- [Specialized Agents](specialized-agents.md) - Public domain-specific facades for browser automation, data analysis, content creation, market research, and technical support. diff --git a/openwiki/agents/internet-and-research.md b/openwiki/agents/internet-and-research.md new file mode 100644 index 0000000..a0cabaf --- /dev/null +++ b/openwiki/agents/internet-and-research.md @@ -0,0 +1,76 @@ +--- +type: component guide +title: Internet and Research Agents +description: Construction, tool registration, execution, chat, and research-history behavior for the core single-agent runtime. +tags: [agents, runtime, research] +--- + +# Internet and Research Agents + +`agentic_internet/agents/internet_agent.py` owns the default single-agent path. `InternetAgent` is the base public facade; `ResearchAgent` adds depth-specific prompts and in-memory history. CLI `chat`, `run`, `research`, and default `tools` all enter here. + +## Construction + +```mermaid +sequenceDiagram + participant Caller + participant IA as InternetAgent + participant MU as model_utils + participant CFG as settings + participant SA as smolagents + Caller->>IA: construct model tools and agent_type + IA->>MU: initialize_model + MU->>CFG: resolve alias provider and key + MU-->>IA: model + IA->>IA: assemble default tools when tools is falsey + IA->>SA: create ToolCallingAgent or CodeAgent + SA-->>IA: executable agent +``` + +*Construction resolves the model before default tools and the underlying smolagents agent.* + +`__init__` accepts `model_id`, `tools`, `verbose`, `max_iterations`, `planning_enabled`, `agent_type`, and extra authorized imports. Important invariants: + +- `tools or self._get_default_tools()` means `tools=[]` cannot express a tool-free agent; supply a nonempty custom list or change this behavior deliberately. +- `_initialize_model` delegates to `initialize_model`; a falsey result raises `ModelInitializationError` during construction. +- `agent_type == "code"` creates `CodeAgent(max_steps=max_iterations)` with fixed authorized imports (`pandas`, `numpy`, JSON/CSV/regex/date/time, `requests`, `urllib`, math/statistics/collections) plus caller additions. +- Every other value creates `ToolCallingAgent`; unknown values are not rejected. The tool-calling branch does not pass `max_iterations`. +- `planning_enabled` is stored but does not alter agent creation or execution. + +The broader isolation implications are compared in [System Architecture](../architecture/overview.md). + +## Default tools + +`_get_default_tools` uses the import-time `settings` singleton: + +| Gate | Registered tools | +|---|---| +| `settings.tools.web_search_enabled` | `WebSearchTool`, `WebScraperTool`, `NewsSearchTool` | +| Above plus `settings.exa_api_key` | `ExaSearchTool`, `ExaFindSimilarTool` | +| `settings.tools.browser_enabled` plus `settings.browser_use_api_key` | sync, async, and structured Browser Use tools | +| `settings.tools.code_execution_enabled` | `PythonExecutorTool`, `DataAnalysisTool` | +| Best-effort always | `smolagents.load_tool("calculator")`; failure is debug-logged and ignored | + +MCP tools are never defaults; the [MCP CLI path](../tools/mcp-integration.md) injects them explicitly. Tool contracts live under [Web Search](../tools/web-search-and-scraping.md), [Exa](../tools/exa-search.md), [Browser Automation](../tools/browser-automation.md), and [Code Execution](../tools/code-execution-and-data.md). + +## Run and chat lifecycle + +`run(task, show_result=True, **kwargs)` optionally renders the task, calls `self.agent.run(task, **kwargs)`, optionally renders returned Markdown, and returns `str(result)`. It catches every exception, logs the traceback, and returns `"Error executing task: ..."`. Callers must inspect outcomes; exception-to-string conversion often leaves CLI exit status zero. + +`chat()` is a terminal loop. `exit`, `quit`, and `bye` stop; `help` and `tools` display local help; all other input calls `run`. A local `{user, agent}` history is appended but never returned or persisted. `KeyboardInterrupt` exits, while other loop errors are printed and the loop continues. `_show_tools` expects each tool to expose a string `description`. + +## ResearchAgent + +`ResearchAgent` delegates construction and initializes `research_history`. `research(topic, depth)` selects fixed `quick`, `moderate`, or `deep` prompts, calls `run(show_result=False)`, then appends and returns: + +```text +{topic, depth, findings, timestamp} +``` + +The timestamp is `pandas.Timestamp.now().isoformat()`. An unknown depth uses the moderate prompt but preserves the unknown value in returned metadata; the CLI prevents this through its own validation. `get_research_history()` returns the mutable internal list, not a defensive copy. + +## Extension and validation + +Override `_initialize_model`, `_get_default_tools`, or `_create_agent` for controlled variants; prefer injecting a nonempty tool list for consumers. When adding a default tool, implement and export it under `agentic_internet.tools`, add the correct setting/key gate here, update any K-LLM inventory separately, and test both enabled and disabled registration. + +Focused coverage is currently narrow: `tests/test_exa_search.py` proves conditional Exa registration. There is no isolated test for model failure, agent construction, `run`, chat, `ResearchAgent`, planning, or tool-calling iteration limits. Use `uv run pytest tests/test_exa_search.py` plus the focused suite for the tool/model code changed; run `uv run pytest` for constructor-wide changes. \ No newline at end of file diff --git a/openwiki/agents/specialized-agents.md b/openwiki/agents/specialized-agents.md new file mode 100644 index 0000000..40e2df6 --- /dev/null +++ b/openwiki/agents/specialized-agents.md @@ -0,0 +1,38 @@ +--- +type: component guide +title: Specialized Agents +description: Public domain-specific facades for browser automation, data analysis, content creation, market research, and technical support. +tags: [agents, public-api] +--- + +# Specialized Agents + +`agentic_internet/agents/specialized_agents.py` provides five root-exported subclasses of [InternetAgent](internet-and-research.md). They do not introduce new execution engines: each chooses constructor defaults, creates a detailed prompt, calls inherited `run`, and sometimes wraps the string in metadata. Prompt requests are intentions, not postcondition enforcement. + +## Agent contracts + +| Class and defaults | Methods and returned shape | +|---|---| +| `BrowserAutomationAgent`, `agent_type="code"`, `max_iterations=20` | `scrape_structured_data(url, data_schema)` requests JSON and returns a decoded dict only when the final text is a JSON object; arrays/scalars/invalid JSON become `{"raw_result": result}`. `fill_form` returns text. `monitor_website` performs one check and returns URL, target, findings, and a suggested interval—it does not schedule monitoring. | +| `DataAnalysisAgent`, code mode | `analyze_dataset(data, analysis_type)` returns `{analysis_type, insights}`. `compare_datasets` turns optional criteria into prompt bullets and returns text. Inputs are string-interpolated without size or schema controls. | +| `ContentCreationAgent`, tool-calling mode | `write_article(topic, style, word_count, sources_required)` returns topic/style/content/target count; it does not verify count or citations. `summarize_content(content, summary_type)` returns text. | +| `MarketResearchAgent`, tool-calling mode, `max_iterations=15` | `analyze_competitor` defaults to products, pricing, position, strengths, and weaknesses and returns `{company, aspects, analysis}`. `market_trends` returns `{industry, timeframe, trends}`. | +| `TechnicalSupportAgent`, code mode | `troubleshoot(problem_description, system_info)` JSON-renders optional system information and returns `{problem, system_info, solution}`. `code_review(code, language, focus_areas)` embeds a language fence and returns language, normalized focus list, and review. | + +All inherited failures usually appear inside a result string because `InternetAgent.run` catches exceptions. A metadata wrapper can therefore look structurally successful while its content starts with `Error executing task:`. Code-mode classes inherit the local execution/network implications documented in [System Architecture](../architecture/overview.md). + +## Ownership and dependencies + +The module depends only on JSON handling and `InternetAgent`; actual models and tools come from the base class. `agentic_internet/__init__.py` and `agents/__init__.py` expose all five classes. `examples/advanced_usage.py` demonstrates selected methods but requires real model/provider configuration and is not a hermetic test. + +## Extension recipe + +For another domain facade: + +1. Subclass `InternetAgent` and set defaults with `kwargs.setdefault` so callers can override them. +2. Keep side effects in registered tools rather than the prompt wrapper. +3. Define whether the method returns raw text or a stable dict. If parsing model output, preserve malformed output explicitly rather than silently discarding it. +4. Export from `agents/__init__.py` and root `agentic_internet/__init__.py` if it is public. +5. Add a focused test with construction bypassed or model/tool mocks, checking prompt-critical behavior and malformed/failure output—not only metadata. + +`tests/test_specialized_agents.py` verifies object-versus-nonobject JSON handling for `scrape_structured_data` and article metadata. Constructor defaults and the other helper methods have no focused coverage. Run that file for facade changes and the base/tool suites when changing inherited behavior. \ No newline at end of file diff --git a/openwiki/architecture/index.md b/openwiki/architecture/index.md new file mode 100644 index 0000000..4cef8ba --- /dev/null +++ b/openwiki/architecture/index.md @@ -0,0 +1,3 @@ +# Files + +- [System Architecture](overview.md) - Runtime layers, dependency flow, orchestration paths, state ownership, and cross-cutting trust boundaries for Agentic Internet. diff --git a/openwiki/architecture/overview.md b/openwiki/architecture/overview.md new file mode 100644 index 0000000..4c4382d --- /dev/null +++ b/openwiki/architecture/overview.md @@ -0,0 +1,80 @@ +--- +type: architecture guide +title: System Architecture +description: Runtime layers, dependency flow, orchestration paths, state ownership, and cross-cutting trust boundaries for Agentic Internet. +tags: [architecture, security, runtime] +--- + +# System Architecture + +Agentic Internet is one Python 3.11+ library and Typer CLI. It composes `smolagents` models and agents with local and hosted tools; it is not a web service and owns no database, queue, migration set, or durable application state. `pyproject.toml` installs `agentic-internet = agentic_internet.cli:app`; `agentic_internet/__main__.py` provides `python -m agentic_internet`. Root `main.py` is unrelated scaffold code. + +## Layer and ownership map + +```mermaid +flowchart TD + Human["CLI or Python caller"] --> Interfaces["cli.py and package exports"] + Interfaces --> Core["InternetAgent and ResearchAgent"] + Interfaces --> Orch["Search, Code Mode, and K-LLM orchestration"] + Core --> Models["settings.py and model_utils.py"] + Core --> Tools["Web, Exa, browser, code, and MCP tools"] + Orch --> Models + Orch --> Tools + Tools --> Providers["Search APIs, Browser Use Cloud, MCP servers"] + Models --> LLMs["OpenRouter or direct model providers"] +``` + +*The public interfaces select an agent/orchestrator; agents own behavior, tools own side effects, and configuration resolves model/provider boundaries.* + +| Layer | Owners | Rule | +|---|---|---| +| Interfaces | `cli.py`, root/subpackage `__init__.py` | Parse and route; library behavior remains in package modules. | +| Configuration | `config/settings.py`, `utils/model_utils.py`, `utils/openrouter_models.py` | Resolve aliases, providers, keys, and live model metadata. | +| Agents | `agents/internet_agent.py`, `basic_agent.py`, `specialized_agents.py` | Compose models and tools and normalize final results. | +| Orchestration | `search_orchestrator.py`, `code_mode.py`, `multi_model_serpapi.py`, recipes/runtime | Coordinate workers, tool facades, synthesis, and context. | +| Capabilities | `tools/` | Isolate external HTTP/SDK/MCP calls and local execution. | +| Verification | `tests/` | Focused mocked/unit behavior; root `test_*.py` scripts are excluded live checks. | + +See [Settings and Models](../configuration/settings-and-models.md) for provider resolution, [Python API](../interfaces/python-api.md) for exports, and [CLI](../interfaces/cli.md) for all routes. + +## Three orchestration paths + +1. **Single agent:** `InternetAgent` resolves a model, assembles default or supplied tools, creates `ToolCallingAgent` or `CodeAgent`, then delegates `run`. `ResearchAgent` and the specialized agents are prompt/output facades over this path. +2. **Search orchestrator:** `SearchOrchestrator` runs named workers sequentially or in a thread pool, aggregates outcomes, and optionally invokes a synthesis `CodeAgent`. `WebSearchTool` can use it, then fall back to direct search. +3. **K-LLM:** `MultiModelSerpAPISystem` resolves a declarative recipe into tool bundles and worker agents, wraps workers as `AgentTool`s, and exposes them to a coordinator `CodeAgent`. Routing policy is prompt guidance, not a separate scheduler. + +Code Mode is a fourth composition surface rather than an independent planner: it gives a `CodeAgent` only `search` and `execute` meta-tools, with real tools hidden behind `ToolFacade`. + +## State and lifecycle + +Runtime state is process-local: + +- `ResearchAgent.research_history`, search-orchestrator history/counters, and K-LLM `ContextWindow`, `AgentMemory`, and `TaskContext` disappear with the process. +- Browser and MCP clients represent external connection lifecycles; MCP tools are valid only while their context manager remains open. +- `Settings.model_post_init` creates `~/.cache/agentic_internet`, but package code does not persist application records there. +- External providers own remote task/model/search state. API credentials enter through environment-derived settings or direct `os.getenv` calls. + +## Canonical code-execution trust comparison + +“Code execution” names materially different boundaries. None should be treated as safe for arbitrary untrusted input without a separately enforced sandbox. + +| Path | Execution and imports | Side effects and limits | Fallback and evidence | +|---|---|---|---| +| `PythonExecutorTool` | Built-in `exec` after `_ASTSafetyValidator`; restricted builtins; preloads `numpy`, `pandas`, `requests`, `json`, math/time utilities. Blocks selected modules, calls, and dunder attributes. | Fresh bindings per call and 10,000-character final output, but no enforced timeout, CPU, memory, process, or network isolation. `requests` permits network access; traceback text is returned. | No remote executor. `tests/test_code_execution.py` checks major denylist cases and output behavior, not timeout, network, or exhaustion. See [Code Execution and Data](../tools/code-execution-and-data.md). | +| Code Mode `ExecuteTool` | Fresh smolagents `LocalPythonExecutor` per call with `api`, `json`, and `print`; default authorized imports include `os`, plus caller additions. The facade can invoke every wrapped local or MCP tool. | State does not persist between calls. Max print output is 15,000, but no repository-enforced resource timeout. `os` and facade tools amplify filesystem, network, and remote side effects. | Factory request for E2B silently falls back to local when `E2B_API_KEY` is absent. Tests cover facade execution and fallback construction, not isolation or live E2B. See [Code Mode](../orchestration/code-mode.md). | +| `InternetAgent(agent_type="code")` | smolagents `CodeAgent`; authorizes data/JSON/time/math plus `requests` and `urllib`, and caller additions. | `max_iterations` becomes `max_steps`, not a wall-clock/resource bound. Network-capable imports and registered tools remain side-effecting. | No remote executor selection here and no focused construction/security tests. See [Internet and Research Agents](../agents/internet-and-research.md). | +| K-LLM workers/coordinator | Code-capable recipe workers and the always-code coordinator use broad authorized imports and direct/worker tools. | Workflow has `asyncio.wait_for` around coordinator completion, but that is orchestration timeout, not executor resource isolation. SerpAPI, web, code, and worker tools may perform external effects. | Worker type depends on model heuristics; partial teams are allowed. No focused end-to-end coordinator or timeout tests. See [K-LLM Use Cases](../orchestration/k-llm-use-cases.md). | + +MCP increases the boundary: discovered tools can run remote code or services, and Code Mode makes them callable from generated Python. Keep the MCP discovery context open, require explicit trust at the application boundary, and do not assume AST policy protects a separate CodeAgent or remote tool. See [MCP Integration](../tools/mcp-integration.md). + +## External trust boundaries + +- Web scraping follows redirects and validates only HTTP(S) syntax; private/link-local destinations and response size before parsing are not blocked. +- Browser tasks and task text leave the process for Browser Use Cloud. +- MCP stdio can execute an arbitrary configured command/path and receives the full parent environment; HTTP endpoints lack local host/scheme policy. +- `config --show` serializes settings containing API-key fields. Do not expose its output in logs. +- Most runtime failures become human-readable result strings, so a successful process exit does not guarantee a successful agent/provider operation. + +## Change discipline + +Preserve dependency direction: tools must not import CLI or agents; agents compose tools; interfaces route into agents. A public extension requires implementation, subpackage/root exports where intended, default registration or recipe inventory changes, consumer import documentation, and a focused test. Use the intent table in [Quickstart](../quickstart.md) and validation ownership in [Testing and Operations](../development/testing-and-operations.md). \ No newline at end of file diff --git a/openwiki/configuration/index.md b/openwiki/configuration/index.md new file mode 100644 index 0000000..bd1f8ac --- /dev/null +++ b/openwiki/configuration/index.md @@ -0,0 +1,3 @@ +# Files + +- [Settings and Models](settings-and-models.md) - Pydantic configuration lifecycle, environment bindings, model alias and provider resolution, fallbacks, and catalog ownership. diff --git a/openwiki/configuration/settings-and-models.md b/openwiki/configuration/settings-and-models.md new file mode 100644 index 0000000..4f0f897 --- /dev/null +++ b/openwiki/configuration/settings-and-models.md @@ -0,0 +1,52 @@ +--- +type: configuration guide +title: Settings and Models +description: Pydantic configuration lifecycle, environment bindings, model alias and provider resolution, fallbacks, and catalog ownership. +tags: [configuration, models, providers] +--- + +# Settings and Models + +`agentic_internet/config/settings.py` defines configuration schemas and creates the import-time `settings = Settings()` singleton. `utils/model_utils.py` turns settings into smolagents model objects; `utils/openrouter_models.py` fetches a live public catalog for CLI display. + +## Schemas and environment lifecycle + +`ModelConfig` owns default model (`openrouter/anthropic/claude-opus-4.8`), provider `auto`, sampling/token settings, and OpenRouter/OpenAI/Anthropic/Hugging Face alias maps. `AgentConfig` owns verbosity, iteration, memory/tool-choice/planning values. `ToolConfig` owns web/code/browser/file flags and search-result count. `Settings` contains provider/tool keys, nested configs, cache path, and log level. + +`load_dotenv()` runs on import. Explicit field factories bind `HUGGINGFACE_TOKEN`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `OPENROUTER_API_KEY`, `BROWSER_USE_API_KEY`, and `EXA_API_KEY`. SerpAPI modules read `SERPAPI_API_KEY` directly; it is not a `Settings` field. Because `Settings` is `BaseModel`, not `BaseSettings`, `.env.example` names `ENVIRONMENT`, `LOG_LEVEL`, `MODEL_NAME`, `MODEL_TEMPERATURE`, and `MAX_TOKENS` do not configure fields. Nested model values require programmatic mutation/construction. + +`model_post_init` creates `~/.cache/agentic_internet`. The singleton snapshots environment at import; later `os.environ` changes do not refresh it. `validate_startup` warns about every missing model-provider key, inserts a no-provider-key warning when all are absent, and warns about browser configuration, but not SerpAPI or Exa. It is a callable helper and is not automatically invoked by singleton construction or the CLI. `config --show` calls `settings.model_dump()` and renders the complete nested object without a redaction pass, including key-bearing fields; `--set` does not mutate anything. Never document or print real keys. + +## Model resolution + +```mermaid +flowchart TD + Requested["Requested ID or configured default"] --> Alias["resolve_model_id"] + Alias --> Provider["get_model_provider"] + Provider --> Key{"Provider key available"} + Key -->|yes| Build["Create LiteLLMModel or InferenceClientModel"] + Key -->|no| Fallback["get_any_available_model"] + Build --> Success{"Construction succeeds"} + Success -->|yes| Return["Return model"] + Success -->|no| Fallback + Fallback --> Order["OpenRouter then OpenAI then Anthropic then Hugging Face"] + Order --> Return +``` + +*Unknown provider, missing key, or constructor failure enters the same provider-priority fallback.* + +Alias lookup order is OpenRouter, OpenAI, Anthropic, then Hugging Face. Overlapping short aliases therefore prefer OpenRouter; use an unambiguous provider ID to select a direct API. Provider detection then follows this exact order: resolved IDs in the four explicit maps; an `openrouter/` marker; OpenAI name patterns; `claude`; known slash-prefixed providers routed to OpenRouter only when its key exists; Hugging Face organization patterns; and finally, only when configured provider is `auto`, a slash ID with OpenRouter key followed by available OpenAI, Anthropic, then Hugging Face credentials. A non-`auto` configured provider is the final answer when earlier checks do not match; otherwise provider is `None`. + +`get_api_key_for_provider` selects `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or `HUGGINGFACE_TOKEN`. `get_default_model_for_provider` reads these configured fallback IDs: OpenRouter `openrouter/anthropic/claude-opus-4.8`, OpenAI `gpt-5.2`, Anthropic `claude-opus-4.8`, and Hugging Face `meta-llama/llama-4-scout`. `list_available_models` reports each provider's configured model-map values only when that provider key exists; `get_model_info` reports requested ID, detected provider, key availability, and generation settings. `_create_model_for_provider` ensures one `openrouter/` prefix for OpenRouter `LiteLLMModel`, uses LiteLLM for OpenAI/Anthropic, and `InferenceClientModel` for Hugging Face. Unknown providers raise `ModelInitializationError` inside the factory; missing keys, unknown detection, and constructor exceptions cause `initialize_model` to try `get_any_available_model` in OpenRouter/OpenAI/Anthropic/Hugging Face order. If every attempt fails it returns `None`. + +## Catalogs and drift + +There are three independent hard-coded catalogs: `ModelConfig` aliases, CLI categorized models, and K-LLM `ModelManager` models/roles. The live OpenRouter helper filters provider models with tools/structured output/reasoning capabilities and sorts newest-first, falling back to CLI static data on fetch failure. K-LLM still uses Claude 4.5 for orchestration while central settings/static CLI default to 4.8. Documentation/examples contain more versions. Update each actual consumer deliberately; do not assume one catalog propagates. + +`ModelManager` additionally requires `OPENROUTER_API_KEY`; it has no free Hugging Face fallback despite an example claim. See [K-LLM Use Cases](../orchestration/k-llm-use-cases.md). + +## Change and validation + +To add a provider end to end: add its credential field/environment binding; alias map and fallback model in `ModelConfig`; explicit-map and pattern/configured-provider detection in `get_model_provider`; key selection, available-model reporting, and model info; concrete smolagents construction in `_create_model_for_provider`; fallback priority in `get_any_available_model`; startup warning policy; and any static/live CLI plus K-LLM `ModelManager` inventory/role mappings that should expose it. Verify prefix/ID conventions and duplicate alias precedence. For an ordinary setting, add a real field factory or settings-source mechanism, sample placeholder, consumer gate, and reload-aware test. + +Focused proof includes `tests/test_model_utils.py::TestInitializeModel::test_resolves_short_alias_before_creating_model`, `TestCreateModelForProvider::test_openrouter_adds_prefix`, `TestCreateModelForProvider::test_openrouter_no_double_prefix`, `TestInitializeModel::test_fallback_on_no_provider`, and `test_fallback_on_no_api_key`; startup warnings are pinned by `tests/test_settings.py::TestSettings::test_validate_startup_no_keys` and `test_validate_startup_with_key`. The settings/model/OpenRouter files also cover defaults, provider/key lookups, live filtering, and summaries. They do not cover inert sample variables, singleton refresh, cache creation, alias collisions, or catalog consistency. \ No newline at end of file diff --git a/openwiki/development/index.md b/openwiki/development/index.md new file mode 100644 index 0000000..a6c0f41 --- /dev/null +++ b/openwiki/development/index.md @@ -0,0 +1,3 @@ +# Files + +- [Testing and Operations](testing-and-operations.md) - Local quality gates, test ownership, examples, CI, and the scheduled OpenWiki documentation automation lifecycle. diff --git a/openwiki/development/testing-and-operations.md b/openwiki/development/testing-and-operations.md new file mode 100644 index 0000000..981f9cc --- /dev/null +++ b/openwiki/development/testing-and-operations.md @@ -0,0 +1,79 @@ +--- +type: development guide +title: Testing and Operations +description: Local quality gates, test ownership, examples, CI, and the scheduled OpenWiki documentation automation lifecycle. +tags: [development, testing, ci, operations] +--- + +# Testing and Operations + +The project uses Python 3.11+, `uv`, pytest, Ruff, mypy, and a repository harness. `pyproject.toml` defines runtime/dev dependencies and pytest scope; `Makefile` mirrors the expected local gates. + +## Local commands + +| Intent | Narrow command | Broader command | +|---|---|---| +| Unit behavior | `uv run pytest tests/.py` | `uv run pytest` | +| Lint | `uv run ruff check agentic_internet tests` | `make lint` | +| Format check | `uv run ruff format --check agentic_internet tests` | `make format` | +| Types | `uv run mypy agentic_internet` | `make typecheck` | +| Package | `uv build` | `make build` | +| Harness principles | `python3 .opencode/tools/golden_principles.py` | `make golden` | +| Full local gate | — | `make check` | + +Pytest discovers only `tests/`. Root `test_agent.py` and `test_simple_agent.py` are credentialed live scripts outside normal collection despite their names. The declared `integration` marker should protect future external-service tests; focused tests currently rely heavily on mocks. + +## Test ownership + +| Area | Focused files | +|---|---| +| Settings/model catalogs | `test_settings.py`, `test_model_utils.py`, `test_openrouter_models.py` | +| Core/basic/specialized agents | `test_basic_agent.py`, `test_specialized_agents.py`; core Internet/Research coverage is sparse | +| Search/K-LLM | `test_search_orchestrator.py`, `test_use_cases.py`, `test_orchestration_runtime.py`, `test_context_engineering.py`, `test_cli_use_cases.py` | +| Code Mode/execution | `test_code_mode.py`, `test_code_execution.py`, `test_cli_mcp.py` | +| Web/Exa/browser | `test_web_search.py`, `test_exa_search.py`, `test_browser_use.py` | +| MCP | `test_mcp_integration.py`, `test_cli_mcp.py`; availability-dependent cases can skip | +| Errors | `test_exceptions.py` | + +Tests establish behavior, not complete security guarantees. External lifecycle, resource isolation, SerpAPI/K-LLM end-to-end behavior, Browser async streams, and live MCP cleanup are notable gaps described on owning pages. + +## Python CI workflow + +`.github/workflows/ci.yml` runs on pushes and pull requests to `master`/`main`. Its Python 3.11 preflight performs `uv sync --all-extras --dev`, Ruff format/lint, mypy, pytest, golden-principles checks, then TruffleHog. Checkout/setup actions use moving major tags and TruffleHog uses `@main`, unlike the commit-SHA pinning in OpenWiki automation; dependency/action supply-chain policy is therefore inconsistent. + +## OpenWiki update workflow + +`.github/workflows/openwiki-update.yml` runs manually and daily at `0 8 * * *`. It grants `contents: write` and `pull-requests: write`, so changes to this workflow require security review. + +```mermaid +sequenceDiagram + participant Trigger as Schedule or operator + participant Job as OpenWiki Update job + participant CLI as openwiki CLI + participant Git as Repository branch + participant PR as Pull request + Trigger->>Job: start update + Job->>Git: checkout full history + Job->>Job: set up Node 22 and install pinned npm tools + Job->>CLI: openwiki code --update --print + CLI-->>Git: mutate generated/documentation surfaces + Job->>PR: commit to openwiki/update and open or refresh PR +``` + +*Full history lets OpenWiki diff against its last documented commit; allowed paths are staged into an automated PR rather than pushed directly to the default branch.* + +Operational details: + +- Checkout and Node setup are pinned to commit SHAs; `create-pull-request` is also SHA-pinned. Global npm packages are version-pinned: `openwiki@0.3.3`, `mermaid@11.16.0`, and `jsdom@29.1.1`, but installation still executes registry-delivered package code with job permissions. +- `fetch-depth: 0` is required because incremental update compares HEAD to the previously documented commit; shallow history would produce an empty/incorrect change summary. +- Runtime uses Baseten with `BASETEN_API_KEY` and model `moonshotai/Kimi-K3`. `OPENWIKI_LANGSMITH_API_KEY` authenticates connector pulls. Optional `LANGSMITH_API_KEY` plus `LANGCHAIN_PROJECT=openwiki` and tracing enabled sends run traces to LangSmith. Keep secrets scoped to this job and assume prompts/traces may contain repository content; never echo values. +- `openwiki code --update --print` can generate/update wiki content. The PR action stages only `openwiki`, `AGENTS.md`, `CLAUDE.md`, and `.github/workflows/openwiki-update.yml`; this mutation allowlist is broader than this documentation run’s manual write policy. +- The action writes branch `openwiki/update`, commit/title `docs: update OpenWiki`, and creates or refreshes an automated PR. Human review remains the merge boundary; inspect documentation claims and any instruction/workflow modifications closely. + +## Examples and operational boundaries + +`examples/basic_usage.py`, `advanced_usage.py`, `multi_model_example.py`, `orchestrated_search_example.py`, and MCP examples demonstrate consumers but are not hermetic checks. Most need real provider keys/network/local servers. `example_mcp_server.py` requires undeclared `fastmcp`; the multi-model example claims a free-model fallback that `ModelManager` does not implement. README snapshots also omit MCP and contain stale test count/Black guidance; source, tests, Makefile, and workflows are authoritative. + +## Validation strategy + +Start with the owning focused suite listed above. Add adjacent caller tests when changing an export, registration, recipe bundle, or CLI route. Use the full local gate before merging broad changes. External checks must be explicit `integration`, use placeholder/sample credentials only, avoid logging configuration, and clean up remote tasks/processes. \ No newline at end of file diff --git a/openwiki/index.md b/openwiki/index.md new file mode 100644 index 0000000..c5f53d0 --- /dev/null +++ b/openwiki/index.md @@ -0,0 +1,17 @@ +--- +okf_version: "0.1" +--- + +# Files + +- [Agentic Internet Code Wiki](quickstart.md) - Entry point to the Agentic Internet architecture, public APIs, orchestration systems, tools, configuration, and focused change routes. + +# Directories + +- [agents](agents/) +- [architecture](architecture/) +- [configuration](configuration/) +- [development](development/) +- [interfaces](interfaces/) +- [orchestration](orchestration/) +- [tools](tools/) diff --git a/openwiki/interfaces/cli.md b/openwiki/interfaces/cli.md new file mode 100644 index 0000000..b3d49fe --- /dev/null +++ b/openwiki/interfaces/cli.md @@ -0,0 +1,77 @@ +--- +type: interface guide +title: Command-Line Interface +description: Complete Typer command routing, output semantics, operational caveats, and focused validation for the installed CLI. +tags: [cli, interfaces, operations] +--- + +# Command-Line Interface + +`agentic_internet/cli.py:app` is installed as `agentic-internet`; `python -m agentic_internet` and `python -m agentic_internet.cli` reach the same app. Root `main.py` does not. Most commands catch exceptions, print an error, and raise Typer exit 1, but underlying agents often return error strings without raising. + +## Agent and research commands + +| Command | Runtime route | Important behavior | +|---|---|---| +| `chat` | construct `InternetAgent` → `chat()` | Model, verbose, and max iterations are forwarded. Interactive built-ins are `help`, `tools`, and exit words. | +| `run TASK` | `InternetAgent(model_id=--model, verbose=--verbose/--quiet, max_iterations=--max-iterations)` → `InternetAgent.run(task)` | This route does not expose `agent_type`, so construction uses the `tool_calling` default. Optional `--output` calls `Path.write_text(str(result))` and prints a saved message. Without output, display comes from the agent; quiet mode can suppress useful result display. A raised constructor, filesystem, or uncaught CLI error is printed and exits 1. An execution exception caught by `InternetAgent.run` becomes an error-like string, can be written normally, and usually leaves exit 0. | +| `research TOPIC` | validate depth → `ResearchAgent.research` | Depth must be quick/moderate/deep. With output, exact `json` uses JSON; every other format writes hand-built Markdown. Format has no effect without output. | +| `config` | inspect global `settings` | `--show` serializes settings, including secret-bearing fields; do not log it. `--set` only reports unsupported runtime mutation. | +| `tools` | construct `InternetAgent` and inspect `agent.tools` | Default listing unnecessarily requires model initialization. `--multi` is a hard-coded SerpAPI list; `--use-cases` derives recipe data. | + +Core construction and failure semantics are in [Internet and Research Agents](../agents/internet-and-research.md). + +## Multi-model commands + +`multi TASK` resolves `--use-case`, parses repeated `--worker-model worker=model`, constructs `MultiModelSerpAPISystem`, sets up recipe workers, then executes with `asyncio.run`. Only the first `--models` value becomes both default worker model and coordinator. `--news`, `--workers`, verbosity/quiet, and later model values are accepted but unused. Output JSON wraps workflow `results` that are themselves JSON text. + +`orchestrate TASK` uses the default `research` recipe, coordinator option, and fixed 600-second timeout. Its worker list and verbosity options are display/accepted inputs only and do not configure workers. Default coordinator remains Claude 4.5. + +`news QUERY` turns timeframe, source, and limit into prompt text, then invokes the same general workflow with 300 seconds. These constraints are not enforced provider parameters. JSON formatting is honored only when writing a JSON output file; otherwise result printing is generic. + +See [K-LLM Use Cases](../orchestration/k-llm-use-cases.md) before changing these routes. + +## Model and utility commands + +`models` can show a static categorized catalog or, with `--live`, fetch/filter the public OpenRouter inventory and fall back to static data. Static categories include `general`, `code`, `research`, `news`, and `science`; help omits `science`, and unknown categories produce an empty table. `version` prints root `__version__`. + +## MCP command group + +```mermaid +flowchart TD + MCP["mcp run task"] --> Input{"path or URL"} + Input --> Trust{"--trust present"} + Trust -->|no| Fail["Exit 1"] + Trust -->|yes| Context["Open MCP discovery context"] + Context --> Kind{"agent-type code"} + Kind -->|yes| Code["create_code_mode_agent"] + Kind -->|no| Agent["InternetAgent with MCP tools"] + Code --> Run["Run task inside context"] + Agent --> Run + Run --> Close["Close context"] +``` + +*`mcp run` requires explicit trust and keeps remote tools alive through execution.* + +- `mcp list` loads contiguous `MCP_SERVER_N_*` environment definitions and displays configuration. +- `mcp info` reports package/transports/environment conventions; it mentions `fastmcp`, which is not a declared dependency. +- `mcp run` first imports the MCP helpers and exits 1 with installation guidance when `is_mcp_available()` is false. It then requires path or URL and `--trust`. `--agent-type` is a plain string but is explicitly restricted to exactly `tool_calling` or `code`; any other value prints the received value and exits 1. The route forwards `structured_output`, then selects [Code Mode](../orchestration/code-mode.md) or tool-calling `InternetAgent`. When both path and URL are supplied, path wins. +- `mcp test` connects and lists tools but has no trust option; it unconditionally sets trust true. + +Transport and environment risks are canonical in [MCP Integration](../tools/mcp-integration.md). + +## Error and output discipline + +Treat returned strings beginning with error language or workflow JSON `error` fields as failures even when exit status is zero. Output file writes are direct `Path.write_text` calls without atomic replacement. CLI model/tool listing may instantiate network-configured objects. Never paste `config --show` output into issues. + +## Focused validation + +`tests/test_cli_mcp.py` pins Code Mode, trust, structured-output, model, quiet verbosity, and task routing. `tests/test_cli_use_cases.py` pins use-case/default model/worker override/coordinator routing and recipe listing. Other commands and output files lack focused tests. For CLI changes run: + +```bash +uv run pytest tests/test_cli_mcp.py tests/test_cli_use_cases.py +uv run python -m agentic_internet.cli --help +uv run python -m agentic_internet.cli mcp --help +``` + +Add `CliRunner` cases for every changed validation, routing, output, and exit-status branch. \ No newline at end of file diff --git a/openwiki/interfaces/index.md b/openwiki/interfaces/index.md new file mode 100644 index 0000000..9f8a942 --- /dev/null +++ b/openwiki/interfaces/index.md @@ -0,0 +1,4 @@ +# Files + +- [Command-Line Interface](cli.md) - Complete Typer command routing, output semantics, operational caveats, and focused validation for the installed CLI. +- [Python API](python-api.md) - Public package exports, subpackage extension points, result conventions, and complete change surfaces for library consumers. diff --git a/openwiki/interfaces/python-api.md b/openwiki/interfaces/python-api.md new file mode 100644 index 0000000..6e4cbf1 --- /dev/null +++ b/openwiki/interfaces/python-api.md @@ -0,0 +1,69 @@ +--- +type: API guide +title: Python API +description: Public package exports, subpackage extension points, result conventions, and complete change surfaces for library consumers. +tags: [python-api, public-api, extensions] +--- + +# Python API + +The root package is a curated facade; concrete tools and recipe internals live in subpackages. `__version__` is `0.1.0`. + +## Root exports + +`agentic_internet/__init__.py` exports: + +- `InternetAgent`, `ResearchAgent` and all five [Specialized Agents](../agents/specialized-agents.md) +- `SearchOrchestrator`, `create_search_orchestrator` +- `ToolFacade`, `create_code_mode_agent` +- global `settings` +- principal custom exceptions +- conditionally: `ModelManager`, `MultiModelSerpAPISystem`, five SerpAPI tool classes, and `MULTI_MODEL_AVAILABLE` + +The multi-model import catches `ImportError`; unavailable names become `None`. Consumers must check `MULTI_MODEL_AVAILABLE`, not assume declared dependencies imported successfully. + +```python +from agentic_internet import InternetAgent, ResearchAgent + +agent = InternetAgent() +result = agent.run("Find and summarize current sources") +research = ResearchAgent().research("agent security", depth="deep") +``` + +Runtime results are often strings, including failures. `InternetAgent.run`, `BasicAgent.run`, provider tools, worker wrappers, and multi-model execution convert exceptions into error text/JSON text. Construction/configuration errors such as `ModelInitializationError` can still raise. Do not use type/exit status alone as success evidence. + +## Subpackage surfaces + +`agentic_internet.agents` additionally exports `BasicAgent`, recipe dataclasses/lookups, and core factories. Runtime bundle helpers and many K-LLM context/result classes require direct module imports. `agentic_internet.tools` exports all concrete web/Exa/browser/code tools and optional MCP APIs; root does not. `agentic_internet.config` exposes settings classes/singleton. `agentic_internet.utils` exposes model initialization helpers but not live OpenRouter helpers. + +For exact behavior follow [Internet and Research Agents](../agents/internet-and-research.md), [Search Orchestrator](../orchestration/search-orchestrator.md), [Code Mode](../orchestration/code-mode.md), [K-LLM Use Cases](../orchestration/k-llm-use-cases.md), and the individual tool pages. + +## Exception taxonomy + +`exceptions.py` defines the project hierarchy for package/config/model/tool/search/browser/code/MCP failures. Many adapters catch these or broader exceptions and return strings, so exceptions mainly describe construction/validation boundaries and direct helper use. Preserve typed exceptions where callers can recover before execution; preserve documented string contracts where smolagents expects tool output. + +## Public extension change surfaces + +### Add a built-in tool + +1. Implement a `smolagents.Tool` with stable `name`, `description`, `inputs`, `output_type`, and `forward` behavior. +2. Export it from `agentic_internet/tools/__init__.py`. +3. If default, add feature/key gating in `InternetAgent._get_default_tools`. +4. If recipe-addressable, construct it in `MultiModelSerpAPISystem.create_use_case_tool_inventory` and map its exact name in `TOOL_BUNDLES`. +5. Document the consumer import path and add schema, success, unavailable/error, registration, and security-boundary tests. + +### Add an agent facade + +Implement/subclass under `agents`, export from `agents/__init__.py`, add root export if broadly public, preserve constructor override behavior, and test malformed/failure outputs as well as happy metadata. + +### Add a K-LLM recipe + +Update the built-in registry and bundle inventory, then test normalization/listing, missing tools, model overrides/type selection, coordinator behavior, and CLI discoverability. See the detailed [recipe procedure](../orchestration/k-llm-use-cases.md#adding-a-use-case). + +### Add a CLI route + +Keep behavior in package modules, register on `app` or `mcp_app`, add CLI runner tests for validation/routing/output/exit status, and update [CLI](cli.md). The installed console script and module entrypoints already share `cli:app`/`main`. + +## Validation + +Package export behavior has focused coverage mainly for MCP and conditional surfaces. Run the implementation’s focused test plus `uv run python -m agentic_internet.cli --help`, `uv run pytest`, and `uv run ruff check agentic_internet tests` for public-surface changes. Build with `uv build` when packaging/export metadata changes. \ No newline at end of file diff --git a/openwiki/orchestration/code-mode.md b/openwiki/orchestration/code-mode.md new file mode 100644 index 0000000..88f72fc --- /dev/null +++ b/openwiki/orchestration/code-mode.md @@ -0,0 +1,61 @@ +--- +type: system guide +title: Code Mode +description: Tool-facade CodeAgent construction, discovery and execution meta-tools, executor selection, and MCP integration boundaries. +tags: [orchestration, code-execution, mcp] +--- + +# Code Mode + +`agentic_internet/agents/code_mode.py` prevents large tool catalogs from entering a `CodeAgent` prompt directly. `create_code_mode_agent` gives the agent two meta-tools—`search` and `execute`—while `ToolFacade` exposes the actual local or MCP tools as Python-callable `api` members. + +## Facade contract + +`ToolFacade(tools)` indexes tools by `tool.name`. Reserved names `search`, `execute`, and `_tools` are skipped; duplicate names are last-write-wins. `search(query)` performs case-insensitive substring matching over names/descriptions and emits signatures from each tool’s `inputs` schema: + +- identifier name: `api.web_search(query: string)` +- nonidentifier name: `api._tools["get-weather"](city: string)` + +Descriptions are truncated to 120 characters; no match is exactly `No matching tools found.` Attribute access returns the callable tool itself. Missing public attributes raise an `AttributeError` that directs the model to `api.search(...)`; private attributes retain normal failure semantics. + +## Meta-tool execution + +```mermaid +sequenceDiagram + participant Agent as CodeAgent + participant Search as SearchTool + participant Exec as ExecuteTool + participant Facade as ToolFacade + participant Tool as Wrapped Tool + Agent->>Search: search keyword + Search->>Facade: discover signatures + Facade-->>Agent: callable examples + Agent->>Exec: generated Python + Exec->>Exec: create fresh LocalPythonExecutor + Exec->>Facade: expose api + Exec->>Tool: api.tool arguments + Tool-->>Exec: result + Exec-->>Agent: expression logs or status +``` + +*Discovery describes wrapped tools; execution runs generated Python with the facade in a fresh local executor.* + +`SearchTool.forward` delegates to the facade. Both meta-tools temporarily remove their nonserializable facade during `to_dict()` and restore it in `finally`, supporting E2B serialization. + +`ExecuteTool` deduplicates default imports plus caller additions. Defaults include `json`, regex/date/time/math/statistics/collections, data libraries, HTTP utilities, and `os`. Every call creates a new `LocalPythonExecutor`, sets 15,000 maximum print output, and exposes `api`, `json`, and `print`. Return precedence is non-`None` last expression, stripped logs, then `Execution successful.`; exceptions become `Execution error: ...`. State does not persist between calls. + +## Factory and executor selection + +`create_code_mode_agent(tools, model_id, verbosity_level, max_steps, executor_type, additional_authorized_imports, **kwargs)` creates one facade and the two meta-tools, resolves the model, and raises `ModelInitializationError` if none is available. It defaults `add_base_tools=False` and attaches `agent.facade` dynamically. + +For `executor_type="e2b"`, an available `E2B_API_KEY` enters executor kwargs. Without it, the factory warning-logs and **silently changes to local execution**. Unknown executor strings pass through to smolagents. Local execution authorizes `os` and can call every facade tool, so it can amplify filesystem, network, browser, and remote MCP effects; output bounds are not CPU/memory/wall-clock isolation. The canonical comparison is [System Architecture](../architecture/overview.md). + +## MCP lifecycle + +`mcp run --agent-type code` opens [MCP discovery](../tools/mcp-integration.md), materializes remote tools, creates this agent, and runs the task before leaving the context. Do not return and use the facade after context exit. Remote tool names colliding with reserved names are unavailable through the facade. `structured_output` affects MCP discovery, not `ExecuteTool` result typing. + +## Extension and validation + +A new wrapped tool needs a valid `name`, `description`, `inputs`, and callable behavior; nonidentifier names remain supported. Add imports only when generated code truly needs them and assess side effects. Preserve facade stripping in serialization changes. + +`tests/test_code_mode.py` checks identifier/nonidentifier discovery, case-insensitive/no-match behavior, direct facade invocation, reserved names, serialization restoration, execution/error strings, CodeAgent creation, and no-key E2B fallback. `tests/test_cli_mcp.py` pins context and factory routing. No test covers live E2B, key forwarding, resource isolation, duplicate names, logs-only/status branches, or model failure. Run both focused files for MCP-facing changes. \ No newline at end of file diff --git a/openwiki/orchestration/index.md b/openwiki/orchestration/index.md new file mode 100644 index 0000000..e00df15 --- /dev/null +++ b/openwiki/orchestration/index.md @@ -0,0 +1,5 @@ +# Files + +- [Code Mode](code-mode.md) - Tool-facade CodeAgent construction, discovery and execution meta-tools, executor selection, and MCP integration boundaries. +- [K-LLM Use-Case Orchestration](k-llm-use-cases.md) - Recipe-driven multi-model workers, SerpAPI capabilities, coordinator execution, and context engineering lifecycle. +- [Search Orchestrator](search-orchestrator.md) - Named search workers, parallel execution, aggregation, optional synthesis, history, and integration with direct web search. diff --git a/openwiki/orchestration/k-llm-use-cases.md b/openwiki/orchestration/k-llm-use-cases.md new file mode 100644 index 0000000..2fd3503 --- /dev/null +++ b/openwiki/orchestration/k-llm-use-cases.md @@ -0,0 +1,82 @@ +--- +type: system guide +title: K-LLM Use-Case Orchestration +description: Recipe-driven multi-model workers, SerpAPI capabilities, coordinator execution, and context engineering lifecycle. +tags: [orchestration, multi-model, serpapi, context] +--- + +# K-LLM Use-Case Orchestration + +`MultiModelSerpAPISystem` in `agents/multi_model_serpapi.py` is the largest runtime. It combines declarative recipes (`use_cases.py`), tool-bundle resolution (`orchestration_runtime.py`), model-specific workers, SerpAPI tools, and a coordinator `CodeAgent`. CLI `multi`, `orchestrate`, and `news` route here. + +## Recipes and bundles + +Frozen `WorkerRecipe` requires nonempty name/description/model role and carries ordered bundles, requested agent type, and `required`. Frozen `UseCaseRecipe` requires an ID, workers with unique names, `max_steps >= 1`, and positive timeout; `k` is worker count. `required` is declarative and not enforced. + +Built-ins are: + +| ID | Workers | Purpose | +|---|---|---| +| `research` | search, ecommerce, local-business, academic researchers | Broad multi-source research | +| `technical_due_diligence` | technical researcher, code analyst, risk synthesizer | Technical/code/risk review | +| `market_intelligence` | market, commerce, local-signal analysts | Market and location signals | + +`get_use_case_recipe` strips/lowercases and maps hyphens to underscores. Unknown IDs raise `ValueError` listing available IDs. `list_use_case_recipes()` returns recipes sorted by registry key, while each recipe preserves declared worker order. `TOOL_BUNDLES` maps logical names such as `web`, `scraper`, `multi_engine`, `shopping`, `maps`, `scholar`, `code_execution`, and `browser` to tool names. + +`resolve_tool_bundles` indexes concrete tools last-name-wins, walks requested bundles in order, adds every present expected tool with global name deduplication, and marks an unknown or wholly unavailable bundle missing. `resolve_use_case_tools` resolves recipe `direct_tool_bundles` separately, then each worker's bundles independently; worker misses become `worker_name:bundle`, and the final missing-label list is order-preserving/deduplicated. `summarize_use_case` exposes recipe ID/description, `k`, coordinator, policy, ordered worker definitions and resolved names, direct tool names, missing bundles, and output contract. Missing bundles warn but do not block workers. Current inventory contains direct web/news/scraper, Python/data tools, and optional SerpAPI tools—not Exa or Browser Use—so those declared names cannot currently resolve. + +## Setup and execution + +```mermaid +sequenceDiagram + participant CLI + participant SYS as MultiModelSerpAPISystem + participant REC as Recipe Runtime + participant MM as ModelManager + participant WK as Worker Agents + participant CO as Coordinator CodeAgent + CLI->>SYS: setup use case and overrides + SYS->>REC: resolve inventory and bundles + loop recipe workers + SYS->>MM: model for override default or role + SYS->>WK: create and wrap as AgentTool + end + CLI->>SYS: execute task + SYS->>MM: coordinator model + SYS->>CO: direct tools plus worker tools + CO->>WK: delegate through generated calls + CO-->>SYS: primary result + SYS-->>CLI: JSON text with summary and metrics +``` + +*The coordinator receives direct tools and successfully built worker wrappers; recipe policy guides its prompt rather than a separate scheduler.* + +`setup_use_case_workers` clears previous workers, stores the active recipe/resolution, and selects each model by worker override, then default model, then recipe role. Unknown override names are ignored. Worker creation is soft-failure and permits partial teams. Requested `CodeAgent` is honored only when a model-ID heuristic says code-capable; otherwise a tool-capable model becomes `ToolCallingAgent`, and only models classified as neither become `BasicAgent`. Explicit recipe `BasicAgent` is not honored for tool-capable models. + +`execute_multi_model_workflow` replaces `current_task_context` with a timestamped `TaskContext(objective=task, total_steps=10)`, then chooses the already-active recipe or looks up `use_case_id`. If no workers exist, it implicitly calls setup with `default_model=orchestrator_model`. A still-empty team returns compact JSON text `{"error": "No workers available. Please check API keys and configuration.", "task": task}` before the main try block. A missing coordinator model returns the distinct `{"error": "No orchestrator model available. Please check API keys.", "task": task}`. + +The coordinator prompt includes recipe ID, description, routing policy, and output contract as instructions; the contract is not schema-validated. Coordinator tools are exactly resolved `direct_tools` (empty without active resolution) followed by every successfully wrapped `worker_tools` value. The coordinator is `CodeAgent(max_steps=recipe.max_steps)` with broad data, HTTP, and utility imports. Synchronous `run` executes in a thread under `asyncio.wait_for`; caller timeout overrides `recipe.timeout_seconds`. + +Success returns indented JSON **text** with keys `primary_result`, `use_case`, `search_performance`, and `cross_engine_analysis`, then calls `update_context_from_result(..., success=True)` to append context/episodic memory. Timeout records failure and returns compact `{"error": "Multi-model workflow execution exceeded seconds", "task": task}`; because it interpolates the original argument rather than effective timeout, recipe-default timeout can report `None`. Any other exception records failure and returns `{"error": "Multi-model workflow execution failed: ...", "task": task}`. The two early missing-worker/model returns do not call `update_context_from_result`. Search performance is computed for successful output from whatever tool searches have already logged; the workflow itself does not advance TaskContext steps or independently add performance records. + +`routing_policy`, including `parallel_then_synthesize`, is prompt content only. Workers are not programmatically run in parallel. See [CLI](../interfaces/cli.md) for options that are accepted but do not change this behavior. + +## Model and SerpAPI ownership + +`ModelManager` requires `OPENROUTER_API_KEY`, builds a hard-coded model suite with per-model soft failures, resolves direct aliases then role mappings, and falls back through Claude, DeepSeek, Gemini, and Mistral. Static capability metadata does not prove actual availability. These catalogs differ from central settings; see [Settings and Models](../configuration/settings-and-models.md). + +The system’s five smolagents tools are `GoogleSearchTool`, `GoogleShoppingTool`, `GoogleMapsLocalTool`, `GoogleScholarTool`, and `MultiEngineSearchTool`. They require `SERPAPI_API_KEY`, normalize provider dictionaries to JSON strings, and catch failures into strings. Multi-engine search supports exact lowercase `google,bing,yahoo,baidu`, isolates per-engine errors, and compares exact links. Result Pydantic models (`SearchResult`, `LocalResult`, `ShoppingResult`, `NewsResult`, `ScholarResult`, `ImageResult`) are available in the module but are not instantiated to enforce tool output. + +## Context engineering + +`ContextWindow` estimates tokens as 1.3 times word count, inserts high-priority content first, and compresses only when more than five items exist. Compression preserves first/last two items with a marker and estimates 60% of capacity; small sets can exceed thresholds. `AgentMemory` tracks search attempts/successes/best parameters and bounded episodic records, but its average calculation is not a cumulative mean. `TaskContext` records search history and cross-engine results; workflow progress is initialized to ten steps but never advanced. Performance average logic and `common_results` are similarly incomplete. Treat metrics as diagnostics, not accounting-grade measurements. + +## Adding a use case + +1. Add a validated `UseCaseRecipe` to `BUILT_IN_USE_CASES`; keep worker names unique and output contract explicit. This automatically feeds sorted `list_use_case_recipes()` and therefore `tools --use-cases` recipe/worker metadata. +2. Add any new logical bundle to `TOOL_BUNDLES`. Implement/export the concrete tool, ensure `create_use_case_tool_inventory` constructs an object whose `name` exactly matches the bundle entry, and include the bundle in the recipe's direct or worker tuple. Without all three, resolution only reports a missing bundle and setup can silently produce an under-equipped worker. +3. Confirm model roles/overrides resolve in `ModelManager`; model precedence is per-worker override, CLI/default model, then `worker.model_role`. Decide whether requested agent type needs stricter runtime enforcement. +4. Update CLI flags/help only if the recipe needs new input beyond registry discovery, and test lookup, bundle resolution/missing behavior, setup arguments, summary, and execution policy. +5. If implementing a new routing policy, add a real execution branch rather than prompt wording alone. + +Specific ownership: `tests/test_orchestration_runtime.py::test_parse_worker_model_overrides_rejects_bad_format`, `test_resolve_use_case_tools_reports_missing_worker_bundle`, and `test_summarize_use_case_includes_k_and_workers`; `tests/test_use_cases.py::test_recipe_rejects_duplicate_workers` and `test_list_use_case_recipes_is_stable`; and `tests/test_cli_use_cases.py::test_multi_command_routes_use_case_and_worker_overrides` plus the recipe-listing CLI test. `test_context_engineering.py` covers memory primitives. There are no mocked end-to-end tests for SerpAPI request normalization, `ModelManager`, worker type selection, partial teams, coordinator tools, timeout, or result serialization. \ No newline at end of file diff --git a/openwiki/orchestration/search-orchestrator.md b/openwiki/orchestration/search-orchestrator.md new file mode 100644 index 0000000..bc2eb65 --- /dev/null +++ b/openwiki/orchestration/search-orchestrator.md @@ -0,0 +1,58 @@ +--- +type: system guide +title: Search Orchestrator +description: Named search workers, parallel execution, aggregation, optional synthesis, history, and integration with direct web search. +tags: [orchestration, search, concurrency] +--- + +# Search Orchestrator + +`agentic_internet/agents/search_orchestrator.py` is a model-agnostic worker coordinator distinct from K-LLM recipes. Public exports are `SearchOrchestrator` and `create_search_orchestrator`; module-level contracts also include `SearchTask`, `SearchResult`, and `SearchAgentWrapper`. + +## Contracts and setup + +`SearchTask` carries query, task ID, agent name, `search_type="general"`, and metadata. `SearchResult` records identity, arbitrary result, success, elapsed seconds, and optional error. A `SearchAgentWrapper` owns one object exposing `run(prompt)`, its specialization, and execution/success counters. + +`setup_default_agents(tools, model=None)` resolves a model when absent, then independently attempts three `ToolCallingAgent(max_steps=10)` workers: `news_researcher`, `tech_researcher`, and `general_researcher`. It separately attempts a `CodeAgent(max_steps=20)` synthesizer with JSON/regex/date imports. Individual construction failures are soft, so setup can leave a partial registry. `create_search_orchestrator` constructs and invokes this setup. + +## Execution + +```mermaid +sequenceDiagram + participant Caller + participant SO as SearchOrchestrator + participant SW as SearchAgentWrapper + participant SYN as Synthesis CodeAgent + Caller->>SO: search query selection parallel + SO->>SO: validate workers and create tasks + par selected workers + SO->>SW: execute specialized prompt + SW-->>SO: SearchResult + end + SO->>SO: aggregate counts timings and excerpts + opt successful results and synthesizer + SO->>SYN: synthesize result excerpts + SYN-->>SO: summary + end + SO-->>Caller: aggregate dictionary +``` + +*Selected workers run sequentially or in threads; synthesis is optional and cannot fail the aggregate.* + +`SearchAgentWrapper.execute` increments `execution_count` before prompt/model work and increments `success_count` only after `agent.run` returns; exceptions become failed `SearchResult(result=None, error=str(e))`. Specializations ask respectively for recent/breaking news; scholarly papers with citations/credible sources; technical docs/tutorials/implementation details; market trends/business data/statistics; or comprehensive multi-perspective detail. Any other specialization passes the raw query unchanged. + +`search(query, agents_to_use=None, parallel=True)` rejects no workers with `{"error": "No agents available. Please add agents first.", "query": ...}` and an empty filtered selection with `{"error": "No valid agents selected", ...}`. It creates timestamp-based task IDs in selected-agent order. Sequential execution preserves that order. For multiple parallel tasks, `ThreadPoolExecutor(max_workers=self.max_workers)` limits concurrent calls while `as_completed` makes output completion-ordered; one task always takes the sequential path even when `parallel=True`. + +Aggregation returns query/timestamp, total/success/failure counts, an execution-time map, successful `agent_results` keyed by name with specialization/result, and `failures` keyed by name with error. A failed worker does not fail successful siblings. Successful text is limited to 1,000 characters; synthesis sees 500 per successful result and runs only when at least one succeeded and `orchestrator_agent` exists. Coordinator exceptions make the included `synthesis` value exactly `None`; worker results remain. Executed aggregates enter `execution_history` with timestamp, query, selected names, and aggregate; preflight error returns do not. `get_performance_report` returns history length as `total_executions` plus per-agent name, specialization, attempts, successes, and computed success rate (zero before attempts). + +`search_async` is `asyncio.to_thread` around synchronous `search`; `use_async` does not create native async worker calls. There is no timeout or cancellation policy. `SearchTask.search_type` and metadata are currently not consumed by wrapper execution. + +## Web-search integration + +[WebSearchTool](../tools/web-search-and-scraping.md) can be built with `use_orchestrator=True` and an orchestrator. It returns synthesis first, then formatted worker results; an empty/error outcome falls back to direct SerpAPI/DDGS. `examples/orchestrated_search_example.py` shows default, custom, integrated, and async usage. + +## Extension and tests + +Register custom workers with `add_agent(name, agent, specialization)`; duplicate names replace wrappers and lose counters. Override `_create_specialized_prompt`, aggregation, or synthesis for domain semantics. Any worker with `run(prompt)` works despite smolagents-focused annotations. + +Focused proof in `tests/test_search_orchestrator.py` includes `TestSearchOrchestrator::test_search_without_agents_returns_error`, `test_search_parallel_with_mixed_outcomes`, `test_synthesis_runs_when_orchestrator_present`, `test_synthesis_handles_failure_gracefully`, and `test_get_performance_report`; wrapper tests pin prompt variants and success/failure counters. Default setup, factory/model resolution, truncation/completion ordering, `max_workers`, and async behavior are untested. Run `uv run pytest tests/test_search_orchestrator.py`; include `tests/test_web_search.py` when changing integration behavior. \ No newline at end of file diff --git a/openwiki/quickstart.md b/openwiki/quickstart.md new file mode 100644 index 0000000..d9d23a0 --- /dev/null +++ b/openwiki/quickstart.md @@ -0,0 +1,103 @@ +--- +type: quickstart guide +title: Agentic Internet Code Wiki +description: Entry point to the Agentic Internet architecture, public APIs, orchestration systems, tools, configuration, and focused change routes. +tags: [quickstart, architecture, navigation] +--- + +# Agentic Internet Code Wiki + +Agentic Internet is a Python 3.11+ package and Typer CLI that composes smolagents with web search/scraping, Browser Use Cloud, local code/data tools, MCP servers, and several orchestration strategies. It owns no database or server; runtime state is in memory and most effects cross into external providers or local executors. + +Start with [System Architecture](architecture/overview.md) for dependency direction, runtime paths, state ownership, and the canonical code-execution trust comparison. The real runtime entrypoints are the installed `agentic-internet` script and `python -m agentic_internet`; root `main.py` is stale scaffolding. + +## Main concepts + +### Interfaces and agents + +- [Command-Line Interface](interfaces/cli.md) documents every root and MCP command, exact routing, inert/prompt-only options, output files, and error/exit semantics. +- [Python API](interfaces/python-api.md) maps root/subpackage exports, result conventions, exceptions, and complete extension surfaces. +- [Internet and Research Agents](agents/internet-and-research.md) owns model initialization, default-tool gates, underlying agent selection, run/chat, and research history. +- [Specialized Agents](agents/specialized-agents.md) covers all browser, data, content, market, and technical-support convenience methods. + +### Orchestration + +- [K-LLM Use-Case Orchestration](orchestration/k-llm-use-cases.md) covers recipes, tool bundles, worker/model selection, SerpAPI tools, coordinator execution, JSON outcomes, and context memory. +- [Search Orchestrator](orchestration/search-orchestrator.md) covers named workers, thread-pool execution, partial failures, synthesis, history, and web-tool integration. +- [Code Mode](orchestration/code-mode.md) covers `ToolFacade`, discovery/execution meta-tools, local/E2B selection, and MCP use. + +### Capability tools + +- [Web Search, News, and Scraping](tools/web-search-and-scraping.md): SerpAPI/DDGS fallback, optional orchestration, HTTP extraction, and SSRF boundary. +- [Exa Search](tools/exa-search.md): semantic search/find-similar requests, content controls, and conditional registration. +- [Browser Automation](tools/browser-automation.md): synchronous, asynchronous, streaming, and nominal structured Browser Use Cloud behavior. +- [Code Execution and Data Analysis](tools/code-execution-and-data.md): AST policy, restricted namespace, pandas operations, and non-sandbox limitations. +- [MCP Integration](tools/mcp-integration.md): stdio/HTTP discovery, context lifetime, environment configuration, trust, and multi-server limits. + +### Configuration and engineering + +- [Settings and Models](configuration/settings-and-models.md) explains actual environment bindings, alias/provider/key/fallback order, duplicated model catalogs, and provider extension steps. +- [Testing and Operations](development/testing-and-operations.md) maps focused tests, local quality gates, examples, Python CI, and the scheduled privileged OpenWiki PR workflow. + +## Runtime at a glance + +```mermaid +flowchart LR + User["CLI or Python caller"] --> Route{"Execution route"} + Route -->|single task| IA["InternetAgent"] + Route -->|parallel search| SO["SearchOrchestrator"] + Route -->|recipe| KL["MultiModelSerpAPISystem"] + Route -->|MCP code mode| CM["ToolFacade CodeAgent"] + IA --> Tools["Built-in or injected tools"] + SO --> Tools + KL --> Tools + CM --> Tools + Tools --> Effects["Providers, cloud browser, MCP, local execution"] +``` + +*All routes ultimately combine a resolved model with tools; their coordination and trust guarantees differ.* + +## Task routing + +| Engineering intent | Canonical page | Owning source entrypoints or symbols | Focused tests | Minimal validation | +|---|---|---|---|---| +| Change ordinary CLI routing/output | [CLI](interfaces/cli.md) | `agentic_internet/cli.py:app`, command function | `test_cli_use_cases.py`, `test_cli_mcp.py` when relevant | `uv run python -m agentic_internet.cli --help` plus changed command test | +| Change base task/chat/research behavior | [Internet and Research Agents](agents/internet-and-research.md) | `InternetAgent`, `ResearchAgent` | Adjacent tool/model tests; direct core coverage is sparse | Focused suite plus `uv run pytest` | +| Add/change a public facade/export | [Python API](interfaces/python-api.md) | root/subpackage `__init__.py`, implementation class/factory | Implementation and import-surface test | `uv run pytest ` and `uv build` | +| Add a specialized helper | [Specialized Agents](agents/specialized-agents.md) | `specialized_agents.py` class/method and exports | `test_specialized_agents.py` | `uv run pytest tests/test_specialized_agents.py` | +| Add a K-LLM recipe or bundle | [K-LLM](orchestration/k-llm-use-cases.md) | `BUILT_IN_USE_CASES`, `TOOL_BUNDLES`, `create_use_case_tool_inventory`, `setup_use_case_workers` | `test_use_cases.py`, `test_orchestration_runtime.py`, `test_cli_use_cases.py` | Run those three files | +| Change coordinator/context behavior | [K-LLM](orchestration/k-llm-use-cases.md) | `execute_multi_model_workflow`, context classes | `test_context_engineering.py`; end-to-end gap remains | Focused context tests plus mocked workflow test you add | +| Change parallel search/synthesis | [Search Orchestrator](orchestration/search-orchestrator.md) | `SearchAgentWrapper`, `SearchOrchestrator.search`, `_aggregate_results` | `test_search_orchestrator.py` | Run that file; add `test_web_search.py` for integration | +| Change Code Mode/facade | [Code Mode](orchestration/code-mode.md) | `ToolFacade`, `ExecuteTool`, `create_code_mode_agent` | `test_code_mode.py`, `test_cli_mcp.py` | Run both files | +| Change web/news/scraping | [Web Tools](tools/web-search-and-scraping.md) | `_search_with_fallback`, tool `forward` methods, `_validate_url` | `test_web_search.py` | Run that file | +| Change Exa support | [Exa Search](tools/exa-search.md) | `ExaResult`, `ExaSearchTool`, `ExaFindSimilarTool`, default registration | `test_exa_search.py` | Run that file | +| Change Browser Use behavior | [Browser Automation](tools/browser-automation.md) | three Browser Use tool classes | `test_browser_use.py` | Run focused file; mark live test `integration` | +| Change local execution/data policy | [Code Execution](tools/code-execution-and-data.md) | `_ASTSafetyValidator`, `PythonExecutorTool`, `DataAnalysisTool` | `test_code_execution.py` | Run that file and add abuse-case test | +| Change MCP transport/configuration | [MCP Integration](tools/mcp-integration.md) | `MCPToolIntegration`, `MCPServerConfig`, manager, `mcp_tools`, environment loader | `test_mcp_integration.py`, `test_cli_mcp.py` | Run both; use marked local-server test for lifecycle | +| Add/change model provider or alias | [Settings and Models](configuration/settings-and-models.md) | `ModelConfig`, `Settings` resolution methods, `_create_model_for_provider`, catalogs | `test_settings.py`, `test_model_utils.py`, `test_openrouter_models.py` | Run all three | +| Change tests, CI, or doc automation | [Testing and Operations](development/testing-and-operations.md) | `pyproject.toml`, `Makefile`, both workflow YAML files | Repository suite/harness | `make check`; inspect workflow permissions and secrets | + +## Safety and result conventions + +- `PythonExecutorTool`, Code Mode, ordinary CodeAgents, and K-LLM CodeAgents have different policies; none is a complete arbitrary-code sandbox. Read the [canonical comparison](architecture/overview.md#canonical-code-execution-trust-comparison). +- Web scraping does not block internal/private/DNS-rebound targets. MCP stdio inherits the full parent environment and MCP trust is enforced differently by CLI routes. +- Many execution/provider failures are returned as strings or JSON text rather than raised. Validate semantic outcome, not only Python type or process status. +- The global settings object snapshots environment at import. `.env.example` includes several model/log variables that current Pydantic code does not bind. + +## Repository-wide validation + +For broad changes: + +```bash +uv run ruff format --check agentic_internet tests +uv run ruff check agentic_internet tests +uv run mypy agentic_internet +uv run pytest +python3 .opencode/tools/golden_principles.py +``` + +External provider/browser/MCP checks should be explicitly marked integration, use placeholders or scoped test credentials, avoid configuration dumps, and clean up remote/process resources. + +## Backlog + +No substantial manifest-backed component was deferred. Live provider behavior and resource-isolation guarantees are documented as test gaps because they require external credentials/services or a dedicated sandbox fixture; their source anchors and narrow next checks are recorded on the owning tool/orchestration pages. \ No newline at end of file diff --git a/openwiki/tools/browser-automation.md b/openwiki/tools/browser-automation.md new file mode 100644 index 0000000..722fff4 --- /dev/null +++ b/openwiki/tools/browser-automation.md @@ -0,0 +1,43 @@ +--- +type: tool guide +title: Browser Automation +description: Browser Use Cloud synchronous, asynchronous, streaming, and nominal structured-extraction tool behavior. +tags: [tools, browser, cloud] +--- + +# Browser Automation + +`agentic_internet/tools/browser_use.py` exports three Browser Use Cloud adapters. [InternetAgent](../agents/internet-and-research.md) registers them only when browser tools are enabled and `BROWSER_USE_API_KEY` is present. + +| Tool | Client and behavior | +|---|---| +| `BrowserUseTool` / `browser_use` | Persistent `BrowserUse` client; `tasks.run(task=...)`; returns `done_output` or status text. Its `structured_output` input is ignored. | +| `AsyncBrowserUseTool` / `async_browser_use` | Persistent `AsyncBrowserUse`; `forward` invokes `asyncio.run`. Simple mode awaits `tasks.run`; stream mode creates a task, iterates updates, accumulates last-step text, and returns final output at status `finished` or an incomplete-stream message. | +| `StructuredBrowserUseTool` / `structured_browser_use` | Uses `AsyncBrowserUse` via `asyncio.run`, but does not parse or enforce the supplied JSON-schema string; it returns normal `done_output` or `No structured data extracted.` | + +```mermaid +stateDiagram-v2 + [*] --> Unavailable: SDK or key missing + [*] --> Created: stream task created + Created --> Streaming: updates received + Streaming --> Streaming: nonfinal update + Streaming --> Finished: status finished + Streaming --> Incomplete: stream ends + Finished --> [*] + Incomplete --> [*] + Unavailable --> [*] +``` + +*The async streaming tool recognizes only SDK updates and final status; it has no repository-level timeout or cancellation state.* + +The module’s `WebArticle`, `ProductInfo`, `ContactInfo`, and `SearchResults` Pydantic models are examples only and are not bound to tool output. Missing SDK/key and all SDK exceptions become strings. + +## Lifecycle and trust + +Clients have no explicit close/context-manager integration. `asyncio.run` fails inside an already-running event-loop thread; that failure is converted to a tool error. There is no timeout, retry, task cancellation/cleanup, polling bound, schema/task size check, URL policy, or buffered-step bound. Task content and accessed-site data cross the Browser Use Cloud trust boundary; SDK error details return to the model. + +## Extension and validation + +Override `_run_simple`, `_run_with_stream`, or `_extract_structured_data` for alternate lifecycle behavior. Real structured support must parse/validate the schema, pass the provider’s supported structured contract, and validate returned data rather than merely renaming the tool. + +`tests/test_browser_use.py` covers sync no-key, successful output, status fallback, SDK error, and no-key behavior for async/structured tools. Async success/stream transitions, schema handling, ignored inputs, event-loop conflict, timeout, and client cleanup are untested. Run that file; use a separately marked integration test for live cloud behavior. \ No newline at end of file diff --git a/openwiki/tools/code-execution-and-data.md b/openwiki/tools/code-execution-and-data.md new file mode 100644 index 0000000..bff3ca6 --- /dev/null +++ b/openwiki/tools/code-execution-and-data.md @@ -0,0 +1,45 @@ +--- +type: tool guide +title: Code Execution and Data Analysis +description: AST-validated local Python execution, pandas operations, output selection, and non-sandbox security boundaries. +tags: [tools, code-execution, data-analysis, security] +--- + +# Code Execution and Data Analysis + +`agentic_internet/tools/code_execution.py` exports two smolagents tools: `PythonExecutorTool` and `DataAnalysisTool`. They are default capabilities when `settings.tools.code_execution_enabled` and also enter the K-LLM tool inventory. They are separate from CodeAgent/Code Mode execution; compare all paths in [System Architecture](../architecture/overview.md). + +## PythonExecutorTool + +```mermaid +flowchart TD + Input["Python source"] --> Parse["ast.parse"] + Parse --> Validate["_ASTSafetyValidator"] + Validate --> Safe{"No violations"} + Safe -->|no| Error["Error string"] + Safe -->|yes| Namespace["Copy restricted base namespace"] + Namespace --> Exec["compile and exec with redirected output"] + Exec --> Result{"result variable is not None"} + Result -->|yes| Value["Return result"] + Result -->|no| Stdout{"stdout exists"} + Stdout -->|yes| Logs["Return stdout"] + Stdout -->|no| Stderr{"stderr exists"} + Stderr -->|yes| Warn["Return stderr"] + Stderr -->|no| Done["Successful no-output message"] +``` + +*AST policy runs before a fresh execution namespace; output precedence is result, stdout, stderr, then status.* + +The validator blocks selected module roots (`os`, subprocess/filesystem/socket/reflection/serialization families), dangerous direct calls (`eval`, `exec`, `compile`, `open`, `__import__`, reflection), and major dunder traversal attributes. The namespace exposes restricted builtins plus preloaded NumPy, pandas, requests, JSON, regex/date/time/math/statistics/collections utilities. Syntax/unsafe/runtime errors become strings; runtime errors include traceback text. Final output is capped at 10,000 characters. + +This is defense in depth, **not a sandbox**. `PythonExecutorTool.EXECUTION_TIMEOUT_HINT = 30` is advisory documentation only and is never enforced. CPU, memory, process/thread, wall time, and network are unrestricted; preloaded `requests` enables outbound/internal requests. Rich library objects expand attack surface. Imports allowed by AST can still fail because `__import__` is absent. Fresh namespace copies isolate user bindings, not shared module objects, and output truncation occurs after execution. + +## DataAnalysisTool + +Inputs are `data` text and an operation. Supported operations are `describe`, `info`, `correlations`, `summary`, and `missing`; unknown operations return a message before parsing. The tool first tries JSON into `pandas.DataFrame`, then CSV on JSON/value failure. Results are text. Correlations reject data without numeric columns. There are no input-byte, row, nesting, memory, or execution-time limits. + +## Policy changes and tests + +Policy seams are allow/block constants, `_ASTSafetyValidator`, `_validate_code`, namespace construction, and output limits. Any relaxation needs an abuse-case test; do not infer safety from an AST allowlist alone. For untrusted code, use an externally enforced container/remote sandbox with network and resource policy. + +`tests/test_code_execution.py` checks safe code; blocked OS/subprocess/ctypes, dunder, eval/exec/open paths; stdout/result/preloaded modules; syntax/runtime errors; final truncation; JSON/CSV and all data operations. It does not prove timeout, resource/network isolation, import-runtime consistency, or shared-object safety. Run `uv run pytest tests/test_code_execution.py`. \ No newline at end of file diff --git a/openwiki/tools/exa-search.md b/openwiki/tools/exa-search.md new file mode 100644 index 0000000..d146f71 --- /dev/null +++ b/openwiki/tools/exa-search.md @@ -0,0 +1,30 @@ +--- +type: tool guide +title: Exa Search +description: Semantic search and find-similar tool schemas, content controls, client lifecycle, and conditional registration. +tags: [tools, search, exa] +--- + +# Exa Search + +`agentic_internet/tools/exa_search.py` adapts `exa_py` into `ExaSearchTool` (`exa_search`) and `ExaFindSimilarTool` (`exa_find_similar`). Both return formatted strings rather than structured objects and are exported from `agentic_internet.tools`. + +## Normalization and request flow + +`ExaResult` is the internal normalized dataclass: title, URL, optional text, summary, author, publish date, score, and highlights. `from_sdk` tolerates missing fields. `snippet(max_chars)` prefers summary, joined highlights, text, then `No description`, with ellipsis truncation. + +`ExaSearchTool.forward` checks SDK/key availability, creates a new `Exa(api_key)` client, adds `x-exa-integration: agentic-internet`, builds content controls, normalizes optional search type/category allowlists, adds domain/date filters, calls `search_and_contents`, normalizes `response.results`, and formats ranked text. Unknown search types/categories are omitted so SDK defaults apply. `ExaFindSimilarTool` follows the same content path around `find_similar_and_contents(url=...)` and can exclude the source domain. + +Constructor knobs control result count, text, highlights, summaries, query-focused summaries, and text character limit. Explicit zero count/limit values fall back to defaults because constructors use `or`. There is no alternate provider fallback, retry, close protocol, cache, or backoff; SDK exceptions become strings containing provider detail. + +## Registration and boundaries + +[InternetAgent](../agents/internet-and-research.md) adds both tools only when web search is enabled and `settings.exa_api_key` is truthy. Registration checks the import-time settings field, but tool constructors reread `os.getenv("EXA_API_KEY")`; mutating only settings can therefore register unavailable tool instances. K-LLM bundle declarations mention Exa names, but its inventory does not instantiate these tools. + +Queries, domain lists, dates, and source URLs lack local structural validation. In particular, find-similar does not reuse the web scraper’s URL syntax validation. Returned exception text may expose SDK/server details. + +## Extension and tests + +Use `_build_contents_kwargs`, normalizers, `ExaResult`, and `_format_results` rather than duplicating provider adaptation. A public change may require tool export, default-agent gate, K-LLM inventory/bundle wiring, sample environment placeholder, and enabled/disabled tests. + +`tests/test_exa_search.py` covers field defaults, snippet preference/truncation, formatting, exact content kwargs, normalizers, key/SDK absence, integration header and filter forwarding, unknown type omission, errors, similar-page arguments, and default registration names. It does not cover constructor environment reads, date/domain/URL validity, zero options, client cleanup, or all find-similar failure branches. \ No newline at end of file diff --git a/openwiki/tools/index.md b/openwiki/tools/index.md new file mode 100644 index 0000000..25bed20 --- /dev/null +++ b/openwiki/tools/index.md @@ -0,0 +1,7 @@ +# Files + +- [Browser Automation](browser-automation.md) - Browser Use Cloud synchronous, asynchronous, streaming, and nominal structured-extraction tool behavior. +- [Code Execution and Data Analysis](code-execution-and-data.md) - AST-validated local Python execution, pandas operations, output selection, and non-sandbox security boundaries. +- [Exa Search](exa-search.md) - Semantic search and find-similar tool schemas, content controls, client lifecycle, and conditional registration. +- [MCP Integration](mcp-integration.md) - Optional MCP availability, stdio and streamable HTTP discovery, configuration management, CLI routing, and connection trust boundaries. +- [Web Search, News, and Scraping](web-search-and-scraping.md) - Contracts, provider fallback, optional orchestration, and HTTP trust boundaries for direct web tools. diff --git a/openwiki/tools/mcp-integration.md b/openwiki/tools/mcp-integration.md new file mode 100644 index 0000000..e24877f --- /dev/null +++ b/openwiki/tools/mcp-integration.md @@ -0,0 +1,57 @@ +--- +type: integration guide +title: MCP Integration +description: Optional MCP availability, stdio and streamable HTTP discovery, configuration management, CLI routing, and connection trust boundaries. +tags: [tools, mcp, integration, security] +--- + +# MCP Integration + +`agentic_internet/tools/mcp_integration.py` bridges smolagents `ToolCollection.from_mcp` to local stdio and remote streamable HTTP servers. MCP tools are not defaults; CLI routes or consumers open a context and inject discovered tools into [InternetAgent](../agents/internet-and-research.md) or [Code Mode](../orchestration/code-mode.md). + +## Availability and exports + +MCP is available only when both `mcp.StdioServerParameters` and `smolagents.ToolCollection` import. `check_mcp_available` raises `MCPNotAvailableError`; `is_mcp_available` returns a boolean. `agentic_internet.tools` conditionally exposes `MCP_AVAILABLE`, integration/config/manager types, `mcp_tools`, environment loading, and the boolean helper; failed import leaves safe false/`None` placeholders. Root `agentic_internet` does not export MCP APIs. + +## Connection lifecycle + +```mermaid +sequenceDiagram + participant Caller + participant Helper as mcp_tools + participant Integration as MCPToolIntegration + participant Collection as ToolCollection + participant Server as MCP Server + participant Agent + Caller->>Helper: enter path or URL context + Helper->>Integration: connect transport config + Integration->>Collection: from_mcp trust and structured flags + Collection->>Server: open and discover tools + Collection-->>Caller: yield remote tool list + Caller->>Agent: construct and run inside context + Agent->>Server: invoke remote tool + Caller->>Helper: exit context + Helper->>Collection: close session +``` + +*Discovery and every remote invocation must finish before context exit.* + +For stdio, configuration may be a path or `{path, command}`. The path is resolved absolute, command defaults to `python`, and supplied overrides merge into a copy of **all** `os.environ`; a child can therefore receive unrelated secrets. Existence/type/permissions/allowlisted command are not validated. + +HTTP accepts `streamable-http` or `http`, normalizes a root URL to `/mcp/`, and passes `{"url": ..., "transport": "streamable-http"}`. Scheme, embedded credentials, private hosts, headers/auth, and TLS policy are not validated. If helper receives both path and URL, path wins. + +`connect(..., trust_remote_code, structured_output)` conditionally forwards structured output and yields `tool_collection.tools`. The integration API does not enforce trust: `mcp run` requires `--trust`, while `mcp test` unconditionally enables trust. Treat server code/tool descriptions/results as untrusted. Code Mode lets generated Python invoke every nonreserved remote tool, amplifying effects. + +## Configuration and manager + +`MCPServerConfig` round-trips name, server config, transport, environment, trust, and structured flags. `MCPServerManager` stores configs, overwrites duplicate names, and opens connections on demand. `connect_all` fully supports zero, one, or exactly two servers; for more than two it warns and yields only the first server’s tools. + +`load_mcp_config_from_env` scans contiguous `MCP_SERVER_1_*`, `MCP_SERVER_2_*`, etc. It stops at the first missing `TYPE`, so numbering gaps hide later entries. Stdio requires `PATH` and captures `ENV_*`; HTTP requires `URL`; trust/structured booleans accept true/1/yes. Invalid entries warn and skip. + +The instance fields `_context_manager` and `_tools` are not populated by current classmethod connection flow, so `get_tools_list()` ordinarily remains empty. + +## CLI and extension checks + +`mcp list` shows environment configs; `info` explains support; `run` selects stdio/HTTP and tool-calling/code agent; `test` discovers and displays tools. See [CLI](../interfaces/cli.md) for option/error behavior. Example files demonstrate use but require external dependencies/server processes; `example_mcp_server.py` imports undeclared `fastmcp`. + +`tests/test_mcp_integration.py` covers availability, exports, config round-trip, manager storage, parameter shapes, and basic environment loading; many cases skip without MCP. `tests/test_cli_mcp.py` pins Code Mode/structured-output routing. There is no live context/cleanup, URL normalization/security, arbitrary command/environment leak, numbering-gap, connection failure, or >2-server test. Use a marked local-server integration fixture before changing lifecycle semantics. \ No newline at end of file diff --git a/openwiki/tools/web-search-and-scraping.md b/openwiki/tools/web-search-and-scraping.md new file mode 100644 index 0000000..1891203 --- /dev/null +++ b/openwiki/tools/web-search-and-scraping.md @@ -0,0 +1,49 @@ +--- +type: tool guide +title: Web Search, News, and Scraping +description: Contracts, provider fallback, optional orchestration, and HTTP trust boundaries for direct web tools. +tags: [tools, search, scraping] +--- + +# Web Search, News, and Scraping + +`agentic_internet/tools/web_search.py` owns three `smolagents.Tool` classes exported from `agentic_internet.tools`: `WebSearchTool`, `WebScraperTool`, and `NewsSearchTool`. [InternetAgent](../agents/internet-and-research.md) registers them when web search is enabled; K-LLM creates them independently in its use-case inventory. + +## Search fallback + +```mermaid +flowchart TD + Start["WebSearchTool forward"] --> Orch{"Orchestrator enabled"} + Orch -->|yes| Run["SearchOrchestrator search"] + Run --> Useful{"Synthesis or agent results"} + Useful -->|yes| Return["Return orchestrated text"] + Useful -->|no| Direct["Direct provider path"] + Orch -->|no| Direct + Direct --> Key{"SERPAPI_API_KEY present"} + Key -->|yes| Serp["SerpAPI GoogleSearch"] + Serp --> Valid{"Formatted nonempty result"} + Valid -->|yes| Return + Valid -->|no| DDG["DuckDuckGo DDGS"] + Key -->|no| DDG + DDG --> Return +``` + +*Orchestration and SerpAPI failures degrade to DuckDuckGo rather than raising.* + +`WebSearchTool` accepts `query`; direct results are numbered title/snippet/URL text, limited by helper default `num=5` (and SerpAPI slices to that count). Optional `use_orchestrator` plus an injected [SearchOrchestrator](../orchestration/search-orchestrator.md) prefers a `**Orchestrated Search Results (Synthesized):**` response, then per-agent output; an exception or no usable synthesis/results logs and enters direct fallback. `NewsSearchTool` accepts `query` and uses SerpAPI `tbm="nws"` or `DDGS.news(..., max_results=5)`. Search tools read `SERPAPI_API_KEY` directly; `ToolConfig.max_search_results` is not wired into them. + +SerpAPI unavailable, missing key, empty provider list, or exception produces internal `None` and therefore DDGS fallback. DDGS absence returns exactly `DuckDuckGo search is not available. Install with: pip install duckduckgo-search`; an empty list returns `No search results found.`; provider failure returns `Error performing DuckDuckGo search: `. There is no retry, caching, backoff, or rate-limit policy. + +## Scraper lifecycle + +`WebScraperTool.forward(url)` validates only an `http` or `https` scheme with a nonempty network location. It creates an `httpx.Client(follow_redirects=True, timeout=30)`, sends a browser-like user agent, raises on status, parses with BeautifulSoup, removes script/style nodes, collapses whitespace, and truncates extracted text to 3,000 characters. HTTP and general failures become explanatory strings. + +Success returns `Content from :\n\n`. URL policy returns `Invalid URL scheme ''. Only http and https are supported.`, `Invalid URL: missing hostname.`, or `Malformed URL: `. HTTP status failures return `HTTP error when accessing `, and other failures return `Error scraping : `. + +Security boundary: this is not SSRF-safe. Validation checks only scheme and hostname syntax. It does not reject `localhost`, loopback, private, link-local, or metadata targets; protect against DNS rebinding; revalidate redirect targets; restrict content type; or bound response/download bytes before parsing. Do not expose it to arbitrary untrusted URLs in a privileged network without an outbound proxy and address policy. + +## Extension and validation + +Provider formatting/fallback lives in `_search_serpapi`, `_search_ddgs`, and `_search_with_fallback`; URL syntax policy is `_validate_url`. If adding a search provider, preserve deterministic fallback semantics and distinguish unavailable, empty, and failed outcomes. If hardening scraping, test initial and redirect addresses, DNS resolution, content type/size, and timeout behavior. + +Focused proof in `tests/test_web_search.py` includes `TestSearchWithFallback::test_falls_back_to_ddgs`, `TestValidateUrl::test_invalid_scheme`, `TestWebScraperTool::test_rejects_ftp_url`, `TestWebScraperTool::test_successful_scrape`, and `TestNewsSearchTool::test_forward_uses_news_params`; adjacent cases cover SerpAPI empty/unavailable/error and DDGS formatting/unavailability. It does not cover orchestrator branches, news formatting, redirect/SSRF/DNS policy, script removal, truncation, status errors, or download bounds. Run this suite plus `tests/test_search_orchestrator.py` when altering orchestration. \ No newline at end of file diff --git a/tests/test_code_execution.py b/tests/test_code_execution.py index 0ea7d8e..50501e4 100644 --- a/tests/test_code_execution.py +++ b/tests/test_code_execution.py @@ -107,6 +107,16 @@ def test_blocks_dunder_escape(self): result = self.tool.forward("x = ().__class__.__bases__[0].__subclasses__()") assert "Error" in result + def test_indirect_import_alias_blocked(self): + # Aliasing __import__ bypasses the AST call check; the runtime guard must still block. + result = self.tool.forward('f = __import__\nresult = f("os")') + assert "blocked module" in result + + def test_allowed_runtime_import(self): + # Allowed modules must still import through the guarded __import__. + result = self.tool.forward("import math\nresult = math.sqrt(4)") + assert "2.0" in result + class TestDataAnalysisTool: def setup_method(self):