diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..ca2b787 Binary files /dev/null and b/.DS_Store differ diff --git a/.artifacts/execute/2026-04-14_search-cli.md b/.artifacts/execute/2026-04-14_search-cli.md deleted file mode 100644 index 4cd7294..0000000 --- a/.artifacts/execute/2026-04-14_search-cli.md +++ /dev/null @@ -1,176 +0,0 @@ ---- -title: "search-cli execution log" -link: "search-cli-execute" -type: debug_history -ontological_relations: - - relates_to: [[search-cli-plan]] -tags: [execute, search-cli, rust] -uuid: "b2c3d4e5-f6a7-8901-bcde-f23456789012" -created_at: "2026-04-14T12:50:00Z" -plan_path: ".artifacts/plan/2026-04-14_search-cli/PLAN.md" -start_commit: "dee9ad9" -env: {target: "local", notes: ""} ---- - -## Pre-Flight Checks -- Branch: main -- Rollback commit: dee9ad9 -- DoR satisfied: yes -- Access/secrets: present (BRAVE_API_KEY in .env) -- Fixtures/data: ready -- Ready: yes - -## Task Execution - -### T001 – Bootstrap Rust project and dependencies -- Status: completed -- Commit: a974611 -- Files: Cargo.toml, src/main.rs -- Commands: cargo check → success -- Tests: n/a -- Notes: initialized cargo project and added all deps - -### T002 – Implement domain core types -- Status: completed -- Commit: d9d6f23 -- Files: src/domain/mod.rs, src/domain/types.rs, src/domain/query.rs, src/domain/result.rs, src/domain/error.rs -- Commands: cargo check → success (12 dead_code warnings expected) -- Tests: n/a -- Notes: domain layer complete - -### T003 – Implement transport layer (HttpClient trait + reqwest adapter) -- Status: completed -- Commit: 75546e2 -- Files: src/transport/mod.rs, src/transport/http.rs -- Commands: cargo check → success (15 dead_code warnings expected) -- Tests: n/a -- Notes: fixed borrow-checker issue by extracting status before resp.text() - -## Gate Results -- Tests: n/a (no tests yet per plan) -- Type checks: cargo check passes -- Linters: n/a - -## Success Criteria -- [x] T001 completed -- [x] T002 completed -- [x] T003 completed -- [x] Execution log saved - -### Harness – Add justfile with check recipe -- Status: completed -- Commit: ae7335d -- Files: justfile, src/main.rs -- Commands: - - cargo fmt --check → pass - - cargo clippy -- -D warnings -W clippy::complexity -W clippy::cognitive_complexity → pass - - cargo test → pass (0 tests) -- Notes: added `justfile` with `check` recipe; `just` is not installed locally but the recipe is ready. Added `#![allow(dead_code)]` to main.rs as a temporary measure while the codebase is partially built. - -### T004 – Implement SearchProvider trait and capabilities -- Status: completed -- Commit: 01bb1ca -- Files: src/domain/mod.rs, src/domain/provider.rs -- Commands: cargo check → success -- Tests: n/a -- Notes: trait and capabilities defined - -### T005 – Implement Brave DTOs -- Status: completed -- Commit: 8fca979 -- Files: src/providers/mod.rs, src/providers/brave/mod.rs, src/providers/brave/dto.rs -- Commands: cargo check → success -- Tests: n/a -- Notes: created DTOs with Deserialize; added placeholder files for client/config/mapper - -### T006 – Implement Brave mapper -- Status: completed -- Commit: d8a90c5 -- Files: src/providers/brave/mapper.rs -- Commands: cargo test → 4 passed -- Tests: pass -- Notes: fixed borrow-checker issue by extracting total_estimated before consuming dto.web - -### T007 – Implement BraveProvider -- Status: completed -- Commit: c073686 -- Files: src/providers/brave/client.rs, src/providers/brave/config.rs -- Commands: cargo test → 5 passed -- Tests: pass (mock HTTP test) -- Real API test: ✅ `cargo run -- "rust programming"` returned 3 web results from Brave -- Notes: fixed .env format (removed trailing "brave"). main.rs restored to plan state after ad-hoc real test. - -### T008 – Implement SearchService -- Status: completed -- Commit: 3139ed6 -- Files: src/app/mod.rs, src/app/search_service.rs -- Commands: cargo test → 6 passed -- Tests: pass (mock provider delegation) -- Notes: SearchService created with Box - -### T009 – Implement CLI argument parsing -- Status: completed -- Commit: 5a07f88 -- Files: src/cli/mod.rs, src/cli/args.rs -- Commands: cargo run -- --help → displays options; cargo run -- "rust" → prints parsed args -- Tests: n/a -- Notes: added CliSearchType and CliSafeSearch with manual mapping to domain types - -### T010 – Implement output rendering -- Status: completed -- Commit: 6b0e92a -- Files: src/cli/output.rs -- Commands: cargo test → 7 passed -- Tests: pass (mixed results text rendering) -- Notes: renderer handles all four search result types - -### T011 – Wire main.rs and run end-to-end -- Status: completed -- Commit: 3431087 -- Files: src/main.rs -- Commands: cargo run -- "rust programming" → returned 20 web results from Brave -- Tests: pass (live E2E web search) -- Notes: user scoped out images/videos E2E verification; web search confirmed working - -### T012 – Add boundary tests and CI-ready verification -- Status: completed -- Commit: 832e24d -- Files: src/providers/brave/mapper.rs, src/providers/brave/client.rs, src/app/search_service.rs, src/domain/error.rs, src/domain/provider.rs, src/domain/types.rs -- Commands: - - cargo fmt --check → pass - - cargo clippy -- -D warnings -W clippy::complexity -W clippy::cognitive_complexity → pass - - cargo test → 7 passed -- Tests: pass (mapper 4, provider 1, service 1, renderer 1) -- Notes: added #[allow(dead_code)] to domain items planned for future use (TimeRange, ProviderCapabilities, etc.) - -## Gate Results -- Tests: 7/7 passed -- Coverage: n/a (no coverage tool configured) -- Type checks: cargo check → pass -- Linters: cargo clippy with -D warnings + complexity lints → pass -- Format: cargo fmt --check → pass - -## Issues & Resolutions -- T012 – clippy dead_code failures on domain types → added targeted #[allow(dead_code)] attributes - -## Success Criteria -- [x] All planned gates passed -- [x] Execution log saved -- [x] T004–T012 completed - -### Post-T012 – Architecture enforcement -- Status: completed -- Commit: 32f852e -- Files: tests/architecture_test.rs, architecture-report.html -- Commands: cargo test → 12 passed (7 unit + 5 architecture) -- Tests: pass - - domain isolation - - transport isolation - - provider isolation - - app isolation - - render_text isolation -- Notes: HARNESS.md and AGENTS.md updated to reflect enforcement layer - -## Next Steps -- QA review or extend CLI with additional providers/features - diff --git a/.artifacts/execute/2026-04-15_16-35-04_exa-provider.md b/.artifacts/execute/2026-04-15_16-35-04_exa-provider.md deleted file mode 100644 index c5429fd..0000000 --- a/.artifacts/execute/2026-04-15_16-35-04_exa-provider.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -title: "exa-provider execution log" -link: "exa-provider-execute" -type: debug_history -ontological_relations: - - relates_to: [[exa-provider-implementation-plan]] -tags: [execute, exa-provider] -uuid: "c3722e53-79f7-4e5f-9749-fe82838c4b10" -created_at: "2026-04-15T21:35:04Z" -owner: "fabian" -plan_path: ".artifacts/plan/2026-04-15_16-22-21_exa-provider/PLAN.md" -start_commit: "78610af" -env: {target: "local", notes: "Executing against local checkout only."} ---- - -## Pre-Flight Checks -- Branch: exa -- Rollback commit: 2d1593b -- DoR satisfied: yes -- Access/secrets: present (`EXA_API_KEY` in `.env`) -- Fixtures/data: ready -- Ready: yes - -## Task Execution - -### T001 – Extend shared HTTP transport for JSON POST requests -- Status: completed -- Commit: d0bf480 -- Files: `src/transport/http.rs` -- Commands: `cargo test transport::http::tests::test_post_json_decodes_success_response` -> pass -- Tests: pass -- Coverage delta: not measured -- Notes: Added generic `post_json`, shared reqwest response decoding, and a local TCP listener POST regression. The new trait method carries a default error implementation so existing mocks keep compiling until the explicit T006 mock update. - -### T002 – Add Exa provider scaffolding, config, and serde DTOs -- Status: completed -- Commit: 39900d3 -- Files: `src/providers/mod.rs`, `src/providers/exa/mod.rs`, `src/providers/exa/config.rs`, `src/providers/exa/dto.rs`, `src/providers/exa/client.rs`, `src/providers/exa/mapper.rs` -- Commands: `cargo test providers::exa::dto::tests::test_exa_search_response_deserializes_minimal_payload` -> pass -- Tests: pass -- Coverage delta: not measured -- Notes: Added the Exa module export, env-backed config, narrow request/response DTOs, and placeholder `client`/`mapper` modules so the scaffolding compiles cleanly before T003/T004 fill them in. - -### T003 – Map Exa responses into the existing domain result model -- Status: completed -- Commit: f9a1255 -- Files: `src/providers/exa/mapper.rs` -- Commands: `cargo test providers::exa::mapper::tests::test_map_news_response_prefers_summary_and_preserves_author` -> pass -- Tests: pass -- Coverage delta: not measured -- Notes: Added `map_web_response` and `map_news_response`, kept the original query string outside the DTO boundary, preferred `summary` over `text`, and left `total_estimated`/`next_page` unset for Exa. - -### T004 – Implement the Exa provider client and query translation -- Status: completed -- Commit: 050636c -- Files: `Cargo.toml`, `Cargo.lock`, `src/providers/exa/client.rs` -- Commands: `cargo test providers::exa::client::tests::test_exa_provider_news_search_posts_expected_payload` -> pass -- Tests: pass -- Coverage delta: not measured -- Notes: Added `ExaProvider`, explicit invalid-query rejection for unsupported Exa inputs, JSON POST request translation, and ISO-8601 published-date window handling using `chrono`. - -### T005 – Wire provider selection into the CLI and binary composition -- Status: completed -- Commit: f0d02bb -- Files: `src/cli/args.rs`, `src/main.rs` -- Commands: `cargo test cli::args::tests::test_cli_provider_parses_exa_and_defaults_to_brave` -> pass -- Tests: pass -- Coverage delta: not measured -- Notes: Added `--provider`/`-p`, kept Brave as the default runtime path, and moved provider-specific config loading into the `main.rs` provider match without changing `SearchQuery` construction. - -### T006 – Refresh regression coverage and architecture notes for multi-provider runtime -- Status: completed -- Commit: pending -- Files: `src/providers/brave/client.rs`, `src/providers/exa/client.rs`, `docs/architecture.md` -- Commands: `cargo test providers::exa::client::tests::test_exa_provider_rejects_unsupported_query_fields` -> pass -- Tests: pass -- Coverage delta: not measured -- Notes: Updated the Brave test mock for the expanded transport trait, added the Exa unsupported-input regression, and documented runtime provider selection plus the GET/POST split between providers. - -## Gate Results -- Tests: `cargo test` -> 13 unit tests passed, 5 architecture tests passed -- Coverage: not measured by repo gate -- Type checks: covered by `cargo clippy -- -D warnings -W clippy::complexity -W clippy::cognitive_complexity` -> pass -- Linters: `cargo fmt --check` -> pass after `cargo fmt` -- Docs: `mdbook build` -> pass -- Smoke: `cargo run -- "rust" --provider exa --search-type news --limit 1` -> pass, returned 1 Exa news result - -## Issues & Resolutions -- T006 – `just check` initially failed on `cargo fmt --check` for `src/providers/exa/client.rs` -> ran `cargo fmt` and reran the full gate successfully - -## Success Criteria -- [x] All planned gates passed -- [x] Rollout completed or rolled back -- [x] KPIs/SLOs within thresholds -- [x] Execution log saved - -## Next Steps -- QA from execute using `.artifacts/execute/2026-04-15_16-35-04_exa-provider.md` diff --git a/.artifacts/plan/2026-04-14_search-cli/PLAN.md b/.artifacts/plan/2026-04-14_search-cli/PLAN.md deleted file mode 100644 index 9f75918..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/PLAN.md +++ /dev/null @@ -1,1075 +0,0 @@ ---- -title: "Search CLI implementation plan" -link: "search-cli-plan" -type: implementation_plan -ontological_relations: - - relates_to: [[PRD]] -tags: [plan, search-cli, rust, brave, coding] -uuid: "a1b2c3d4-e5f6-7890-abcd-ef1234567890" -created_at: "2026-04-14T12:45:00Z" -parent_research: "PRD.md" -git_commit_at_plan: "618c875" ---- - -## Goal - -Build a Rust CLI application that performs web/news/images/video searches via the Brave Search API. The architecture must be provider-agnostic at the domain and application layers, with Brave implemented as the first provider behind a trait boundary. - -**Out of scope**: CI/CD, packaging (deb/homebrew), adding additional providers (Exa/Tavily), user documentation, deployment. - -## Scope & Assumptions - -**IN scope**: -- Rust project bootstrapping with Cargo -- Domain types (`SearchQuery`, `SearchResponse`, `SearchResult`, errors) -- `SearchProvider` trait with capabilities -- `HttpClient` transport abstraction -- Brave provider implementation (DTOs, mapper, `BraveProvider`) -- Application orchestration (`SearchService`) -- CLI argument parsing with `clap` -- Text output rendering -- Basic boundary tests - -**OUT of scope**: -- Additional search providers -- JSON/table output modes (text only for v0) -- Pagination execution (token modeled but not CLI-exposed) -- Configuration files (API key read from env only) - -**Assumptions**: -- Stable Rust toolchain available -- `reqwest` + `tokio` for async HTTP -- `serde` + `serde_json` for serialization -- `thiserror` for error types -- `async-trait` for trait async methods -- `.env` file present with `BRAVE_API_KEY` (existing) - -## Deliverables - -- `Cargo.toml` and `src/main.rs` -- `src/domain/{mod.rs,query.rs,result.rs,provider.rs,error.rs,types.rs}` -- `src/providers/{mod.rs,brave/{mod.rs,client.rs,config.rs,dto.rs,mapper.rs}}` -- `src/transport/{mod.rs,http.rs}` -- `src/app/{mod.rs,search_service.rs}` -- `src/cli/{mod.rs,args.rs,output.rs}` -- Unit tests for mapper and boundary tests for service - -## Readiness - -- Repository cloned and `.env` present with `BSAqJrJYLQsHK0YGQR82odpW20MuDel brave` -- Rust toolchain installed (`cargo --version` works) -- Internet access for fetching crates - -## Milestones - -- **M1**: Skeleton & domain types — project compiles with domain layer complete -- **M2**: Provider layer — Brave DTOs, mapper, and `BraveProvider` compile and can call Brave API -- **M3**: App & CLI — `SearchService`, CLI args, output rendering wired together -- **M4**: Tests & integration — basic tests pass and CLI runs end-to-end against Brave - -## Ticket Index - - - -| Task | Title | Ticket | -|---|---|---| -| T001 | Bootstrap Rust project and dependencies | [tickets/T001.md](tickets/T001.md) | -| T002 | Implement domain core types | [tickets/T002.md](tickets/T002.md) | -| T003 | Implement transport layer (HttpClient trait + reqwest adapter) | [tickets/T003.md](tickets/T003.md) | -| T004 | Implement SearchProvider trait and capabilities | [tickets/T004.md](tickets/T004.md) | -| T005 | Implement Brave DTOs | [tickets/T005.md](tickets/T005.md) | -| T006 | Implement Brave mapper | [tickets/T006.md](tickets/T006.md) | -| T007 | Implement BraveProvider | [tickets/T007.md](tickets/T007.md) | -| T008 | Implement SearchService | [tickets/T008.md](tickets/T008.md) | -| T009 | Implement CLI argument parsing | [tickets/T009.md](tickets/T009.md) | -| T010 | Implement output rendering | [tickets/T010.md](tickets/T010.md) | -| T011 | Wire main.rs and run end-to-end | [tickets/T011.md](tickets/T011.md) | -| T012 | Add boundary tests and CI-ready verification | [tickets/T012.md](tickets/T012.md) | - - - -## Work Breakdown (Tasks) - -### T001: Bootstrap Rust project and dependencies - -**Summary**: Initialize Cargo project and add all required dependencies to `Cargo.toml`. - -**Owner**: backend - -**Estimate**: 15m - -**Dependencies**: - -**Target milestone**: M1 - -**Acceptance test**: `cargo check` runs without errors on a bare `main.rs`. - -**Files/modules touched**: -- `Cargo.toml` -- `src/main.rs` - -**Steps**: -1. Run `cargo init --name search-cli` in the repo root. -2. Add dependencies to `Cargo.toml`: - ```toml - [dependencies] - tokio = { version = "1", features = ["full"] } - reqwest = { version = "0.12", features = ["json"] } - serde = { version = "1.0", features = ["derive"] } - serde_json = "1.0" - thiserror = "1.0" - async-trait = "0.1" - clap = { version = "4", features = ["derive"] } - dotenvy = "0.15" - ``` -3. Replace `src/main.rs` with a minimal `async fn main() {}`. -4. Run `cargo check` and confirm success. - ---- - -### T002: Implement domain core types - -**Summary**: Create all provider-agnostic domain types, enums, and the top-level error type. - -**Owner**: backend - -**Estimate**: 30m - -**Dependencies**: T001 - -**Target milestone**: M1 - -**Acceptance test**: `cargo check` passes with all domain modules compiling. - -**Files/modules touched**: -- `src/domain/mod.rs` -- `src/domain/types.rs` -- `src/domain/query.rs` -- `src/domain/result.rs` -- `src/domain/error.rs` - -**Steps**: -1. Create `src/domain/mod.rs` that re-exports all submodules: - ```rust - pub mod error; - pub mod query; - pub mod result; - pub mod types; - ``` -2. In `src/domain/types.rs`, define: - ```rust - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub enum SearchType { Web, News, Images, Videos } - - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub enum SafeSearch { Off, Moderate, Strict } - - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub enum TimeRange { Day, Week, Month, Year } - ``` -3. In `src/domain/query.rs`, define: - ```rust - use crate::domain::types::*; - - #[derive(Debug, Clone, PartialEq, Eq)] - pub struct SearchQuery { - pub text: String, - pub search_type: SearchType, - pub limit: Option, - pub offset: Option, - pub safe_search: Option, - pub country: Option, - pub language: Option, - pub time_range: Option, - } - ``` -4. In `src/domain/result.rs`, define: - ```rust - #[derive(Debug, Clone, PartialEq, Eq)] - pub struct PageToken(pub String); - - #[derive(Debug, Clone, PartialEq)] - pub struct SearchResponse { - pub query: String, - pub provider: String, - pub results: Vec, - pub total_estimated: Option, - pub next_page: Option, - } - - #[derive(Debug, Clone, PartialEq)] - pub enum SearchResult { - Web(WebResult), - News(NewsResult), - Image(ImageResult), - Video(VideoResult), - } - - #[derive(Debug, Clone, PartialEq)] - pub struct WebResult { - pub title: String, - pub url: String, - pub snippet: Option, - pub display_url: Option, - } - - #[derive(Debug, Clone, PartialEq)] - pub struct NewsResult { - pub title: String, - pub url: String, - pub snippet: Option, - pub source: Option, - pub published_at: Option, - } - - #[derive(Debug, Clone, PartialEq)] - pub struct ImageResult { - pub title: String, - pub url: String, - pub thumbnail_url: Option, - pub source: Option, - } - - #[derive(Debug, Clone, PartialEq)] - pub struct VideoResult { - pub title: String, - pub url: String, - pub thumbnail_url: Option, - pub duration: Option, - pub published_at: Option, - } - ``` - Note: use `String` for `provider` in `SearchResponse` to avoid circular dependency issues; we will use `"brave".to_string()`. -5. In `src/domain/error.rs`, define: - ```rust - #[derive(Debug, thiserror::Error)] - pub enum SearchError { - #[error("invalid query: {0}")] - InvalidQuery(String), - #[error("provider configuration error: {0}")] - Config(String), - #[error("authentication failed")] - Auth, - #[error("rate limited")] - RateLimited, - #[error("transport error: {0}")] - Transport(String), - #[error("provider returned invalid data: {0}")] - Decode(String), - #[error("provider error: {0}")] - Provider(String), - } - ``` -6. Run `cargo check` and fix any compilation errors. - ---- - -### T003: Implement transport layer (HttpClient trait + reqwest adapter) - -**Summary**: Define the `HttpClient` trait and a `reqwest`-based adapter so providers are decoupled from the HTTP library. - -**Owner**: backend - -**Estimate**: 25m - -**Dependencies**: T001, T002 - -**Target milestone**: M2 - -**Acceptance test**: `cargo check` compiles `transport/` and a simple `ReqwestHttpClient` struct exists. - -**Files/modules touched**: -- `src/transport/mod.rs` -- `src/transport/http.rs` - -**Steps**: -1. Create `src/transport/mod.rs` that declares `pub mod http;`. -2. In `src/transport/http.rs`, add: - ```rust - use async_trait::async_trait; - use crate::domain::error::SearchError; - - #[async_trait] - pub trait HttpClient: Send + Sync { - async fn get_json( - &self, - url: &str, - headers: Vec<(String, String)>, - query: Vec<(String, String)>, - ) -> Result - where - T: serde::de::DeserializeOwned + Send; - } - ``` -3. Implement `ReqwestHttpClient`: - ```rust - use reqwest::Client; - - pub struct ReqwestHttpClient { - client: Client, - } - - impl ReqwestHttpClient { - pub fn new() -> Self { - Self { client: Client::new() } - } - } - - #[async_trait] - impl HttpClient for ReqwestHttpClient { - async fn get_json( - &self, - url: &str, - headers: Vec<(String, String)>, - query: Vec<(String, String)>, - ) -> Result - where - T: serde::de::DeserializeOwned + Send, - { - let mut req = self.client.get(url); - for (k, v) in headers { - req = req.header(k, v); - } - req = req.query(&query); - let resp = req.send().await.map_err(|e| SearchError::Transport(e.to_string()))?; - - if resp.status() == 401 || resp.status() == 403 { - return Err(SearchError::Auth); - } - if resp.status() == 429 { - return Err(SearchError::RateLimited); - } - if !resp.status().is_success() { - let text = resp.text().await.unwrap_or_default(); - return Err(SearchError::Provider(format!("HTTP {}: {}", resp.status(), text))); - } - - resp.json::().await.map_err(|e| SearchError::Decode(e.to_string())) - } - } - ``` -4. Run `cargo check`. - ---- - -### T004: Implement SearchProvider trait and capabilities - -**Summary**: Define the provider trait and capabilities struct in the domain layer so the app layer depends on abstractions. - -**Owner**: backend - -**Estimate**: 20m - -**Dependencies**: T002 - -**Target milestone**: M2 - -**Acceptance test**: `cargo check` passes with `SearchProvider` trait and `ProviderCapabilities` defined. - -**Files/modules touched**: -- `src/domain/mod.rs` -- `src/domain/provider.rs` - -**Steps**: -1. Add `pub mod provider;` to `src/domain/mod.rs`. -2. In `src/domain/provider.rs`, define: - ```rust - use async_trait::async_trait; - use crate::domain::query::SearchQuery; - use crate::domain::result::SearchResponse; - use crate::domain::error::SearchError; - - #[derive(Debug, Clone)] - pub struct ProviderCapabilities { - pub web: bool, - pub news: bool, - pub images: bool, - pub videos: bool, - pub pagination: bool, - pub safe_search: bool, - pub time_range_filter: bool, - } - - #[async_trait] - pub trait SearchProvider: Send + Sync { - fn id(&self) -> String; - fn capabilities(&self) -> ProviderCapabilities; - async fn search(&self, query: &SearchQuery) -> Result; - } - ``` -3. Run `cargo check`. - ---- - -### T005: Implement Brave DTOs - -**Summary**: Create Brave-specific request/response structs to isolate provider JSON shapes from the domain. - -**Owner**: backend - -**Estimate**: 25m - -**Dependencies**: T001, T002 - -**Target milestone**: M2 - -**Acceptance test**: `cargo check` compiles `providers/brave/dto.rs` with all structs deriving `Deserialize`. - -**Files/modules touched**: -- `src/providers/mod.rs` -- `src/providers/brave/mod.rs` -- `src/providers/brave/dto.rs` - -**Steps**: -1. Create `src/providers/mod.rs` with `pub mod brave;`. -2. Create `src/providers/brave/mod.rs` that declares: - ```rust - pub mod client; - pub mod config; - pub mod dto; - pub mod mapper; - ``` -3. In `src/providers/brave/dto.rs`, define the following `Deserialize` structs based on Brave API v1 response shapes: - ```rust - use serde::Deserialize; - - #[derive(Debug, Deserialize)] - pub struct BraveWebResponse { - pub query: Option, - pub web: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveQuery { - pub original: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveWebResults { - pub results: Option>, - pub total: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveWebResult { - pub title: Option, - pub url: Option, - pub description: Option, - pub display_url: Option, - } - - // News - #[derive(Debug, Deserialize)] - pub struct BraveNewsResponse { - pub query: Option, - pub news: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveNewsResults { - pub results: Option>, - } - - #[derive(Debug, Deserialize)] - pub struct BraveNewsResult { - pub title: Option, - pub url: Option, - pub description: Option, - pub source: Option, - pub age: Option, - } - - // Images - #[derive(Debug, Deserialize)] - pub struct BraveImagesResponse { - pub query: Option, - pub image_results: Option>, - } - - #[derive(Debug, Deserialize)] - pub struct BraveImageResult { - pub title: Option, - pub url: Option, - pub thumbnail: Option, - pub source: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveThumbnail { - pub src: Option, - } - - // Videos - #[derive(Debug, Deserialize)] - pub struct BraveVideosResponse { - pub query: Option, - pub videos: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveVideosResults { - pub results: Option>, - } - - #[derive(Debug, Deserialize)] - pub struct BraveVideoResult { - pub title: Option, - pub url: Option, - pub thumbnail: Option, - pub duration: Option, - pub age: Option, - } - ``` -4. Run `cargo check`. - ---- - -### T006: Implement Brave mapper - -**Summary**: Write mapper functions that convert Brave DTOs into domain `SearchResponse` and `SearchResult` enums. - -**Owner**: backend - -**Estimate**: 30m - -**Dependencies**: T004, T005 - -**Target milestone**: M2 - -**Acceptance test**: Unit tests in `mapper.rs` verify that sample Brave DTOs map to correct domain results. - -**Files/modules touched**: -- `src/providers/brave/mapper.rs` - -**Steps**: -1. In `src/providers/brave/mapper.rs`, implement four mapper functions: - ```rust - use crate::domain::result::*; - use crate::providers::brave::dto::*; - - pub fn map_web_response(dto: BraveWebResponse) -> SearchResponse { - let query_text = dto.query.and_then(|q| q.original).unwrap_or_default(); - let results = dto.web.and_then(|w| w.results).unwrap_or_default(); - SearchResponse { - query: query_text, - provider: "brave".to_string(), - total_estimated: dto.web.as_ref().and_then(|w| w.total), - next_page: None, - results: results.into_iter().map(|r| { - SearchResult::Web(WebResult { - title: r.title.unwrap_or_default(), - url: r.url.unwrap_or_default(), - snippet: r.description, - display_url: r.display_url, - }) - }).collect(), - } - } - ``` -2. Implement `map_news_response`, `map_images_response`, `map_videos_response` following the same pattern, mapping to `SearchResult::News`, `SearchResult::Image`, and `SearchResult::Video` respectively. -3. Add `#[cfg(test)]` module with basic unit tests that construct minimal DTOs and assert the mapped domain output. -4. Run `cargo test --lib` for the mapper tests. - ---- - -### T007: Implement BraveProvider - -**Summary**: Build the concrete `BraveProvider` that implements `SearchProvider`, dispatches by `SearchType`, and calls the Brave API. - -**Owner**: backend - -**Estimate**: 45m - -**Dependencies**: T003, T004, T005, T006 - -**Target milestone**: M2 - -**Acceptance test**: A simple integration test (or manual `cargo run`) performs a web search via Brave and prints results. For automated acceptance, mock the `HttpClient` trait to return a sample JSON string and verify `BraveProvider` returns a `SearchResponse`. - -**Files/modules touched**: -- `src/providers/brave/client.rs` -- `src/providers/brave/config.rs` - -**Steps**: -1. In `src/providers/brave/config.rs`, define: - ```rust - #[derive(Debug, Clone)] - pub struct BraveConfig { - pub api_key: String, - pub base_url: String, - } - - impl BraveConfig { - pub fn from_env() -> Result { - let api_key = std::env::var("BRAVE_API_KEY")?; - Ok(Self { - api_key, - base_url: "https://api.search.brave.com/res/v1".to_string(), - }) - } - } - ``` -2. In `src/providers/brave/client.rs`, define: - ```rust - use async_trait::async_trait; - use crate::domain::error::SearchError; - use crate::domain::provider::{ProviderCapabilities, SearchProvider}; - use crate::domain::query::SearchQuery; - use crate::domain::result::SearchResponse; - use crate::domain::types::SearchType; - use crate::transport::http::HttpClient; - use crate::providers::brave::config::BraveConfig; - use crate::providers::brave::dto::*; - use crate::providers::brave::mapper::*; - - pub struct BraveProvider { - client: C, - config: BraveConfig, - } - - impl BraveProvider { - pub fn new(client: C, config: BraveConfig) -> Self { - Self { client, config } - } - } - - #[async_trait] - impl SearchProvider for BraveProvider { - fn id(&self) -> String { - "brave".to_string() - } - - fn capabilities(&self) -> ProviderCapabilities { - ProviderCapabilities { - web: true, - news: true, - images: true, - videos: true, - pagination: false, - safe_search: true, - time_range_filter: true, - } - } - - async fn search(&self, query: &SearchQuery) -> Result { - let endpoint = match query.search_type { - SearchType::Web => "web/search", - SearchType::News => "news/search", - SearchType::Images => "images/search", - SearchType::Videos => "videos/search", - }; - let url = format!("{}/{}", self.config.base_url, endpoint); - - let mut params: Vec<(String, String)> = vec![ - ("q".to_string(), query.text.clone()), - ]; - if let Some(limit) = query.limit { - params.push(("count".to_string(), limit.to_string())); - } - if let Some(offset) = query.offset { - params.push(("offset".to_string(), offset.to_string())); - } - if let Some(ss) = query.safe_search { - let val = match ss { - crate::domain::types::SafeSearch::Off => "off", - crate::domain::types::SafeSearch::Moderate => "moderate", - crate::domain::types::SafeSearch::Strict => "strict", - }; - params.push(("safesearch".to_string(), val.to_string())); - } - if let Some(ref country) = query.country { - params.push(("country".to_string(), country.clone())); - } - if let Some(ref lang) = query.language { - params.push(("search_lang".to_string(), lang.clone())); - } - if let Some(ref tr) = query.time_range { - let val = match tr { - crate::domain::types::TimeRange::Day => "day", - crate::domain::types::TimeRange::Week => "week", - crate::domain::types::TimeRange::Month => "month", - crate::domain::types::TimeRange::Year => "year", - }; - params.push(("freshness".to_string(), val.to_string())); - } - - let headers = vec![ - ("Accept".to_string(), "application/json".to_string()), - ("X-Subscription-Token".to_string(), self.config.api_key.clone()), - ]; - - match query.search_type { - SearchType::Web => { - let dto: BraveWebResponse = self.client.get_json(&url, headers, params).await?; - Ok(map_web_response(dto)) - } - SearchType::News => { - let dto: BraveNewsResponse = self.client.get_json(&url, headers, params).await?; - Ok(map_news_response(dto)) - } - SearchType::Images => { - let dto: BraveImagesResponse = self.client.get_json(&url, headers, params).await?; - Ok(map_images_response(dto)) - } - SearchType::Videos => { - let dto: BraveVideosResponse = self.client.get_json(&url, headers, params).await?; - Ok(map_videos_response(dto)) - } - } - } - } - ``` -3. Add a `#[cfg(test)]` mock `HttpClient` that returns a hardcoded JSON string and assert `BraveProvider::search` produces the expected `SearchResponse`. -4. Run `cargo test --lib`. - ---- - -### T008: Implement SearchService - -**Summary**: Build the application-layer `SearchService` that accepts a `dyn SearchProvider` and orchestrates searches. - -**Owner**: backend - -**Estimate**: 20m - -**Dependencies**: T004, T007 - -**Target milestone**: M3 - -**Acceptance test**: `cargo check` compiles `SearchService` and a test proves it delegates to a mocked `SearchProvider`. - -**Files/modules touched**: -- `src/app/mod.rs` -- `src/app/search_service.rs` - -**Steps**: -1. Create `src/app/mod.rs` with `pub mod search_service;`. -2. In `src/app/search_service.rs`, define: - ```rust - use crate::domain::error::SearchError; - use crate::domain::provider::SearchProvider; - use crate::domain::query::SearchQuery; - use crate::domain::result::SearchResponse; - - pub struct SearchService { - provider: Box, - } - - impl SearchService { - pub fn new(provider: Box) -> Self { - Self { provider } - } - - pub async fn search(&self, query: SearchQuery) -> Result { - self.provider.search(&query).await - } - } - ``` -3. Add a test that creates a mock `SearchProvider` (implement the trait on a simple struct) and verify `SearchService::search` returns the mock response. -4. Run `cargo test --lib`. - ---- - -### T009: Implement CLI argument parsing - -**Summary**: Use `clap` to parse user input into a `SearchCommand` struct with query, search type, limit, and output format. - -**Owner**: backend - -**Estimate**: 25m - -**Dependencies**: T001, T002 - -**Target milestone**: M3 - -**Acceptance test**: `cargo run -- --help` displays all options and `cargo run -- "rust"` defaults to web search. - -**Files/modules touched**: -- `src/cli/mod.rs` -- `src/cli/args.rs` - -**Steps**: -1. Create `src/cli/mod.rs` with: - ```rust - pub mod args; - pub mod output; - ``` -2. In `src/cli/args.rs`, define: - ```rust - use clap::{Parser, ValueEnum}; - use crate::domain::types::SearchType; - - #[derive(Debug, Clone, ValueEnum)] - pub enum CliSearchType { - Web, - News, - Images, - Videos, - } - - impl From for SearchType { - fn from(val: CliSearchType) -> Self { - match val { - CliSearchType::Web => SearchType::Web, - CliSearchType::News => SearchType::News, - CliSearchType::Images => SearchType::Images, - CliSearchType::Videos => SearchType::Videos, - } - } - } - - #[derive(Parser, Debug)] - #[command(name = "search-cli")] - #[command(about = "Provider-agnostic search CLI")] - pub struct CliArgs { - #[arg(help = "Search query text")] - pub query: String, - - #[arg(short, long, value_enum, default_value = "web")] - pub search_type: CliSearchType, - - #[arg(short, long)] - pub limit: Option, - - #[arg(long)] - pub offset: Option, - - #[arg(long, value_enum)] - pub safe_search: Option, - - #[arg(long)] - pub country: Option, - - #[arg(long)] - pub language: Option, - } - ``` - Note: `SafeSearch` already derives the necessary traits, but `clap::ValueEnum` requires additional derive. If compilation fails, add a manual mapping instead of using `value_enum` on `SafeSearch`. Create a `CliSafeSearch` enum and map it if needed. -3. Update `src/main.rs` to parse args: - ```rust - use clap::Parser; - use search_cli::cli::args::CliArgs; - - #[tokio::main] - async fn main() { - let _args = CliArgs::parse(); - println!("{:?}", _args); - } - ``` - (Temporarily expose `cli` module from a lib or keep everything in `main.rs` if the project is bin-only.) - - **Important**: Since this is a binary crate, either make `main.rs` contain the modules directly, or create `src/lib.rs`. To keep it simple, declare modules in `main.rs`: - ```rust - mod app; - mod cli; - mod domain; - mod providers; - mod transport; - ``` - Update `main.rs` to include these mod declarations and parse args. -4. Run `cargo run -- --help` and confirm output. -5. Run `cargo run -- "rust"` and confirm it prints the parsed args. - ---- - -### T010: Implement output rendering - -**Summary**: Create a simple text renderer that prints `SearchResponse` results in a human-readable format. - -**Owner**: backend - -**Estimate**: 20m - -**Dependencies**: T002, T009 - -**Target milestone**: M3 - -**Acceptance test**: A test provides a `SearchResponse` and the renderer returns a string containing all result titles and URLs. - -**Files/modules touched**: -- `src/cli/output.rs` - -**Steps**: -1. In `src/cli/output.rs`, define: - ```rust - use crate::domain::result::{SearchResponse, SearchResult}; - - pub fn render_text(response: &SearchResponse) -> String { - let mut lines = vec![ - format!("Provider: {}", response.provider), - format!("Query: {}", response.query), - format!("Results: {}", response.results.len()), - String::new(), - ]; - for (i, result) in response.results.iter().enumerate() { - match result { - SearchResult::Web(r) => { - lines.push(format!("{}. [{}]", i + 1, r.title)); - lines.push(format!(" URL: {}", r.url)); - if let Some(s) = &r.snippet { - lines.push(format!(" {}", s)); - } - } - SearchResult::News(r) => { - lines.push(format!("{}. [NEWS] {}", i + 1, r.title)); - lines.push(format!(" URL: {}", r.url)); - if let Some(s) = &r.source { - lines.push(format!(" Source: {}", s)); - } - } - SearchResult::Image(r) => { - lines.push(format!("{}. [IMAGE] {}", i + 1, r.title)); - lines.push(format!(" URL: {}", r.url)); - } - SearchResult::Video(r) => { - lines.push(format!("{}. [VIDEO] {}", i + 1, r.title)); - lines.push(format!(" URL: {}", r.url)); - } - } - lines.push(String::new()); - } - lines.join("\n") - } - ``` -2. Add a unit test that constructs a `SearchResponse` with mixed results and asserts the rendered string contains expected substrings. -3. Run `cargo test --lib`. - ---- - -### T011: Wire main.rs and run end-to-end - -**Summary**: Connect CLI parsing, `SearchService`, `BraveProvider`, and output rendering in `main.rs` so the CLI performs a live search. - -**Owner**: backend - -**Estimate**: 25m - -**Dependencies**: T007, T008, T009, T010 - -**Target milestone**: M4 - -**Acceptance test**: `cargo run -- "rust programming"` successfully queries Brave and prints at least one search result. If API key is missing, it prints a clean error message. - -**Files/modules touched**: -- `src/main.rs` - -**Steps**: -1. In `src/main.rs`, replace contents with: - ```rust - mod app; - mod cli; - mod domain; - mod providers; - mod transport; - - use clap::Parser; - use cli::args::CliArgs; - use cli::output::render_text; - use app::search_service::SearchService; - use domain::query::SearchQuery; - use domain::types::SafeSearch; - use providers::brave::client::BraveProvider; - use providers::brave::config::BraveConfig; - use transport::http::ReqwestHttpClient; - - #[tokio::main] - async fn main() { - dotenvy::dotenv().ok(); - - let args = CliArgs::parse(); - - let config = match BraveConfig::from_env() { - Ok(c) => c, - Err(e) => { - eprintln!("Failed to load Brave config: {}", e); - std::process::exit(1); - } - }; - - let client = ReqwestHttpClient::new(); - let provider = BraveProvider::new(client, config); - let service = SearchService::new(Box::new(provider)); - - let query = SearchQuery { - text: args.query, - search_type: args.search_type.into(), - limit: args.limit, - offset: args.offset, - safe_search: args.safe_search, - country: args.country, - language: args.language, - time_range: None, - }; - - match service.search(query).await { - Ok(response) => { - println!("{}", render_text(&response)); - } - Err(e) => { - eprintln!("Search failed: {}", e); - std::process::exit(1); - } - } - } - ``` - Note: If `SafeSearch` does not implement `clap::ValueEnum`, adjust `CliArgs` to use a local `CliSafeSearch` enum and map it in the `SearchQuery` construction. -2. Ensure `cargo check` passes. -3. Run `cargo run -- "rust programming"` and verify it returns search results. -4. Run `cargo run -- "rust" --search-type news` and verify news results. -5. Run `cargo run -- "cats" --search-type images --limit 3` and verify image results. - ---- - -### T012: Add boundary tests and CI-ready verification - -**Summary**: Add integration-style tests at the app and provider boundaries, and ensure `cargo test` passes cleanly. - -**Owner**: backend - -**Estimate**: 30m - -**Dependencies**: T011 - -**Target milestone**: M4 - -**Acceptance test**: `cargo test` passes with at least one test for mapper, one for `BraveProvider` with mock HTTP, and one for `SearchService` with mock provider. - -**Files/modules touched**: -- `src/providers/brave/mapper.rs` -- `src/providers/brave/client.rs` -- `src/app/search_service.rs` - -**Steps**: -1. In `src/providers/brave/mapper.rs`, ensure the existing `#[cfg(test)]` module covers all four search types. -2. In `src/providers/brave/client.rs`, ensure the mock HTTP test validates that headers include `X-Subscription-Token` and that query params include `q`, `count`, and `safesearch`. -3. In `src/app/search_service.rs`, ensure the mock provider test validates delegation. -4. Run `cargo test` and fix any failures. -5. Run `cargo clippy` (if available) and fix warnings. - -## Risks & Mitigations - -| Risk | Mitigation | -|------|------------| -| Brave API schema differs from documented DTOs | Start with minimal fields; mapper defaults `Option` values to empty strings rather than failing | -| `clap` `ValueEnum` incompatibility with domain enums | Use separate CLI enums and map them explicitly to domain types | -| Rate limiting during manual testing | Keep query counts low; use `limit` flag; mock HTTP for automated tests | -| Missing env var in CI/test environments | `BraveConfig::from_env()` returns clear error; tests use mock config | - -## Test Strategy - -- **Mapper tests**: Construct minimal DTOs, assert domain output (T006). -- **Provider tests**: Mock `HttpClient` to return JSON, assert `BraveProvider` produces `SearchResponse` (T007). -- **Service tests**: Mock `SearchProvider`, assert `SearchService` delegates correctly (T008). -- **Renderer tests**: Provide `SearchResponse`, assert string output contains expected data (T010). -- **E2E manual**: Run CLI with live Brave API for web, news, images, and video queries (T011). - -## References - -- PRD.md sections 1–10 (domain design, provider trait, Brave mapping, module layout) -- `Cargo.toml` dependency versions aligned with PRD async/HTTP stack -- Brave Search API docs: https://api-dashboard.search.brave.com/documentation - -## Final Gate - -- **Output summary**: plan dir `.artifacts/plan/2026-04-14_search-cli/`, 4 milestones, 12 tickets -- **Next step**: proceed to execute-phase with `.artifacts/plan/2026-04-14_search-cli/PLAN.md` diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/INDEX.md b/.artifacts/plan/2026-04-14_search-cli/tickets/INDEX.md deleted file mode 100644 index 1ab5c79..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/INDEX.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: "Ticket Index" -type: ticket_index -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# Ticket Index - -| Task | Title | Ticket | -|---|---|---| -| T001 | Bootstrap Rust project and dependencies | [T001](./T001.md) | -| T002 | Implement domain core types | [T002](./T002.md) | -| T003 | Implement transport layer (HttpClient trait + reqwest adapter) | [T003](./T003.md) | -| T004 | Implement SearchProvider trait and capabilities | [T004](./T004.md) | -| T005 | Implement Brave DTOs | [T005](./T005.md) | -| T006 | Implement Brave mapper | [T006](./T006.md) | -| T007 | Implement BraveProvider | [T007](./T007.md) | -| T008 | Implement SearchService | [T008](./T008.md) | -| T009 | Implement CLI argument parsing | [T009](./T009.md) | -| T010 | Implement output rendering | [T010](./T010.md) | -| T011 | Wire main.rs and run end-to-end | [T011](./T011.md) | -| T012 | Add boundary tests and CI-ready verification | [T012](./T012.md) | diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T001.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T001.md deleted file mode 100644 index c052e60..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T001.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: "T001: Bootstrap Rust project and dependencies" -type: plan_ticket -task_id: "T001" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T001: Bootstrap Rust project and dependencies - -**Summary**: Initialize Cargo project and add all required dependencies to `Cargo.toml`. - -**Owner**: backend - -**Estimate**: 15m - -**Dependencies**: - -**Target milestone**: M1 - -**Acceptance test**: `cargo check` runs without errors on a bare `main.rs`. - -**Files/modules touched**: -- `Cargo.toml` -- `src/main.rs` - -**Steps**: -1. Run `cargo init --name search-cli` in the repo root. -2. Add dependencies to `Cargo.toml`: - ```toml - [dependencies] - tokio = { version = "1", features = ["full"] } - reqwest = { version = "0.12", features = ["json"] } - serde = { version = "1.0", features = ["derive"] } - serde_json = "1.0" - thiserror = "1.0" - async-trait = "0.1" - clap = { version = "4", features = ["derive"] } - dotenvy = "0.15" - ``` -3. Replace `src/main.rs` with a minimal `async fn main() {}`. -4. Run `cargo check` and confirm success. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T002.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T002.md deleted file mode 100644 index 87157ac..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T002.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -title: "T002: Implement domain core types" -type: plan_ticket -task_id: "T002" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T002: Implement domain core types - -**Summary**: Create all provider-agnostic domain types, enums, and the top-level error type. - -**Owner**: backend - -**Estimate**: 30m - -**Dependencies**: T001 - -**Target milestone**: M1 - -**Acceptance test**: `cargo check` passes with all domain modules compiling. - -**Files/modules touched**: -- `src/domain/mod.rs` -- `src/domain/types.rs` -- `src/domain/query.rs` -- `src/domain/result.rs` -- `src/domain/error.rs` - -**Steps**: -1. Create `src/domain/mod.rs` that re-exports all submodules: - ```rust - pub mod error; - pub mod query; - pub mod result; - pub mod types; - ``` -2. In `src/domain/types.rs`, define: - ```rust - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub enum SearchType { Web, News, Images, Videos } - - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub enum SafeSearch { Off, Moderate, Strict } - - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub enum TimeRange { Day, Week, Month, Year } - ``` -3. In `src/domain/query.rs`, define: - ```rust - use crate::domain::types::*; - - #[derive(Debug, Clone, PartialEq, Eq)] - pub struct SearchQuery { - pub text: String, - pub search_type: SearchType, - pub limit: Option, - pub offset: Option, - pub safe_search: Option, - pub country: Option, - pub language: Option, - pub time_range: Option, - } - ``` -4. In `src/domain/result.rs`, define: - ```rust - #[derive(Debug, Clone, PartialEq, Eq)] - pub struct PageToken(pub String); - - #[derive(Debug, Clone, PartialEq)] - pub struct SearchResponse { - pub query: String, - pub provider: String, - pub results: Vec, - pub total_estimated: Option, - pub next_page: Option, - } - - #[derive(Debug, Clone, PartialEq)] - pub enum SearchResult { - Web(WebResult), - News(NewsResult), - Image(ImageResult), - Video(VideoResult), - } - - #[derive(Debug, Clone, PartialEq)] - pub struct WebResult { - pub title: String, - pub url: String, - pub snippet: Option, - pub display_url: Option, - } - - #[derive(Debug, Clone, PartialEq)] - pub struct NewsResult { - pub title: String, - pub url: String, - pub snippet: Option, - pub source: Option, - pub published_at: Option, - } - - #[derive(Debug, Clone, PartialEq)] - pub struct ImageResult { - pub title: String, - pub url: String, - pub thumbnail_url: Option, - pub source: Option, - } - - #[derive(Debug, Clone, PartialEq)] - pub struct VideoResult { - pub title: String, - pub url: String, - pub thumbnail_url: Option, - pub duration: Option, - pub published_at: Option, - } - ``` - Note: use `String` for `provider` in `SearchResponse` to avoid circular dependency issues; we will use `"brave".to_string()`. -5. In `src/domain/error.rs`, define: - ```rust - #[derive(Debug, thiserror::Error)] - pub enum SearchError { - #[error("invalid query: {0}")] - InvalidQuery(String), - #[error("provider configuration error: {0}")] - Config(String), - #[error("authentication failed")] - Auth, - #[error("rate limited")] - RateLimited, - #[error("transport error: {0}")] - Transport(String), - #[error("provider returned invalid data: {0}")] - Decode(String), - #[error("provider error: {0}")] - Provider(String), - } - ``` -6. Run `cargo check` and fix any compilation errors. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T003.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T003.md deleted file mode 100644 index 059d404..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T003.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -title: "T003: Implement transport layer (HttpClient trait + reqwest adapter)" -type: plan_ticket -task_id: "T003" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T003: Implement transport layer (HttpClient trait + reqwest adapter) - -**Summary**: Define the `HttpClient` trait and a `reqwest`-based adapter so providers are decoupled from the HTTP library. - -**Owner**: backend - -**Estimate**: 25m - -**Dependencies**: T001, T002 - -**Target milestone**: M2 - -**Acceptance test**: `cargo check` compiles `transport/` and a simple `ReqwestHttpClient` struct exists. - -**Files/modules touched**: -- `src/transport/mod.rs` -- `src/transport/http.rs` - -**Steps**: -1. Create `src/transport/mod.rs` that declares `pub mod http;`. -2. In `src/transport/http.rs`, add: - ```rust - use async_trait::async_trait; - use crate::domain::error::SearchError; - - #[async_trait] - pub trait HttpClient: Send + Sync { - async fn get_json( - &self, - url: &str, - headers: Vec<(String, String)>, - query: Vec<(String, String)>, - ) -> Result - where - T: serde::de::DeserializeOwned + Send; - } - ``` -3. Implement `ReqwestHttpClient`: - ```rust - use reqwest::Client; - - pub struct ReqwestHttpClient { - client: Client, - } - - impl ReqwestHttpClient { - pub fn new() -> Self { - Self { client: Client::new() } - } - } - - #[async_trait] - impl HttpClient for ReqwestHttpClient { - async fn get_json( - &self, - url: &str, - headers: Vec<(String, String)>, - query: Vec<(String, String)>, - ) -> Result - where - T: serde::de::DeserializeOwned + Send, - { - let mut req = self.client.get(url); - for (k, v) in headers { - req = req.header(k, v); - } - req = req.query(&query); - let resp = req.send().await.map_err(|e| SearchError::Transport(e.to_string()))?; - - if resp.status() == 401 || resp.status() == 403 { - return Err(SearchError::Auth); - } - if resp.status() == 429 { - return Err(SearchError::RateLimited); - } - if !resp.status().is_success() { - let text = resp.text().await.unwrap_or_default(); - return Err(SearchError::Provider(format!("HTTP {}: {}", resp.status(), text))); - } - - resp.json::().await.map_err(|e| SearchError::Decode(e.to_string())) - } - } - ``` -4. Run `cargo check`. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T004.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T004.md deleted file mode 100644 index 2958375..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T004.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: "T004: Implement SearchProvider trait and capabilities" -type: plan_ticket -task_id: "T004" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T004: Implement SearchProvider trait and capabilities - -**Summary**: Define the provider trait and capabilities struct in the domain layer so the app layer depends on abstractions. - -**Owner**: backend - -**Estimate**: 20m - -**Dependencies**: T002 - -**Target milestone**: M2 - -**Acceptance test**: `cargo check` passes with `SearchProvider` trait and `ProviderCapabilities` defined. - -**Files/modules touched**: -- `src/domain/mod.rs` -- `src/domain/provider.rs` - -**Steps**: -1. Add `pub mod provider;` to `src/domain/mod.rs`. -2. In `src/domain/provider.rs`, define: - ```rust - use async_trait::async_trait; - use crate::domain::query::SearchQuery; - use crate::domain::result::SearchResponse; - use crate::domain::error::SearchError; - - #[derive(Debug, Clone)] - pub struct ProviderCapabilities { - pub web: bool, - pub news: bool, - pub images: bool, - pub videos: bool, - pub pagination: bool, - pub safe_search: bool, - pub time_range_filter: bool, - } - - #[async_trait] - pub trait SearchProvider: Send + Sync { - fn id(&self) -> String; - fn capabilities(&self) -> ProviderCapabilities; - async fn search(&self, query: &SearchQuery) -> Result; - } - ``` -3. Run `cargo check`. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T005.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T005.md deleted file mode 100644 index f0ba742..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T005.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -title: "T005: Implement Brave DTOs" -type: plan_ticket -task_id: "T005" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T005: Implement Brave DTOs - -**Summary**: Create Brave-specific request/response structs to isolate provider JSON shapes from the domain. - -**Owner**: backend - -**Estimate**: 25m - -**Dependencies**: T001, T002 - -**Target milestone**: M2 - -**Acceptance test**: `cargo check` compiles `providers/brave/dto.rs` with all structs deriving `Deserialize`. - -**Files/modules touched**: -- `src/providers/mod.rs` -- `src/providers/brave/mod.rs` -- `src/providers/brave/dto.rs` - -**Steps**: -1. Create `src/providers/mod.rs` with `pub mod brave;`. -2. Create `src/providers/brave/mod.rs` that declares: - ```rust - pub mod client; - pub mod config; - pub mod dto; - pub mod mapper; - ``` -3. In `src/providers/brave/dto.rs`, define the following `Deserialize` structs based on Brave API v1 response shapes: - ```rust - use serde::Deserialize; - - #[derive(Debug, Deserialize)] - pub struct BraveWebResponse { - pub query: Option, - pub web: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveQuery { - pub original: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveWebResults { - pub results: Option>, - pub total: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveWebResult { - pub title: Option, - pub url: Option, - pub description: Option, - pub display_url: Option, - } - - // News - #[derive(Debug, Deserialize)] - pub struct BraveNewsResponse { - pub query: Option, - pub news: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveNewsResults { - pub results: Option>, - } - - #[derive(Debug, Deserialize)] - pub struct BraveNewsResult { - pub title: Option, - pub url: Option, - pub description: Option, - pub source: Option, - pub age: Option, - } - - // Images - #[derive(Debug, Deserialize)] - pub struct BraveImagesResponse { - pub query: Option, - pub image_results: Option>, - } - - #[derive(Debug, Deserialize)] - pub struct BraveImageResult { - pub title: Option, - pub url: Option, - pub thumbnail: Option, - pub source: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveThumbnail { - pub src: Option, - } - - // Videos - #[derive(Debug, Deserialize)] - pub struct BraveVideosResponse { - pub query: Option, - pub videos: Option, - } - - #[derive(Debug, Deserialize)] - pub struct BraveVideosResults { - pub results: Option>, - } - - #[derive(Debug, Deserialize)] - pub struct BraveVideoResult { - pub title: Option, - pub url: Option, - pub thumbnail: Option, - pub duration: Option, - pub age: Option, - } - ``` -4. Run `cargo check`. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T006.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T006.md deleted file mode 100644 index 8506614..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T006.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -title: "T006: Implement Brave mapper" -type: plan_ticket -task_id: "T006" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T006: Implement Brave mapper - -**Summary**: Write mapper functions that convert Brave DTOs into domain `SearchResponse` and `SearchResult` enums. - -**Owner**: backend - -**Estimate**: 30m - -**Dependencies**: T004, T005 - -**Target milestone**: M2 - -**Acceptance test**: Unit tests in `mapper.rs` verify that sample Brave DTOs map to correct domain results. - -**Files/modules touched**: -- `src/providers/brave/mapper.rs` - -**Steps**: -1. In `src/providers/brave/mapper.rs`, implement four mapper functions: - ```rust - use crate::domain::result::*; - use crate::providers::brave::dto::*; - - pub fn map_web_response(dto: BraveWebResponse) -> SearchResponse { - let query_text = dto.query.and_then(|q| q.original).unwrap_or_default(); - let results = dto.web.and_then(|w| w.results).unwrap_or_default(); - SearchResponse { - query: query_text, - provider: "brave".to_string(), - total_estimated: dto.web.as_ref().and_then(|w| w.total), - next_page: None, - results: results.into_iter().map(|r| { - SearchResult::Web(WebResult { - title: r.title.unwrap_or_default(), - url: r.url.unwrap_or_default(), - snippet: r.description, - display_url: r.display_url, - }) - }).collect(), - } - } - ``` -2. Implement `map_news_response`, `map_images_response`, `map_videos_response` following the same pattern, mapping to `SearchResult::News`, `SearchResult::Image`, and `SearchResult::Video` respectively. -3. Add `#[cfg(test)]` module with basic unit tests that construct minimal DTOs and assert the mapped domain output. -4. Run `cargo test --lib` for the mapper tests. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T007.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T007.md deleted file mode 100644 index 622d2a3..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T007.md +++ /dev/null @@ -1,160 +0,0 @@ ---- -title: "T007: Implement BraveProvider" -type: plan_ticket -task_id: "T007" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T007: Implement BraveProvider - -**Summary**: Build the concrete `BraveProvider` that implements `SearchProvider`, dispatches by `SearchType`, and calls the Brave API. - -**Owner**: backend - -**Estimate**: 45m - -**Dependencies**: T003, T004, T005, T006 - -**Target milestone**: M2 - -**Acceptance test**: A simple integration test (or manual `cargo run`) performs a web search via Brave and prints results. For automated acceptance, mock the `HttpClient` trait to return a sample JSON string and verify `BraveProvider` returns a `SearchResponse`. - -**Files/modules touched**: -- `src/providers/brave/client.rs` -- `src/providers/brave/config.rs` - -**Steps**: -1. In `src/providers/brave/config.rs`, define: - ```rust - #[derive(Debug, Clone)] - pub struct BraveConfig { - pub api_key: String, - pub base_url: String, - } - - impl BraveConfig { - pub fn from_env() -> Result { - let api_key = std::env::var("BRAVE_API_KEY")?; - Ok(Self { - api_key, - base_url: "https://api.search.brave.com/res/v1".to_string(), - }) - } - } - ``` -2. In `src/providers/brave/client.rs`, define: - ```rust - use async_trait::async_trait; - use crate::domain::error::SearchError; - use crate::domain::provider::{ProviderCapabilities, SearchProvider}; - use crate::domain::query::SearchQuery; - use crate::domain::result::SearchResponse; - use crate::domain::types::SearchType; - use crate::transport::http::HttpClient; - use crate::providers::brave::config::BraveConfig; - use crate::providers::brave::dto::*; - use crate::providers::brave::mapper::*; - - pub struct BraveProvider { - client: C, - config: BraveConfig, - } - - impl BraveProvider { - pub fn new(client: C, config: BraveConfig) -> Self { - Self { client, config } - } - } - - #[async_trait] - impl SearchProvider for BraveProvider { - fn id(&self) -> String { - "brave".to_string() - } - - fn capabilities(&self) -> ProviderCapabilities { - ProviderCapabilities { - web: true, - news: true, - images: true, - videos: true, - pagination: false, - safe_search: true, - time_range_filter: true, - } - } - - async fn search(&self, query: &SearchQuery) -> Result { - let endpoint = match query.search_type { - SearchType::Web => "web/search", - SearchType::News => "news/search", - SearchType::Images => "images/search", - SearchType::Videos => "videos/search", - }; - let url = format!("{}/{}", self.config.base_url, endpoint); - - let mut params: Vec<(String, String)> = vec![ - ("q".to_string(), query.text.clone()), - ]; - if let Some(limit) = query.limit { - params.push(("count".to_string(), limit.to_string())); - } - if let Some(offset) = query.offset { - params.push(("offset".to_string(), offset.to_string())); - } - if let Some(ss) = query.safe_search { - let val = match ss { - crate::domain::types::SafeSearch::Off => "off", - crate::domain::types::SafeSearch::Moderate => "moderate", - crate::domain::types::SafeSearch::Strict => "strict", - }; - params.push(("safesearch".to_string(), val.to_string())); - } - if let Some(ref country) = query.country { - params.push(("country".to_string(), country.clone())); - } - if let Some(ref lang) = query.language { - params.push(("search_lang".to_string(), lang.clone())); - } - if let Some(ref tr) = query.time_range { - let val = match tr { - crate::domain::types::TimeRange::Day => "day", - crate::domain::types::TimeRange::Week => "week", - crate::domain::types::TimeRange::Month => "month", - crate::domain::types::TimeRange::Year => "year", - }; - params.push(("freshness".to_string(), val.to_string())); - } - - let headers = vec![ - ("Accept".to_string(), "application/json".to_string()), - ("X-Subscription-Token".to_string(), self.config.api_key.clone()), - ]; - - match query.search_type { - SearchType::Web => { - let dto: BraveWebResponse = self.client.get_json(&url, headers, params).await?; - Ok(map_web_response(dto)) - } - SearchType::News => { - let dto: BraveNewsResponse = self.client.get_json(&url, headers, params).await?; - Ok(map_news_response(dto)) - } - SearchType::Images => { - let dto: BraveImagesResponse = self.client.get_json(&url, headers, params).await?; - Ok(map_images_response(dto)) - } - SearchType::Videos => { - let dto: BraveVideosResponse = self.client.get_json(&url, headers, params).await?; - Ok(map_videos_response(dto)) - } - } - } - } - ``` -3. Add a `#[cfg(test)]` mock `HttpClient` that returns a hardcoded JSON string and assert `BraveProvider::search` produces the expected `SearchResponse`. -4. Run `cargo test --lib`. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T008.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T008.md deleted file mode 100644 index a795a1f..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T008.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: "T008: Implement SearchService" -type: plan_ticket -task_id: "T008" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T008: Implement SearchService - -**Summary**: Build the application-layer `SearchService` that accepts a `dyn SearchProvider` and orchestrates searches. - -**Owner**: backend - -**Estimate**: 20m - -**Dependencies**: T004, T007 - -**Target milestone**: M3 - -**Acceptance test**: `cargo check` compiles `SearchService` and a test proves it delegates to a mocked `SearchProvider`. - -**Files/modules touched**: -- `src/app/mod.rs` -- `src/app/search_service.rs` - -**Steps**: -1. Create `src/app/mod.rs` with `pub mod search_service;`. -2. In `src/app/search_service.rs`, define: - ```rust - use crate::domain::error::SearchError; - use crate::domain::provider::SearchProvider; - use crate::domain::query::SearchQuery; - use crate::domain::result::SearchResponse; - - pub struct SearchService { - provider: Box, - } - - impl SearchService { - pub fn new(provider: Box) -> Self { - Self { provider } - } - - pub async fn search(&self, query: SearchQuery) -> Result { - self.provider.search(&query).await - } - } - ``` -3. Add a test that creates a mock `SearchProvider` (implement the trait on a simple struct) and verify `SearchService::search` returns the mock response. -4. Run `cargo test --lib`. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T009.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T009.md deleted file mode 100644 index d070b42..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T009.md +++ /dev/null @@ -1,110 +0,0 @@ ---- -title: "T009: Implement CLI argument parsing" -type: plan_ticket -task_id: "T009" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T009: Implement CLI argument parsing - -**Summary**: Use `clap` to parse user input into a `SearchCommand` struct with query, search type, limit, and output format. - -**Owner**: backend - -**Estimate**: 25m - -**Dependencies**: T001, T002 - -**Target milestone**: M3 - -**Acceptance test**: `cargo run -- --help` displays all options and `cargo run -- "rust"` defaults to web search. - -**Files/modules touched**: -- `src/cli/mod.rs` -- `src/cli/args.rs` - -**Steps**: -1. Create `src/cli/mod.rs` with: - ```rust - pub mod args; - pub mod output; - ``` -2. In `src/cli/args.rs`, define: - ```rust - use clap::{Parser, ValueEnum}; - use crate::domain::types::SearchType; - - #[derive(Debug, Clone, ValueEnum)] - pub enum CliSearchType { - Web, - News, - Images, - Videos, - } - - impl From for SearchType { - fn from(val: CliSearchType) -> Self { - match val { - CliSearchType::Web => SearchType::Web, - CliSearchType::News => SearchType::News, - CliSearchType::Images => SearchType::Images, - CliSearchType::Videos => SearchType::Videos, - } - } - } - - #[derive(Parser, Debug)] - #[command(name = "search-cli")] - #[command(about = "Provider-agnostic search CLI")] - pub struct CliArgs { - #[arg(help = "Search query text")] - pub query: String, - - #[arg(short, long, value_enum, default_value = "web")] - pub search_type: CliSearchType, - - #[arg(short, long)] - pub limit: Option, - - #[arg(long)] - pub offset: Option, - - #[arg(long, value_enum)] - pub safe_search: Option, - - #[arg(long)] - pub country: Option, - - #[arg(long)] - pub language: Option, - } - ``` - Note: `SafeSearch` already derives the necessary traits, but `clap::ValueEnum` requires additional derive. If compilation fails, add a manual mapping instead of using `value_enum` on `SafeSearch`. Create a `CliSafeSearch` enum and map it if needed. -3. Update `src/main.rs` to parse args: - ```rust - use clap::Parser; - use search_cli::cli::args::CliArgs; - - #[tokio::main] - async fn main() { - let _args = CliArgs::parse(); - println!("{:?}", _args); - } - ``` - (Temporarily expose `cli` module from a lib or keep everything in `main.rs` if the project is bin-only.) - - **Important**: Since this is a binary crate, either make `main.rs` contain the modules directly, or create `src/lib.rs`. To keep it simple, declare modules in `main.rs`: - ```rust - mod app; - mod cli; - mod domain; - mod providers; - mod transport; - ``` - Update `main.rs` to include these mod declarations and parse args. -4. Run `cargo run -- --help` and confirm output. -5. Run `cargo run -- "rust"` and confirm it prints the parsed args. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T010.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T010.md deleted file mode 100644 index c029dd5..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T010.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: "T010: Implement output rendering" -type: plan_ticket -task_id: "T010" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T010: Implement output rendering - -**Summary**: Create a simple text renderer that prints `SearchResponse` results in a human-readable format. - -**Owner**: backend - -**Estimate**: 20m - -**Dependencies**: T002, T009 - -**Target milestone**: M3 - -**Acceptance test**: A test provides a `SearchResponse` and the renderer returns a string containing all result titles and URLs. - -**Files/modules touched**: -- `src/cli/output.rs` - -**Steps**: -1. In `src/cli/output.rs`, define: - ```rust - use crate::domain::result::{SearchResponse, SearchResult}; - - pub fn render_text(response: &SearchResponse) -> String { - let mut lines = vec![ - format!("Provider: {}", response.provider), - format!("Query: {}", response.query), - format!("Results: {}", response.results.len()), - String::new(), - ]; - for (i, result) in response.results.iter().enumerate() { - match result { - SearchResult::Web(r) => { - lines.push(format!("{}. [{}]", i + 1, r.title)); - lines.push(format!(" URL: {}", r.url)); - if let Some(s) = &r.snippet { - lines.push(format!(" {}", s)); - } - } - SearchResult::News(r) => { - lines.push(format!("{}. [NEWS] {}", i + 1, r.title)); - lines.push(format!(" URL: {}", r.url)); - if let Some(s) = &r.source { - lines.push(format!(" Source: {}", s)); - } - } - SearchResult::Image(r) => { - lines.push(format!("{}. [IMAGE] {}", i + 1, r.title)); - lines.push(format!(" URL: {}", r.url)); - } - SearchResult::Video(r) => { - lines.push(format!("{}. [VIDEO] {}", i + 1, r.title)); - lines.push(format!(" URL: {}", r.url)); - } - } - lines.push(String::new()); - } - lines.join("\n") - } - ``` -2. Add a unit test that constructs a `SearchResponse` with mixed results and asserts the rendered string contains expected substrings. -3. Run `cargo test --lib`. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T011.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T011.md deleted file mode 100644 index 61bbb4c..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T011.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -title: "T011: Wire main.rs and run end-to-end" -type: plan_ticket -task_id: "T011" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T011: Wire main.rs and run end-to-end - -**Summary**: Connect CLI parsing, `SearchService`, `BraveProvider`, and output rendering in `main.rs` so the CLI performs a live search. - -**Owner**: backend - -**Estimate**: 25m - -**Dependencies**: T007, T008, T009, T010 - -**Target milestone**: M4 - -**Acceptance test**: `cargo run -- "rust programming"` successfully queries Brave and prints at least one search result. If API key is missing, it prints a clean error message. - -**Files/modules touched**: -- `src/main.rs` - -**Steps**: -1. In `src/main.rs`, replace contents with: - ```rust - mod app; - mod cli; - mod domain; - mod providers; - mod transport; - - use clap::Parser; - use cli::args::CliArgs; - use cli::output::render_text; - use app::search_service::SearchService; - use domain::query::SearchQuery; - use domain::types::SafeSearch; - use providers::brave::client::BraveProvider; - use providers::brave::config::BraveConfig; - use transport::http::ReqwestHttpClient; - - #[tokio::main] - async fn main() { - dotenvy::dotenv().ok(); - - let args = CliArgs::parse(); - - let config = match BraveConfig::from_env() { - Ok(c) => c, - Err(e) => { - eprintln!("Failed to load Brave config: {}", e); - std::process::exit(1); - } - }; - - let client = ReqwestHttpClient::new(); - let provider = BraveProvider::new(client, config); - let service = SearchService::new(Box::new(provider)); - - let query = SearchQuery { - text: args.query, - search_type: args.search_type.into(), - limit: args.limit, - offset: args.offset, - safe_search: args.safe_search, - country: args.country, - language: args.language, - time_range: None, - }; - - match service.search(query).await { - Ok(response) => { - println!("{}", render_text(&response)); - } - Err(e) => { - eprintln!("Search failed: {}", e); - std::process::exit(1); - } - } - } - ``` - Note: If `SafeSearch` does not implement `clap::ValueEnum`, adjust `CliArgs` to use a local `CliSafeSearch` enum and map it in the `SearchQuery` construction. -2. Ensure `cargo check` passes. -3. Run `cargo run -- "rust programming"` and verify it returns search results. -4. Run `cargo run -- "rust" --search-type news` and verify news results. -5. Run `cargo run -- "cats" --search-type images --limit 3` and verify image results. - ---- diff --git a/.artifacts/plan/2026-04-14_search-cli/tickets/T012.md b/.artifacts/plan/2026-04-14_search-cli/tickets/T012.md deleted file mode 100644 index 6921ebe..0000000 --- a/.artifacts/plan/2026-04-14_search-cli/tickets/T012.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: "T012: Add boundary tests and CI-ready verification" -type: plan_ticket -task_id: "T012" -parent_plan: "../PLAN.md" -created_at: "2026-04-14T17:44:55Z" -tags: [ticket, plan] ---- - -# T012: Add boundary tests and CI-ready verification - -**Summary**: Add integration-style tests at the app and provider boundaries, and ensure `cargo test` passes cleanly. - -**Owner**: backend - -**Estimate**: 30m - -**Dependencies**: T011 - -**Target milestone**: M4 - -**Acceptance test**: `cargo test` passes with at least one test for mapper, one for `BraveProvider` with mock HTTP, and one for `SearchService` with mock provider. - -**Files/modules touched**: -- `src/providers/brave/mapper.rs` -- `src/providers/brave/client.rs` -- `src/app/search_service.rs` - -**Steps**: -1. In `src/providers/brave/mapper.rs`, ensure the existing `#[cfg(test)]` module covers all four search types. -2. In `src/providers/brave/client.rs`, ensure the mock HTTP test validates that headers include `X-Subscription-Token` and that query params include `q`, `count`, and `safesearch`. -3. In `src/app/search_service.rs`, ensure the mock provider test validates delegation. -4. Run `cargo test` and fix any failures. -5. Run `cargo clippy` (if available) and fix warnings. - -## Risks & Mitigations - -| Risk | Mitigation | -|------|------------| -| Brave API schema differs from documented DTOs | Start with minimal fields; mapper defaults `Option` values to empty strings rather than failing | -| `clap` `ValueEnum` incompatibility with domain enums | Use separate CLI enums and map them explicitly to domain types | -| Rate limiting during manual testing | Keep query counts low; use `limit` flag; mock HTTP for automated tests | -| Missing env var in CI/test environments | `BraveConfig::from_env()` returns clear error; tests use mock config | - -## Test Strategy - -- **Mapper tests**: Construct minimal DTOs, assert domain output (T006). -- **Provider tests**: Mock `HttpClient` to return JSON, assert `BraveProvider` produces `SearchResponse` (T007). -- **Service tests**: Mock `SearchProvider`, assert `SearchService` delegates correctly (T008). -- **Renderer tests**: Provide `SearchResponse`, assert string output contains expected data (T010). -- **E2E manual**: Run CLI with live Brave API for web, news, images, and video queries (T011). - -## References - -- PRD.md sections 1–10 (domain design, provider trait, Brave mapping, module layout) -- `Cargo.toml` dependency versions aligned with PRD async/HTTP stack -- Brave Search API docs: https://api-dashboard.search.brave.com/documentation - -## Final Gate - -- **Output summary**: plan dir `.artifacts/plan/2026-04-14_search-cli/`, 4 milestones, 12 tickets -- **Next step**: proceed to execute-phase with `.artifacts/plan/2026-04-14_search-cli/PLAN.md` diff --git a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/PLAN.md b/.artifacts/plan/2026-04-15_16-22-21_exa-provider/PLAN.md deleted file mode 100644 index 63029e4..0000000 --- a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/PLAN.md +++ /dev/null @@ -1,311 +0,0 @@ ---- -title: "Exa provider implementation plan" -link: "exa-provider-implementation-plan" -type: implementation_plan -ontological_relations: - - relates_to: [[exa-provider-contract-map-research]] - - relates_to: [[provider-integration-map-research]] -tags: [plan, exa, provider, rust, coding] -uuid: "ac443d51-2b38-49bd-813b-debcb545248f" -created_at: "2026-04-15T21:23:23Z" -parent_research: ".artifacts/research/2026-04-15_21-19-47_exa-provider-contract-map.md" -git_commit_at_plan: "78610af" ---- - -## Goal - -Add Exa as a selectable search provider in the Rust CLI while keeping the current domain and application contracts unchanged. - -**Out of scope**: replacing Brave as the default provider, changing `SearchQuery` or `SearchResponse`, adding Exa-specific deep-search/output features, adding Exa image/video support, packaging or deployment work. - -## Scope & Assumptions - -**IN scope**: -- Extend the shared HTTP transport so providers can issue JSON `POST` requests. -- Add an `src/providers/exa/` adapter with config, DTOs, mapper, and provider client. -- Preserve the existing domain layer and `SearchService` trait boundary. -- Add CLI-level provider selection so Exa is reachable at runtime. -- Add focused regression tests for the new provider and the widened transport trait. - -**OUT of scope**: -- Any edits to `src/domain/` types or provider trait shape. -- Exa support for `SearchType::Images` or `SearchType::Videos`. -- Silent fallback behavior for unsupported Exa query features. -- New output formats or rich structured rendering. -- User-facing docs beyond minimal developer architecture notes. - -**Assumptions**: -- `EXA_API_KEY` will be supplied in the environment the same way `BRAVE_API_KEY` is today. -- Exa integration will use raw `reqwest` through the existing `HttpClient` abstraction rather than adding an Exa SDK dependency. -- Exa `moderation` is a boolean, so both `SafeSearch::Moderate` and `SafeSearch::Strict` will map to `true`. -- Exa does not expose request fields matching the current `offset` and `language` domain inputs in the referenced docs, so the adapter will reject those when `--provider exa` is selected instead of ignoring them. - -## Deliverables - -- `src/transport/http.rs` updated with shared JSON `POST` support. -- `src/providers/{mod.rs,exa/{mod.rs,config.rs,dto.rs,mapper.rs,client.rs}}` -- `src/cli/args.rs` updated with provider selection. -- `src/main.rs` updated to instantiate either Brave or Exa. -- `Cargo.toml` updated only if an ISO-8601 time helper crate is needed for Exa time-range translation. -- Targeted unit tests in the touched provider and transport modules. -- `docs/architecture.md` refreshed where the runtime provider-selection story changes. - -## Readiness - -- Repo is already bootstrapped and compiles before this work begins. -- Research artifacts exist at: - - `.artifacts/research/2026-04-15_21-10-50_provider-integration-map.md` - - `.artifacts/research/2026-04-15_21-19-47_exa-provider-contract-map.md` -- Current git plan baseline: - - commit: `78610af` - - working tree includes local doc/artifact changes (`AGENTS.md`, removed `PRD.md`, new `.artifacts/research/`) -- Execution requires environment access to both `BRAVE_API_KEY` and `EXA_API_KEY` for live manual smoke checks after implementation. - -## Milestones - -- **M1**: Shared transport and Exa scaffolding compile. -- **M2**: Exa request/response translation works for web and news searches. -- **M3**: CLI and binary composition can select Brave or Exa without touching domain/app layers. -- **M4**: Focused regression tests and architecture notes match the new provider surface. - -## Ticket Index - - - -| Task | Title | Ticket | -|---|---|---| -| T001 | Extend shared HTTP transport for JSON POST requests | [tickets/T001.md](tickets/T001.md) | -| T002 | Add Exa provider scaffolding, config, and serde DTOs | [tickets/T002.md](tickets/T002.md) | -| T003 | Map Exa responses into the existing domain result model | [tickets/T003.md](tickets/T003.md) | -| T004 | Implement the Exa provider client and query translation | [tickets/T004.md](tickets/T004.md) | -| T005 | Wire provider selection into the CLI and binary composition | [tickets/T005.md](tickets/T005.md) | -| T006 | Refresh regression coverage and architecture notes for multi-provider runtime | [tickets/T006.md](tickets/T006.md) | - - - -## Work Breakdown (Tasks) - -### T001: Extend shared HTTP transport for JSON POST requests - -**Summary**: Add JSON `POST` support to the existing `HttpClient` trait and reqwest adapter so provider implementations can call Exa without bypassing the transport layer. - -**Owner**: backend - -**Estimate**: 1h - -**Dependencies**: - -**Target milestone**: M1 - -**Acceptance test**: `cargo test transport::http::tests::test_post_json_decodes_success_response` - -**Files/modules touched**: -- `src/transport/http.rs` - -**Steps**: -1. Add a new async trait method to `HttpClient`, alongside `get_json`, for posting a JSON body and decoding the JSON response. -2. Keep the trait generic over the response type and request body type so provider adapters can reuse it without casting through `serde_json::Value`. -3. Refactor `ReqwestHttpClient` so GET and POST share a single private response-status handling path for auth, rate limit, non-2xx provider errors, and JSON decode errors. -4. Implement the new `post_json` method with reqwest `.post(url).json(&body)`, preserving caller-supplied headers. -5. Add a focused test in `src/transport/http.rs` that spins up a local `tokio::net::TcpListener`, returns a fixed JSON body, and verifies the new POST method decodes it successfully. - -### T002: Add Exa provider scaffolding, config, and serde DTOs - -**Summary**: Create the Exa provider module layout and define the request/response structs needed for the subset of the Exa API this CLI will consume. - -**Owner**: backend - -**Estimate**: 1.5h - -**Dependencies**: T001 - -**Target milestone**: M1 - -**Acceptance test**: `cargo test providers::exa::dto::tests::test_exa_search_response_deserializes_minimal_payload` - -**Files/modules touched**: -- `src/providers/mod.rs` -- `src/providers/exa/mod.rs` -- `src/providers/exa/config.rs` -- `src/providers/exa/dto.rs` - -**Steps**: -1. Export `pub mod exa;` from `src/providers/mod.rs` and create `src/providers/exa/mod.rs` with `pub mod client;`, `pub mod config;`, `pub mod dto;`, and `pub mod mapper;`. -2. Add `ExaConfig` in `src/providers/exa/config.rs` with `api_key` and `base_url`, and implement `from_env()` to read `EXA_API_KEY` and default `base_url` to `https://api.exa.ai`. -3. In `src/providers/exa/dto.rs`, define the request structs needed by the adapter using `#[serde(rename_all = "camelCase")]`: - - `ExaSearchRequest` - - `ExaContentsRequest` -4. In the same file, define only the response structs/fields the adapter will map: - - `ExaSearchResponse` - - `ExaResult` - - any small nested types required for fields consumed by the mapper -5. Keep unknown Exa fields ignored by omission rather than attempting to model the full API surface. -6. Add a DTO unit test that deserializes a minimal Exa search payload containing one result and asserts `request_id`, `search_type`, `published_date`, and `summary` are populated correctly. - -### T003: Map Exa responses into the existing domain result model - -**Summary**: Implement mapper functions that convert Exa search results into the current `SearchResponse` and `SearchResult` variants without changing shared domain types. - -**Owner**: backend - -**Estimate**: 1h - -**Dependencies**: T002 - -**Target milestone**: M2 - -**Acceptance test**: `cargo test providers::exa::mapper::tests::test_map_news_response_prefers_summary_and_preserves_author` - -**Files/modules touched**: -- `src/providers/exa/mapper.rs` - -**Steps**: -1. Add `map_web_response` and `map_news_response` in `src/providers/exa/mapper.rs`. -2. Set `SearchResponse.provider` to `"exa"` and `SearchResponse.query` from the original query string passed into the mapper, because the referenced Exa response contract does not expose the original query text. -3. For web results, map Exa `title`, `url`, and `summary`/`text` into `WebResult`, preferring `summary` and falling back to `text`. -4. For news results, map Exa `title`, `url`, `summary`/`text`, `author`, and `publishedDate` into `NewsResult`, using `author` as `source`. -5. Set `total_estimated` and `next_page` to `None`, matching the current normalized model when the provider does not expose those values in the referenced contract. -6. Add a mapper test that constructs a minimal Exa news DTO and proves the mapper uses `summary` before `text` and carries `author` into `source`. - -### T004: Implement the Exa provider client and query translation - -**Summary**: Add `ExaProvider` that translates the existing `SearchQuery` into Exa request bodies, advertises accurate capabilities, and fails explicitly on unsupported inputs. - -**Owner**: backend - -**Estimate**: 2h - -**Dependencies**: T001, T002, T003 - -**Target milestone**: M2 - -**Acceptance test**: `cargo test providers::exa::client::tests::test_exa_provider_news_search_posts_expected_payload` - -**Files/modules touched**: -- `Cargo.toml` -- `src/providers/exa/client.rs` - -**Steps**: -1. Add a narrow time-handling dependency in `Cargo.toml` only if needed to produce ISO-8601 UTC timestamps for Exa `startPublishedDate` and `endPublishedDate`. -2. Implement `ExaProvider` with `new(client, config)` and a `SearchProvider` implementation whose `id()` returns `"exa"`. -3. Set Exa capabilities to: - - `web = true` - - `news = true` - - `images = false` - - `videos = false` - - `pagination = false` - - `safe_search = true` - - `time_range_filter = true` -4. Translate the shared `SearchQuery` into Exa request fields as follows: - - `text` -> `query` - - `limit` -> `numResults` - - `search_type = Web` -> omit `category` - - `search_type = News` -> `category = "news"` - - `safe_search = Off` -> `moderation = false` - - `safe_search = Moderate | Strict` -> `moderation = true` - - `country` -> `userLocation` - - `time_range` -> `startPublishedDate` and `endPublishedDate` in ISO-8601 UTC - - request `contents.text = true` - - set request `type = "auto"` -5. Reject unsupported Exa query inputs with `SearchError::InvalidQuery` rather than silently ignoring them: - - `SearchType::Images` - - `SearchType::Videos` - - any non-`None` `offset` - - any non-`None` `language` -6. Send `POST {base_url}/search` with the `x-api-key` header through the new transport method and dispatch the decoded response through the Exa mapper selected by the original `SearchType`. -7. Add a mocked-provider test that asserts the request body contains `category: "news"`, `numResults`, `moderation`, and published-date window fields, then verifies the mapped response comes back with provider `"exa"`. - -### T005: Wire provider selection into the CLI and binary composition - -**Summary**: Make the provider selectable at runtime while preserving Brave as the default and leaving the domain/application layers untouched. - -**Owner**: backend - -**Estimate**: 1.5h - -**Dependencies**: T004 - -**Target milestone**: M3 - -**Acceptance test**: `cargo test cli::args::tests::test_cli_provider_parses_exa_and_defaults_to_brave` - -**Files/modules touched**: -- `src/cli/args.rs` -- `src/main.rs` - -**Steps**: -1. Add a `CliProvider` `ValueEnum` to `src/cli/args.rs` with `Brave` and `Exa`. -2. Add a `--provider` flag (short `-p`) to `CliArgs` with default value `brave`. -3. Keep the existing `SearchQuery` construction unchanged so the provider adapter remains responsible for validating unsupported inputs. -4. Update `src/main.rs` to import `ExaProvider` and `ExaConfig`, then match on `args.provider` to build the concrete provider instance: - - Brave branch loads `BraveConfig` and constructs `BraveProvider` - - Exa branch loads `ExaConfig` and constructs `ExaProvider` -5. Keep Brave as the default branch so existing invocations still behave the same when `--provider` is omitted. -6. Add a CLI parsing unit test proving `--provider exa` parses and that the default remains `Brave`. - -### T006: Refresh regression coverage and architecture notes for multi-provider runtime - -**Summary**: Update affected tests and architecture notes so the widened transport trait and multi-provider binary wiring are documented and enforced by focused regressions. - -**Owner**: backend - -**Estimate**: 1h - -**Dependencies**: T001, T004, T005 - -**Target milestone**: M4 - -**Acceptance test**: `cargo test providers::exa::client::tests::test_exa_provider_rejects_unsupported_query_fields` - -**Files/modules touched**: -- `src/providers/brave/client.rs` -- `src/providers/exa/client.rs` -- `docs/architecture.md` - -**Steps**: -1. Update the existing Brave provider test mock in `src/providers/brave/client.rs` so it implements the expanded `HttpClient` trait without changing Brave behavior. -2. Add a focused Exa provider regression test that proves unsupported query fields (`Images`, `Videos`, `offset`, or `language`) return `SearchError::InvalidQuery`. -3. Refresh `docs/architecture.md` so the provider-addition section distinguishes: - - unchanged domain/app boundaries - - provider-specific transport needs - - current binary wiring via CLI provider selection -4. Leave `tests/architecture_test.rs` unchanged unless the implementation introduces a real boundary violation; the current boundary rules should still hold. - -## Risks & Mitigations - -- Exa request contract drift: mitigate by implementing only the fields cited in the current Exa docs and keeping DTOs narrow. -- Safe-search fidelity loss: Exa exposes boolean moderation only, so `Moderate` and `Strict` collapse to the same provider value; document this in code comments near the mapping. -- Unsupported query-field confusion: reject unsupported Exa inputs explicitly instead of silently dropping them. -- Time-range serialization complexity: contain any new date/time dependency to the Exa client request translation path. -- Architecture doc drift: refresh the provider-selection section in `docs/architecture.md` as part of the same change set. - -## Test Strategy - -- Add one focused test per task only, centered on the newly introduced behavior. -- Prefer unit tests with mock transport over live API calls. -- Keep existing Brave tests compiling after the `HttpClient` trait expands. -- Defer live `cargo run -- --provider exa ...` smoke checks to execution/verification, not plan work. - -## References - -- `.artifacts/research/2026-04-15_21-19-47_exa-provider-contract-map.md` -- `.artifacts/research/2026-04-15_21-10-50_provider-integration-map.md` -- `src/domain/provider.rs:8` -- `src/domain/provider.rs:19` -- `src/transport/http.rs:5` -- `src/transport/http.rs:31` -- `src/providers/brave/client.rs:24` -- `src/providers/brave/client.rs:41` -- `src/main.rs:12` -- `src/main.rs:42` -- `src/main.rs:51` -- `src/cli/args.rs:36` -- `docs/architecture.md:151` -- `https://exa.ai/docs/reference/search` -- `https://exa.ai/docs/reference/search-api-guide` -- `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` - -## Final Gate - -- **Output summary**: plan dir path `.artifacts/plan/2026-04-15_16-22-21_exa-provider/`, milestone count `4`, ticket count `6` -- **Next step**: review this plan, then proceed to `grill-me` or `execute-phase` using `.artifacts/plan/2026-04-15_16-22-21_exa-provider/PLAN.md` diff --git a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/INDEX.md b/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/INDEX.md deleted file mode 100644 index c868e8f..0000000 --- a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/INDEX.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -title: "Ticket Index" -type: ticket_index -parent_plan: "../PLAN.md" -created_at: "2026-04-15T21:25:57Z" -tags: [ticket, plan] ---- - -# Ticket Index - -| Task | Title | Ticket | -|---|---|---| -| T001 | Extend shared HTTP transport for JSON POST requests | [T001](./T001.md) | -| T002 | Add Exa provider scaffolding, config, and serde DTOs | [T002](./T002.md) | -| T003 | Map Exa responses into the existing domain result model | [T003](./T003.md) | -| T004 | Implement the Exa provider client and query translation | [T004](./T004.md) | -| T005 | Wire provider selection into the CLI and binary composition | [T005](./T005.md) | -| T006 | Refresh regression coverage and architecture notes for multi-provider runtime | [T006](./T006.md) | diff --git a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T001.md b/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T001.md deleted file mode 100644 index abacc49..0000000 --- a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T001.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: "T001: Extend shared HTTP transport for JSON POST requests" -type: plan_ticket -task_id: "T001" -parent_plan: "../PLAN.md" -created_at: "2026-04-15T21:25:57Z" -tags: [ticket, plan] ---- - -# T001: Extend shared HTTP transport for JSON POST requests - -**Summary**: Add JSON `POST` support to the existing `HttpClient` trait and reqwest adapter so provider implementations can call Exa without bypassing the transport layer. - -**Owner**: backend - -**Estimate**: 1h - -**Dependencies**: - -**Target milestone**: M1 - -**Acceptance test**: `cargo test transport::http::tests::test_post_json_decodes_success_response` - -**Files/modules touched**: -- `src/transport/http.rs` - -**Steps**: -1. Add a new async trait method to `HttpClient`, alongside `get_json`, for posting a JSON body and decoding the JSON response. -2. Keep the trait generic over the response type and request body type so provider adapters can reuse it without casting through `serde_json::Value`. -3. Refactor `ReqwestHttpClient` so GET and POST share a single private response-status handling path for auth, rate limit, non-2xx provider errors, and JSON decode errors. -4. Implement the new `post_json` method with reqwest `.post(url).json(&body)`, preserving caller-supplied headers. -5. Add a focused test in `src/transport/http.rs` that spins up a local `tokio::net::TcpListener`, returns a fixed JSON body, and verifies the new POST method decodes it successfully. diff --git a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T002.md b/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T002.md deleted file mode 100644 index 4c9dac7..0000000 --- a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T002.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: "T002: Add Exa provider scaffolding, config, and serde DTOs" -type: plan_ticket -task_id: "T002" -parent_plan: "../PLAN.md" -created_at: "2026-04-15T21:25:57Z" -tags: [ticket, plan] ---- - -# T002: Add Exa provider scaffolding, config, and serde DTOs - -**Summary**: Create the Exa provider module layout and define the request/response structs needed for the subset of the Exa API this CLI will consume. - -**Owner**: backend - -**Estimate**: 1.5h - -**Dependencies**: T001 - -**Target milestone**: M1 - -**Acceptance test**: `cargo test providers::exa::dto::tests::test_exa_search_response_deserializes_minimal_payload` - -**Files/modules touched**: -- `src/providers/mod.rs` -- `src/providers/exa/mod.rs` -- `src/providers/exa/config.rs` -- `src/providers/exa/dto.rs` - -**Steps**: -1. Export `pub mod exa;` from `src/providers/mod.rs` and create `src/providers/exa/mod.rs` with `pub mod client;`, `pub mod config;`, `pub mod dto;`, and `pub mod mapper;`. -2. Add `ExaConfig` in `src/providers/exa/config.rs` with `api_key` and `base_url`, and implement `from_env()` to read `EXA_API_KEY` and default `base_url` to `https://api.exa.ai`. -3. In `src/providers/exa/dto.rs`, define the request structs needed by the adapter using `#[serde(rename_all = "camelCase")]`: - - `ExaSearchRequest` - - `ExaContentsRequest` -4. In the same file, define only the response structs/fields the adapter will map: - - `ExaSearchResponse` - - `ExaResult` - - any small nested types required for fields consumed by the mapper -5. Keep unknown Exa fields ignored by omission rather than attempting to model the full API surface. -6. Add a DTO unit test that deserializes a minimal Exa search payload containing one result and asserts `request_id`, `search_type`, `published_date`, and `summary` are populated correctly. diff --git a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T003.md b/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T003.md deleted file mode 100644 index 7c8cf17..0000000 --- a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T003.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: "T003: Map Exa responses into the existing domain result model" -type: plan_ticket -task_id: "T003" -parent_plan: "../PLAN.md" -created_at: "2026-04-15T21:25:57Z" -tags: [ticket, plan] ---- - -# T003: Map Exa responses into the existing domain result model - -**Summary**: Implement mapper functions that convert Exa search results into the current `SearchResponse` and `SearchResult` variants without changing shared domain types. - -**Owner**: backend - -**Estimate**: 1h - -**Dependencies**: T002 - -**Target milestone**: M2 - -**Acceptance test**: `cargo test providers::exa::mapper::tests::test_map_news_response_prefers_summary_and_preserves_author` - -**Files/modules touched**: -- `src/providers/exa/mapper.rs` - -**Steps**: -1. Add `map_web_response` and `map_news_response` in `src/providers/exa/mapper.rs`. -2. Set `SearchResponse.provider` to `"exa"` and `SearchResponse.query` from the original query string passed into the mapper, because the referenced Exa response contract does not expose the original query text. -3. For web results, map Exa `title`, `url`, and `summary`/`text` into `WebResult`, preferring `summary` and falling back to `text`. -4. For news results, map Exa `title`, `url`, `summary`/`text`, `author`, and `publishedDate` into `NewsResult`, using `author` as `source`. -5. Set `total_estimated` and `next_page` to `None`, matching the current normalized model when the provider does not expose those values in the referenced contract. -6. Add a mapper test that constructs a minimal Exa news DTO and proves the mapper uses `summary` before `text` and carries `author` into `source`. diff --git a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T004.md b/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T004.md deleted file mode 100644 index acece7b..0000000 --- a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T004.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -title: "T004: Implement the Exa provider client and query translation" -type: plan_ticket -task_id: "T004" -parent_plan: "../PLAN.md" -created_at: "2026-04-15T21:25:57Z" -tags: [ticket, plan] ---- - -# T004: Implement the Exa provider client and query translation - -**Summary**: Add `ExaProvider` that translates the existing `SearchQuery` into Exa request bodies, advertises accurate capabilities, and fails explicitly on unsupported inputs. - -**Owner**: backend - -**Estimate**: 2h - -**Dependencies**: T001, T002, T003 - -**Target milestone**: M2 - -**Acceptance test**: `cargo test providers::exa::client::tests::test_exa_provider_news_search_posts_expected_payload` - -**Files/modules touched**: -- `Cargo.toml` -- `src/providers/exa/client.rs` - -**Steps**: -1. Add a narrow time-handling dependency in `Cargo.toml` only if needed to produce ISO-8601 UTC timestamps for Exa `startPublishedDate` and `endPublishedDate`. -2. Implement `ExaProvider` with `new(client, config)` and a `SearchProvider` implementation whose `id()` returns `"exa"`. -3. Set Exa capabilities to: - - `web = true` - - `news = true` - - `images = false` - - `videos = false` - - `pagination = false` - - `safe_search = true` - - `time_range_filter = true` -4. Translate the shared `SearchQuery` into Exa request fields as follows: - - `text` -> `query` - - `limit` -> `numResults` - - `search_type = Web` -> omit `category` - - `search_type = News` -> `category = "news"` - - `safe_search = Off` -> `moderation = false` - - `safe_search = Moderate | Strict` -> `moderation = true` - - `country` -> `userLocation` - - `time_range` -> `startPublishedDate` and `endPublishedDate` in ISO-8601 UTC - - request `contents.text = true` - - set request `type = "auto"` -5. Reject unsupported Exa query inputs with `SearchError::InvalidQuery` rather than silently ignoring them: - - `SearchType::Images` - - `SearchType::Videos` - - any non-`None` `offset` - - any non-`None` `language` -6. Send `POST {base_url}/search` with the `x-api-key` header through the new transport method and dispatch the decoded response through the Exa mapper selected by the original `SearchType`. -7. Add a mocked-provider test that asserts the request body contains `category: "news"`, `numResults`, `moderation`, and published-date window fields, then verifies the mapped response comes back with provider `"exa"`. diff --git a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T005.md b/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T005.md deleted file mode 100644 index 0430b57..0000000 --- a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T005.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: "T005: Wire provider selection into the CLI and binary composition" -type: plan_ticket -task_id: "T005" -parent_plan: "../PLAN.md" -created_at: "2026-04-15T21:25:57Z" -tags: [ticket, plan] ---- - -# T005: Wire provider selection into the CLI and binary composition - -**Summary**: Make the provider selectable at runtime while preserving Brave as the default and leaving the domain/application layers untouched. - -**Owner**: backend - -**Estimate**: 1.5h - -**Dependencies**: T004 - -**Target milestone**: M3 - -**Acceptance test**: `cargo test cli::args::tests::test_cli_provider_parses_exa_and_defaults_to_brave` - -**Files/modules touched**: -- `src/cli/args.rs` -- `src/main.rs` - -**Steps**: -1. Add a `CliProvider` `ValueEnum` to `src/cli/args.rs` with `Brave` and `Exa`. -2. Add a `--provider` flag (short `-p`) to `CliArgs` with default value `brave`. -3. Keep the existing `SearchQuery` construction unchanged so the provider adapter remains responsible for validating unsupported inputs. -4. Update `src/main.rs` to import `ExaProvider` and `ExaConfig`, then match on `args.provider` to build the concrete provider instance: - - Brave branch loads `BraveConfig` and constructs `BraveProvider` - - Exa branch loads `ExaConfig` and constructs `ExaProvider` -5. Keep Brave as the default branch so existing invocations still behave the same when `--provider` is omitted. -6. Add a CLI parsing unit test proving `--provider exa` parses and that the default remains `Brave`. diff --git a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T006.md b/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T006.md deleted file mode 100644 index c90b9c6..0000000 --- a/.artifacts/plan/2026-04-15_16-22-21_exa-provider/tickets/T006.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: "T006: Refresh regression coverage and architecture notes for multi-provider runtime" -type: plan_ticket -task_id: "T006" -parent_plan: "../PLAN.md" -created_at: "2026-04-15T21:25:57Z" -tags: [ticket, plan] ---- - -# T006: Refresh regression coverage and architecture notes for multi-provider runtime - -**Summary**: Update affected tests and architecture notes so the widened transport trait and multi-provider binary wiring are documented and enforced by focused regressions. - -**Owner**: backend - -**Estimate**: 1h - -**Dependencies**: T001, T004, T005 - -**Target milestone**: M4 - -**Acceptance test**: `cargo test providers::exa::client::tests::test_exa_provider_rejects_unsupported_query_fields` - -**Files/modules touched**: -- `src/providers/brave/client.rs` -- `src/providers/exa/client.rs` -- `docs/architecture.md` - -**Steps**: -1. Update the existing Brave provider test mock in `src/providers/brave/client.rs` so it implements the expanded `HttpClient` trait without changing Brave behavior. -2. Add a focused Exa provider regression test that proves unsupported query fields (`Images`, `Videos`, `offset`, or `language`) return `SearchError::InvalidQuery`. -3. Refresh `docs/architecture.md` so the provider-addition section distinguishes: - - unchanged domain/app boundaries - - provider-specific transport needs - - current binary wiring via CLI provider selection -4. Leave `tests/architecture_test.rs` unchanged unless the implementation introduces a real boundary violation; the current boundary rules should still hold. - -## Risks & Mitigations - -- Exa request contract drift: mitigate by implementing only the fields cited in the current Exa docs and keeping DTOs narrow. -- Safe-search fidelity loss: Exa exposes boolean moderation only, so `Moderate` and `Strict` collapse to the same provider value; document this in code comments near the mapping. -- Unsupported query-field confusion: reject unsupported Exa inputs explicitly instead of silently dropping them. -- Time-range serialization complexity: contain any new date/time dependency to the Exa client request translation path. -- Architecture doc drift: refresh the provider-selection section in `docs/architecture.md` as part of the same change set. - -## Test Strategy - -- Add one focused test per task only, centered on the newly introduced behavior. -- Prefer unit tests with mock transport over live API calls. -- Keep existing Brave tests compiling after the `HttpClient` trait expands. -- Defer live `cargo run -- --provider exa ...` smoke checks to execution/verification, not plan work. - -## References - -- `.artifacts/research/2026-04-15_21-19-47_exa-provider-contract-map.md` -- `.artifacts/research/2026-04-15_21-10-50_provider-integration-map.md` -- `src/domain/provider.rs:8` -- `src/domain/provider.rs:19` -- `src/transport/http.rs:5` -- `src/transport/http.rs:31` -- `src/providers/brave/client.rs:24` -- `src/providers/brave/client.rs:41` -- `src/main.rs:12` -- `src/main.rs:42` -- `src/main.rs:51` -- `src/cli/args.rs:36` -- `docs/architecture.md:151` -- `https://exa.ai/docs/reference/search` -- `https://exa.ai/docs/reference/search-api-guide` -- `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` - -## Final Gate - -- **Output summary**: plan dir path `.artifacts/plan/2026-04-15_16-22-21_exa-provider/`, milestone count `4`, ticket count `6` -- **Next step**: review this plan, then proceed to `grill-me` or `execute-phase` using `.artifacts/plan/2026-04-15_16-22-21_exa-provider/PLAN.md` diff --git a/.artifacts/research/2026-04-15_21-10-50_provider-integration-map.md b/.artifacts/research/2026-04-15_21-10-50_provider-integration-map.md deleted file mode 100644 index ee44cd3..0000000 --- a/.artifacts/research/2026-04-15_21-10-50_provider-integration-map.md +++ /dev/null @@ -1,125 +0,0 @@ ---- -title: "provider integration map research findings" -link: "provider-integration-map-research" -type: research -ontological_relations: - - relates_to: [[search-cli-plan]] -tags: [research, provider, rust, search-cli] -uuid: "d328abd9-9e8f-40a1-af87-65be2ac78dd3" -created_at: "2026-04-15T21:10:50Z" ---- - -## Structure -- `src/domain/` contains provider-agnostic query, result, error, and trait types. -- `src/transport/` contains the HTTP abstraction (`HttpClient`) and the reqwest adapter. -- `src/providers/brave/` contains the Brave adapter split into config, DTO, mapper, and client modules. -- `src/app/` contains `SearchService`, which stores `Box`. -- `src/cli/` contains CLI argument parsing and text rendering. -- `src/main.rs` composes the concrete provider and converts CLI args into a `SearchQuery`. - -## Key Files -- `src/domain/provider.rs:6` defines `ProviderCapabilities`. -- `src/domain/provider.rs:19` defines the `SearchProvider` trait with `id`, `capabilities`, and `search`. -- `src/domain/query.rs:4` defines `SearchQuery`. -- `src/domain/result.rs:5` defines `SearchResponse` with the provider identifier recorded as `String`. -- `src/domain/types.rs:2` defines supported `SearchType` variants: `Web`, `News`, `Images`, `Videos`. -- `src/transport/http.rs:5` defines the `HttpClient` trait used by provider clients. -- `src/transport/http.rs:18` defines `ReqwestHttpClient`. -- `src/providers/mod.rs:1` exports only the `brave` provider module. -- `src/providers/brave/config.rs:2` defines `BraveConfig`. -- `src/providers/brave/config.rs:8` loads configuration from `BRAVE_API_KEY`. -- `src/providers/brave/dto.rs:4` defines `BraveWebResponse`. -- `src/providers/brave/dto.rs:30` defines `BraveNewsResponse`. -- `src/providers/brave/dto.rs:51` defines `BraveImagesResponse`. -- `src/providers/brave/dto.rs:71` defines `BraveVideosResponse`. -- `src/providers/brave/mapper.rs:4` maps `BraveWebResponse` into `SearchResponse`. -- `src/providers/brave/mapper.rs:27` maps `BraveNewsResponse` into `SearchResponse`. -- `src/providers/brave/mapper.rs:50` maps `BraveImagesResponse` into `SearchResponse`. -- `src/providers/brave/mapper.rs:72` maps `BraveVideosResponse` into `SearchResponse`. -- `src/providers/brave/client.rs:12` defines `BraveProvider`. -- `src/providers/brave/client.rs:24` implements `SearchProvider` for `BraveProvider`. -- `src/app/search_service.rs:6` defines `SearchService`. -- `src/app/search_service.rs:11` constructs `SearchService` with `Box`. -- `src/main.rs:12` imports `BraveProvider`. -- `src/main.rs:13` imports `BraveConfig`. -- `src/main.rs:42` loads Brave config. -- `src/main.rs:51` constructs the concrete `BraveProvider`. -- `src/main.rs:52` injects the provider into `SearchService`. -- `src/cli/args.rs:40` defines `CliArgs`. -- `tests/architecture_test.rs:25` enforces that provider modules do not import `cli` or `app`. - -## Patterns Found -- Provider abstraction: - - `src/domain/provider.rs:19` - - `src/app/search_service.rs:7` - - `src/app/search_service.rs:11` - - `src/app/search_service.rs:32` - - `src/providers/brave/client.rs:24` -- Provider-specific DTO to domain mapping: - - `src/providers/brave/mapper.rs:4` - - `src/providers/brave/mapper.rs:27` - - `src/providers/brave/mapper.rs:50` - - `src/providers/brave/mapper.rs:72` -- Provider-specific runtime composition in the binary: - - `src/main.rs:12` - - `src/main.rs:13` - - `src/main.rs:42` - - `src/main.rs:51` - - `src/main.rs:52` -- Provider identifier stored in normalized output: - - `src/domain/result.rs:7` - - `src/providers/brave/mapper.rs:10` - - `src/providers/brave/mapper.rs:32` - - `src/providers/brave/mapper.rs:55` - - `src/providers/brave/mapper.rs:77` - - `src/cli/output.rs:5` - -## Dependencies -- `src/main.rs` imports: - - `src/app/search_service.rs` - - `src/cli/args.rs` - - `src/cli/output.rs` - - `src/domain/query.rs` - - `src/providers/brave/client.rs` - - `src/providers/brave/config.rs` - - `src/transport/http.rs` -- `src/app/search_service.rs` imports: - - `src/domain/error.rs` - - `src/domain/provider.rs` - - `src/domain/query.rs` - - `src/domain/result.rs` -- `src/providers/brave/client.rs` imports: - - `src/domain/error.rs` - - `src/domain/provider.rs` - - `src/domain/query.rs` - - `src/domain/result.rs` - - `src/domain/types.rs` - - `src/providers/brave/config.rs` - - `src/providers/brave/dto.rs` - - `src/providers/brave/mapper.rs` - - `src/transport/http.rs` -- `src/providers/brave/mapper.rs` imports: - - `src/domain/result.rs` - - `src/providers/brave/dto.rs` -- `src/transport/http.rs` imports: - - `src/domain/error.rs` - -## Provider Addition Surface -- New provider modules belong under `src/providers/`, based on the existing `src/providers/brave/` layout and `src/providers/mod.rs:1`. -- The required trait implementation boundary is `src/domain/provider.rs:19`. -- The application layer already accepts any boxed provider at `src/app/search_service.rs:7` and `src/app/search_service.rs:11`. -- The current binary selects Brave directly in `src/main.rs:42`, `src/main.rs:51`, and `src/main.rs:52`. -- The current CLI arguments do not include a provider selector; `src/cli/args.rs:43-66` contains query, about flag, search type, limit, offset, safe search, country, and language. - -## Tests And Enforcement -- `src/providers/brave/mapper.rs:95` contains mapper unit tests for all four search types. -- `src/providers/brave/client.rs:110` contains a provider test using a mocked `HttpClient`. -- `src/app/search_service.rs:20` contains a service test using a mocked `SearchProvider`. -- `tests/architecture_test.rs:25` forbids provider imports from `crate::cli::` and `crate::app::`. -- `justfile:2` defines `just check`, which runs format, clippy, tests, and mdBook build. - -## Operator Artifacts -- `docs/architecture.md:151` documents provider addition as creating a new provider module, implementing `SearchProvider`, and mapping DTOs into `SearchResponse`. -- `docs/architecture.md:155` states that no changes to `domain`, `app`, or `cli` are required for a new provider. -- `src/main.rs:12-13` and `src/main.rs:42-52` show Brave-specific composition in the binary. -- `PRD.md` was not present in the repository root during this scan. diff --git a/.artifacts/research/2026-04-15_21-19-47_exa-provider-contract-map.md b/.artifacts/research/2026-04-15_21-19-47_exa-provider-contract-map.md deleted file mode 100644 index 7d938a5..0000000 --- a/.artifacts/research/2026-04-15_21-19-47_exa-provider-contract-map.md +++ /dev/null @@ -1,203 +0,0 @@ ---- -title: "exa provider contract map research findings" -link: "exa-provider-contract-map-research" -type: research -ontological_relations: - - relates_to: [[search-cli-plan]] -tags: [research, exa, provider, rust, search-cli] -uuid: "8002473b-28c6-40a7-8ac3-3bf9b7c9480c" -created_at: "2026-04-15T21:19:47Z" ---- - -## Structure -- `src/domain/` contains the provider-agnostic query, result, type, and provider trait definitions. -- `src/transport/` contains the HTTP abstraction and reqwest adapter. -- `src/providers/brave/` contains the only current provider implementation. -- `src/main.rs` constructs `BraveProvider` directly and injects it into `SearchService`. -- `src/cli/args.rs` exposes query, search type, limit, offset, safe search, country, and language flags. -- Exa documentation referenced in this scan: - - `https://exa.ai/docs/reference/search-api-guide` - - `https://exa.ai/docs/reference/search` - - `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` - -## Key Files -- `src/domain/types.rs:2` defines `SearchType::{Web, News, Images, Videos}`. -- `src/domain/types.rs:10` defines `SafeSearch::{Off, Moderate, Strict}`. -- `src/domain/types.rs:18` defines `TimeRange::{Day, Week, Month, Year}`. -- `src/domain/query.rs:4` defines `SearchQuery { text, search_type, limit, offset, safe_search, country, language, time_range }`. -- `src/domain/result.rs:5` defines `SearchResponse { query, provider, results, total_estimated, next_page }`. -- `src/domain/result.rs:14` defines `SearchResult::{Web, News, Image, Video}`. -- `src/domain/provider.rs:8` defines `ProviderCapabilities { web, news, images, videos, pagination, safe_search, time_range_filter }`. -- `src/domain/provider.rs:19` defines the `SearchProvider` trait. -- `src/transport/http.rs:5` defines `HttpClient::get_json`. -- `src/transport/http.rs:18` defines `ReqwestHttpClient`. -- `src/providers/mod.rs:1` exports only `brave`. -- `src/providers/brave/config.rs:2` defines `BraveConfig`. -- `src/providers/brave/client.rs:12` defines `BraveProvider`. -- `src/providers/brave/client.rs:24` implements `SearchProvider` for `BraveProvider`. -- `src/providers/brave/mapper.rs:4` maps Brave web DTOs into `SearchResponse`. -- `src/providers/brave/mapper.rs:27` maps Brave news DTOs into `SearchResponse`. -- `src/providers/brave/mapper.rs:50` maps Brave image DTOs into `SearchResponse`. -- `src/providers/brave/mapper.rs:72` maps Brave video DTOs into `SearchResponse`. -- `src/cli/args.rs:43` defines `CliArgs`. -- `src/main.rs:12` imports `BraveProvider`. -- `src/main.rs:13` imports `BraveConfig`. -- `src/main.rs:42` loads Brave config from env. -- `src/main.rs:50` constructs `ReqwestHttpClient`. -- `src/main.rs:51` constructs `BraveProvider`. -- `src/main.rs:52` injects the provider into `SearchService`. - -## Patterns Found -- Shared provider contract: - - `src/domain/provider.rs:19` - - `src/app/search_service.rs:7` - - `src/app/search_service.rs:11` - - `src/providers/brave/client.rs:24` -- Query-to-provider request translation: - - `src/providers/brave/client.rs:41` - - `src/providers/brave/client.rs:50` - - `src/providers/brave/client.rs:57` - - `src/providers/brave/client.rs:65` - - `src/providers/brave/client.rs:68` - - `src/providers/brave/client.rs:71` -- Provider DTO-to-domain result mapping: - - `src/providers/brave/mapper.rs:4` - - `src/providers/brave/mapper.rs:27` - - `src/providers/brave/mapper.rs:50` - - `src/providers/brave/mapper.rs:72` -- Provider selection in the binary: - - `src/main.rs:42` - - `src/main.rs:51` - - `src/main.rs:52` - -## Dependencies -- `src/main.rs` imports: - - `src/app/search_service.rs` - - `src/cli/args.rs` - - `src/cli/output.rs` - - `src/domain/query.rs` - - `src/providers/brave/client.rs` - - `src/providers/brave/config.rs` - - `src/transport/http.rs` -- `src/providers/brave/client.rs` imports: - - `src/domain/error.rs` - - `src/domain/provider.rs` - - `src/domain/query.rs` - - `src/domain/result.rs` - - `src/domain/types.rs` - - `src/providers/brave/config.rs` - - `src/providers/brave/dto.rs` - - `src/providers/brave/mapper.rs` - - `src/transport/http.rs` -- `src/providers/brave/mapper.rs` imports: - - `src/domain/result.rs` - - `src/providers/brave/dto.rs` - -## Current Shared Contract Surface -- `SearchQuery.search_type` uses the shared enum in `src/domain/types.rs:2`. -- `SearchQuery.limit` and `SearchQuery.offset` are optional numeric fields in `src/domain/query.rs:7-8`. -- `SearchQuery.safe_search`, `country`, `language`, and `time_range` are optional fields in `src/domain/query.rs:9-12`. -- `SearchResponse.results` is a vector of `SearchResult` variants in `src/domain/result.rs:8` and `src/domain/result.rs:14-18`. -- `SearchResponse.total_estimated` is optional in `src/domain/result.rs:9`. -- `SearchResponse.next_page` is optional in `src/domain/result.rs:10`. - -## Exa Request Fields Observed -- Exa search endpoint: `POST https://api.exa.ai/search`. - - Source: `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` -- Exa authentication header: `x-api-key`. - - Source: `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` -- Request parameters listed in the Exa coding-agent reference: - - `query` - - `type` - - `stream` - - `numResults` - - `category` - - `userLocation` - - `includeDomains` - - `excludeDomains` - - `startPublishedDate` - - `endPublishedDate` - - `startCrawlDate` - - `endCrawlDate` - - `moderation` - - `additionalQueries` - - `systemPrompt` - - `outputSchema` - - Source: `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` -- Exa contents parameters listed in the Exa coding-agent reference: - - `contents.text` - - `contents.highlights` - - `contents.summary` - - `contents.livecrawlTimeout` - - `contents.maxAgeHours` - - `contents.subpages` - - `contents.subpageTarget` - - `contents.extras.links` - - `contents.extras.imageLinks` - - Source: `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` -- Exa search types listed in the Exa coding-agent reference: - - `auto` - - `fast` - - `instant` - - `deep-lite` - - `deep` - - `deep-reasoning` - - Source: `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` -- Exa categories listed in the Exa coding-agent reference: - - `company` - - `people` - - `research paper` - - `news` - - `personal site` - - `financial report` - - Source: `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` - -## Exa Response Fields Observed -- Exa response top-level fields listed in the Exa coding-agent reference: - - `requestId` - - `searchType` - - `results` - - `output` - - `costDollars` - - Source: `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` -- Exa result object fields listed in the Exa coding-agent reference: - - `title` - - `url` - - `id` - - `publishedDate` - - `author` - - `image` - - `favicon` - - `text` - - `highlights` - - `highlightScores` - - `summary` - - `subpages` - - `extras.links` - - Source: `https://exa.ai/docs/reference/search-api-guide-for-coding-agents` - -## Repo-to-Exa Field Presence Map -- `src/domain/query.rs:5` defines `text`; Exa request docs list `query`. -- `src/domain/query.rs:6` defines `search_type`; Exa request docs list `type` and `category`. -- `src/domain/query.rs:7` defines `limit`; Exa request docs list `numResults`. -- `src/domain/query.rs:8` defines `offset`; no Exa request field named `offset` appears in the referenced Exa docs. -- `src/domain/query.rs:9` defines `safe_search`; Exa request docs list `moderation`. -- `src/domain/query.rs:10` defines `country`; Exa request docs list `userLocation`. -- `src/domain/query.rs:11` defines `language`; no Exa request field named `language` appears in the referenced Exa docs. -- `src/domain/query.rs:12` defines `time_range`; Exa request docs list `startPublishedDate`, `endPublishedDate`, `startCrawlDate`, and `endCrawlDate`. -- `src/domain/result.rs:22-27` defines `WebResult`. -- `src/domain/result.rs:30-36` defines `NewsResult`. -- `src/domain/result.rs:39-44` defines `ImageResult`. -- `src/domain/result.rs:47-53` defines `VideoResult`. -- Exa result docs list `title`, `url`, `publishedDate`, `author`, `image`, `favicon`, `text`, `highlights`, `summary`, `subpages`, and `extras.links`. - -## Current Transport Surface Compared To Exa Docs -- `src/transport/http.rs:6-10` defines `HttpClient::get_json` with `url`, `headers`, and query parameters. -- `src/transport/http.rs:41` builds a reqwest `GET` request. -- Exa docs referenced in this scan describe `POST /search` with a JSON request body. - -## Existing Provider Composition Surface -- `src/providers/mod.rs:1` exposes only the `brave` module. -- `src/main.rs:12-13` imports Brave-specific provider types. -- `src/main.rs:42-52` constructs Brave-specific runtime configuration and provider instances. -- `src/cli/args.rs:50-66` defines CLI flags for `search_type`, `limit`, `offset`, `safe_search`, `country`, and `language`. diff --git a/.cargo-husky/hooks/pre-commit b/.cargo-husky/hooks/pre-commit new file mode 100755 index 0000000..9d98b80 --- /dev/null +++ b/.cargo-husky/hooks/pre-commit @@ -0,0 +1,11 @@ +#!/bin/sh +# +# Pre-commit hook installed by cargo-husky. +# Runs fast quality checks before every commit. +set -e + +echo "[pre-commit] Running cargo fmt --check" +cargo fmt --check + +echo "[pre-commit] Running cargo clippy" +cargo clippy -- -D warnings -W clippy::complexity -W clippy::cognitive_complexity diff --git a/.cargo-husky/hooks/pre-push b/.cargo-husky/hooks/pre-push new file mode 100755 index 0000000..bfdcfa9 --- /dev/null +++ b/.cargo-husky/hooks/pre-push @@ -0,0 +1,5 @@ +#!/bin/sh +set -e + +echo "+ just check" +just check diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..6795ba0 --- /dev/null +++ b/.env.example @@ -0,0 +1,7 @@ +# Brave Search API key +# Get yours at: https://api.search.brave.com/ +BRAVE_API_KEY= + +# Exa API key +# Get yours at: https://exa.ai/ +EXA_API_KEY= diff --git a/.factory/skills/sophon-cli/SKILL.md b/.factory/skills/sophon-cli/SKILL.md new file mode 100644 index 0000000..2b44696 --- /dev/null +++ b/.factory/skills/sophon-cli/SKILL.md @@ -0,0 +1,55 @@ +--- +name: sophon-cli +description: Rust CLI for provider-agnostic web search using Brave Search or Exa APIs + +title: "sophon-cli Agent Skill" +when_to_read: + - "When using or maintaining the repository-specific Codex skill for sophon-cli." + - "When aligning local agent behavior with this project’s commands and boundaries." +summary: "Repository-specific Codex skill instructions for sophon-cli, capturing project commands, validation expectations, and operational guardrails." +ontology_relations: + - relation: "guides" + target: "agent-workflows" + note: "Defines agent behavior for this repository." +--- + +# sophon-cli Agent Skill + +## Project +- Rust CLI binary (`sophon-cli`) with a provider-agnostic domain layer +- Brave Search and Exa adapters behind a trait boundary + +## Key Files +- `src/main.rs` — entrypoint wiring CLI args, app service, providers, and HTTP transport +- `src/domain/` — pure types and traits (no HTTP, no CLI parsing) +- `src/providers/brave/` — Brave-specific DTOs, mapper, config, and client +- `src/providers/exa/` — Exa-specific DTOs, mapper, config, and client +- `src/transport/` — `HttpClient` trait and `ReqwestHttpClient` adapter +- `src/app/` — `SearchService` orchestrator +- `src/cli/` — `clap` argument parsing and text rendering +- `tests/architecture_test.rs` — source-scan tests enforcing layer boundaries + +## Quality Gate +Run `just check` before committing. This runs: +1. `cargo fmt --check` +2. `cargo clippy -- -D warnings -W clippy::complexity -W clippy::cognitive_complexity` +3. `cargo test` +4. `mdbook build` + +## Architecture Boundaries +- `src/domain/` must NOT import `crate::providers`, `crate::transport`, `crate::cli`, `crate::app` +- `src/transport/` must NOT import `crate::providers`, `crate::cli`, `crate::app` +- `src/providers/` must NOT import `crate::cli`, `crate::app` +- `src/app/` must NOT import `crate::cli` +- Only `src/cli/` may import `render_text` + +## Environment Setup +Copy `.env.example` to `.env` and fill in the API key for the provider you want to use: +- `BRAVE_API_KEY` for Brave Search +- `EXA_API_KEY` for Exa + +## Common Tasks +- Add a new provider: create `src/providers//` with `config.rs`, `dto.rs`, `mapper.rs`, `client.rs`, then register in `src/providers/mod.rs` +- Add a domain type: place in `src/domain/` and update `src/domain/mod.rs` +- Modify CLI output: edit `src/cli/output.rs` and add unit tests there +- Update docs: edit files in `docs/` and run `mdbook build` to verify diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..6f39255 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Global fallback +* @alchemiststudiosDOTai diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..2ceacfb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Report a bug or unexpected behavior +title: "[BUG] " +labels: ["bug", "triage"] +assignees: "" + +when_to_read: + - "When filing or maintaining GitHub issue templates for this repository." + - "When changing the triage questions expected from contributors." +summary: "GitHub issue template used to collect structured contributor input for Bug Report." +ontology_relations: + - relation: "supports" + target: "github-issue-triage" + note: "Shapes incoming issue reports for maintainers." +--- + +## Describe the bug + + + +## To Reproduce + +1. Command or code snippet that triggers the issue +2. Environment (OS, Rust version, provider used) +3. Error output or unexpected behavior + +## Expected behavior + + + +## Screenshots / Logs + + + +## Additional context + + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..45156f2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,32 @@ +--- +name: Feature request +about: Suggest an enhancement or new provider +title: "[FEATURE] " +labels: ["feature", "triage"] +assignees: "" + +when_to_read: + - "When filing or maintaining GitHub issue templates for this repository." + - "When changing the triage questions expected from contributors." +summary: "GitHub issue template used to collect structured contributor input for Feature Request." +ontology_relations: + - relation: "supports" + target: "github-issue-triage" + note: "Shapes incoming issue reports for maintainers." +--- + +## Problem / Motivation + + + +## Proposed Solution + + + +## Alternatives Considered + + + +## Additional Context + + diff --git a/.github/labels.yml b/.github/labels.yml new file mode 100644 index 0000000..81be7e0 --- /dev/null +++ b/.github/labels.yml @@ -0,0 +1,44 @@ +# Label definitions for sophon-cli +# These labels are used for priority, type, and area classification. +- name: "P0-critical" + color: "b60205" + description: "Critical - blocks release or main workflow" +- name: "P1-high" + color: "d93f0b" + description: "High priority - significant impact" +- name: "P2-medium" + color: "fbca04" + description: "Medium priority - should be addressed soon" +- name: "P3-low" + color: "0e8a16" + description: "Low priority - nice to have" +- name: "bug" + color: "d73a4a" + description: "Something is broken" +- name: "feature" + color: "a2eeef" + description: "New capability or enhancement" +- name: "chore" + color: "c2e0c6" + description: "Maintenance, tooling, or cleanup" +- name: "docs" + color: "0075ca" + description: "Documentation related" +- name: "area/cli" + color: "0052cc" + description: "Command-line interface layer" +- name: "area/domain" + color: "0052cc" + description: "Provider-agnostic domain types" +- name: "area/providers" + color: "0052cc" + description: "Search provider adapters" +- name: "area/transport" + color: "0052cc" + description: "HTTP client and transport" +- name: "area/tests" + color: "0052cc" + description: "Test suite and harness" +- name: "triage" + color: "cccccc" + description: "Needs initial review and routing" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..4e8bbca --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,33 @@ +--- +title: "Pull Request Template" +when_to_read: + - "When opening or maintaining pull request guidance for contributors." + - "When changing review, validation, or documentation expectations for proposed changes." +summary: "GitHub pull request template that prompts contributors for change scope, validation evidence, and reviewer context." +ontology_relations: + - relation: "supports" + target: "github-pr-review" + note: "Guides pull request authors and reviewers." +--- + +## Description + + + +## Testing Done + + +- [ ] `just check` passes locally (fmt, clippy, tests, mdbook build) +- [ ] Unit tests added or updated for new behavior +- [ ] Architecture boundary tests still pass (`cargo test`) + +## Type of Change + +- [ ] Bug fix +- [ ] New feature +- [ ] Refactor / internal change +- [ ] Documentation update + +## Relevant Context + + diff --git a/.github/workflows/validate-agents.yml b/.github/workflows/validate-agents.yml new file mode 100644 index 0000000..74e0194 --- /dev/null +++ b/.github/workflows/validate-agents.yml @@ -0,0 +1,84 @@ +name: Validate AGENTS.md + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + validate-agents: + name: Validate AGENTS.md Consistency + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Verify AGENTS.md referenced paths exist + run: | + set -euo pipefail + echo "Checking file paths referenced in AGENTS.md..." + + # Extract file paths from AGENTS.md and check they exist + missing=0 + + # Check explicit paths in Repository Map + paths=( + "src/main.rs" + "src/domain/query.rs" + "src/domain/result.rs" + "src/domain/types.rs" + "src/domain/error.rs" + "src/domain/provider.rs" + "src/providers/brave" + "src/transport" + "src/app" + "src/cli/args.rs" + "src/cli/output.rs" + "tests/architecture_test.rs" + "docs" + "README.md" + "justfile" + "Cargo.toml" + "HARNESS.md" + ) + + for path in "${paths[@]}"; do + if [ ! -e "$path" ]; then + echo "MISSING: $path" + missing=$((missing + 1)) + else + echo "OK: $path" + fi + done + + if [ "$missing" -gt 0 ]; then + echo "Error: $missing referenced path(s) missing." + exit 1 + fi + + echo "All referenced paths exist." + + - name: Setup Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Install just, mdbook, and cargo-udeps + uses: taiki-e/install-action@v2 + with: + tool: just,mdbook,cargo-udeps + + - name: Verify canonical command works + run: just check + + - name: Install nightly toolchain for cargo-udeps + uses: dtolnay/rust-toolchain@nightly + + - name: Setup Node.js for jscpd + uses: actions/setup-node@v4 + with: + node-version: '22' + + - name: Install ripgrep for tech debt checks + run: sudo apt-get update && sudo apt-get install -y ripgrep + + - name: Run hygiene checks + run: just hygiene diff --git a/.gitignore b/.gitignore index 8aa3e25..cd82032 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env +.artifacts/ # Added by cargo @@ -6,3 +7,18 @@ /target book/ +# Taskplane runtime artifacts (machine-specific, do not commit) +.pi/batch-state.json +.pi/batch-history.json +.pi/lane-state-* +.pi/merge-result-* +.pi/merge-request-* +.pi/worker-conversation-* +.pi/orch-logs/ +.pi/orch-abort-signal +.pi/settings.json +.worktrees/ +.taskplane-tasks/ + +# Pi project-local packages (if using pi install -l) +.pi/npm/ diff --git a/.jscpd.json b/.jscpd.json new file mode 100644 index 0000000..a2ae8f4 --- /dev/null +++ b/.jscpd.json @@ -0,0 +1,10 @@ +{ + "threshold": 3, + "minLines": 5, + "minTokens": 50, + "reporters": ["consoleFull"], + "absolute": false, + "path": ["src", "tests"], + "pattern": "**/*.rs", + "ignore": ["**/target/**"] +} diff --git a/AGENTS.md b/AGENTS.md index 5015f2b..341ccf2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,17 +5,16 @@ - Provider-agnostic domain layer with a Brave-specific adapter behind a trait boundary. ## Where To Start -- Design rules and boundaries: `PRD.md` -- Implementation plan: `.artifacts/plan/2026-04-14_search-cli/PLAN.md` - Harness map (checks, tests, gaps): `HARNESS.md` +- User-facing docs: `README.md` and `docs/` ## Repository Map - `src/main.rs` — CLI entrypoint; wires `cli::args`, `app::search_service`, `providers::brave`, and `transport::http` - `src/domain/` — provider-agnostic types (`query.rs`, `result.rs`, `types.rs`, `error.rs`, `provider.rs`) - `src/providers/brave/` — Brave-specific DTOs, mapper, config, and `BraveProvider` client - `src/transport/` — `HttpClient` trait and `ReqwestHttpClient` adapter -- `src/app/` — `SearchService` orchestrator (`Box`) -- `src/cli/` — `clap` argument parsing (`args.rs`) and text renderer (`output.rs`) +- `src/app/` — `SearchService` / `FanoutSearchService` orchestrators (`Box`) +- `src/cli/` — `clap` argument parsing (`args.rs`), runner (`runner.rs`), optional post-search Spider scrape (`scrape.rs`), and text renderer (`output.rs`) - `tests/architecture_test.rs` — source-scan tests enforcing layer boundaries - `docs/` — mdBook source: intro, architecture, quickstart @@ -23,7 +22,7 @@ - `just check` — run formatter check, clippy (with complexity/cognitive lints), tests, and mdBook docs build - `cargo test` — run all inline `#[cfg(test)]` unit tests and architecture boundary tests - `cargo run -- ""` — run a live web search (requires `BRAVE_API_KEY` in `.env`) -- `cargo run -- "" --search-type news --limit 3` — run a live news search +- `cargo run -- "" --scrape` — after search, dedupe result URLs and crawl each with headless Chrome (Spider), rendering content excerpts separately from crawl telemetry; `--scrape-limit` is pages per result URL seed, not top-N search results; see `--scrape-timeout-seconds`; `CHROME_URL` for remote browser ## Boundaries - **Domain** (`src/domain/`) — pure types and traits; no HTTP, no CLI parsing @@ -36,6 +35,12 @@ - `justfile` — canonical local check gate - `Cargo.toml` — dependencies and edition 2024 +## Observability +- Structured logging is provided by `tracing` (with `tracing-subscriber` formatting). +- Log output is written to **stderr** so stdout remains clean for CLI results. +- Control verbosity via the `RUST_LOG` environment variable (e.g. `RUST_LOG=debug`). +- Key spans: `main` (startup), `SearchService::search` (orchestration), `BraveProvider::search` / `ExaProvider::search` (provider adapters), `ReqwestHttpClient::{get_json,post_json}` (transport). + ## Change Guardrails - Run `just check` before committing. - Keep domain types provider-agnostic; add provider-specific logic in `src/providers/`. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..070741a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,52 @@ +--- +title: "Changelog" +when_to_read: + - "When reviewing user-visible changes across releases or unreleased work." + - "When preparing release notes or checking recent provider and CLI behavior changes." +summary: "Chronological project change log for sophon-cli, used to track additions, fixes, and behavior changes that matter to operators and users." +ontology_relations: + - relation: "records" + target: "sophon-cli-release-history" + note: "Tracks release-facing changes for the CLI." +--- + +# Changelog + +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- `--provider all` fan-out mode that queries every environment-enabled provider and renders per-provider successes and failures. (#11) +- Provider registry composition layer for built-in provider registration, provider metadata discovery, and `SearchService` construction. (#7) +- Environment-filtered structured tracing spans for startup, search orchestration, provider adapters, and HTTP transport. Logs are written to stderr so CLI result output stays clean. (#8) +- Markdown frontmatter validation and mdBook frontmatter stripping in the canonical `just check` gate. (#9) +- Cargo-managed pre-push hook that runs `just check`. (#9) +- Repository operating surfaces: `.env.example`, CODEOWNERS, issue and PR templates, label definitions, AGENTS validation CI, and a `sophon-cli` agent skill. (#6) + +### Changed + +- `main.rs` now selects providers through `ProviderId` and `ProviderRegistry` instead of directly constructing Brave and Exa clients. (#7) +- Production startup registers only providers with valid environment configuration, and provider-unavailable errors list configured providers. (#7) +- Architecture tests now include the `bootstrap` composition layer boundary. (#7) +- `HARNESS.md` and architecture docs now reflect the bootstrap layer, docs metadata guard, cargo-husky hook, and current validation chain. (#7, #9) +- Exa `/search` requests use highlights plus a query-scoped summary instead of requesting full-page `text` for normal CLI output. (#5) +- Exa snippet normalization now prefers trimmed summaries, then capped joined highlights; `text` is not used as a snippet fallback. (#5) +- CLI news output now prints `snippet` when present, matching web-result rendering. (#5) + +### Fixed + +- Exa web results no longer dump full extracted page markdown into the terminal when `summary` is missing. (#5) + +### Removed + +- Tracked `.artifacts/` planning and execution files; future local artifact output is ignored by Git. (#9) +- Legacy pre-commit configuration in favor of the Cargo-managed pre-push hook. (#9) + +### Security + +- Structured logging avoids recording provider authentication headers or API keys. (#8) diff --git a/Cargo.lock b/Cargo.lock index eb2bfac..e201794 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,54 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -67,6 +115,27 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -84,12 +153,46 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "auto_encoder" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1909575e56fe87516c438b6bf75100b2506200fa22e10cbcb4b18ccf2e4bcc5d" +dependencies = [ + "chardetng", + "encoding_rs", + "memchr", + "phf 0.11.3", +] + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "base64" version = "0.22.1" @@ -102,6 +205,36 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -113,6 +246,34 @@ name = "bytes" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-husky" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad" + +[[package]] +name = "case_insensitive_string" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d1e5af1533db1cecb562d78ac47fe7125d9c729cc8d5bb484db90ad78c8a3d9" +dependencies = [ + "compact_str", + "serde", +] + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] [[package]] name = "cc" @@ -121,6 +282,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -130,6 +293,68 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chardetng" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b8f0b65b7b08ae3c8187e8d77174de20cb6777864c6b832d8ad365999cf1ea" +dependencies = [ + "cfg-if", + "encoding_rs", + "memchr", +] + +[[package]] +name = "chromey" +version = "2.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "285e60c88e686e37ee5c5419ba7495eb96cc3a0520f39dd872e55b1f0545d29c" +dependencies = [ + "aho-corasick", + "base64", + "case_insensitive_string", + "dunce", + "fnv", + "futures-util", + "hashbrown 0.16.1", + "lazy_static", + "libc", + "memchr", + "phf 0.13.1", + "pin-project-lite", + "rand 0.10.1", + "reqwest 0.13.3", + "serde", + "serde_json", + "spider_chromiumoxide_cdp", + "spider_chromiumoxide_types", + "spider_fingerprint", + "spider_network_blocker", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite", + "tracing", + "url", + "winreg", +] + [[package]] name = "chrono" version = "0.4.44" @@ -169,7 +394,7 @@ version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn", @@ -181,12 +406,104 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compact_str" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "convert_case" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -214,95 +531,282 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] -name = "displaydoc" -version = "0.2.5" +name = "cpufeatures" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ - "proc-macro2", - "quote", - "syn", + "libc", ] [[package]] -name = "dotenvy" -version = "0.15.7" +name = "cpufeatures" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] [[package]] -name = "encoding_rs" -version = "0.8.35" +name = "crc32fast" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] [[package]] -name = "equivalent" -version = "1.0.2" +name = "crossbeam-utils" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] -name = "errno" -version = "0.3.14" +name = "crypto-common" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "libc", - "windows-sys 0.61.2", + "generic-array", + "typenum", ] [[package]] -name = "fastrand" -version = "2.4.1" +name = "cssparser" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.13.1", + "smallvec", +] [[package]] -name = "find-msvc-tools" -version = "0.1.9" +name = "cssparser-macros" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn", +] [[package]] -name = "fnv" -version = "1.0.7" +name = "dashmap" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] [[package]] -name = "foldhash" -version = "0.1.5" +name = "data-encoding" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] -name = "foreign-types" -version = "0.3.2" +name = "deranged" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "foreign-types-shared", + "powerfmt", ] [[package]] -name = "foreign-types-shared" -version = "0.1.1" +name = "derive_more" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] [[package]] -name = "form_urlencoded" -version = "1.2.2" +name = "derive_more-impl" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ - "percent-encoding", + "proc-macro2", + "quote", + "rustc_version", + "syn", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures-channel" version = "0.3.32" @@ -310,6 +814,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -318,6 +823,23 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -337,11 +859,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -349,8 +885,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", ] [[package]] @@ -361,7 +913,8 @@ checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -385,13 +938,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + [[package]] name = "hashbrown" version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", ] [[package]] @@ -399,6 +969,19 @@ name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "serde", + "serde_core", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "heck" @@ -406,6 +989,21 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "1.4.0" @@ -479,6 +1077,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", + "webpki-roots", ] [[package]] @@ -695,6 +1294,65 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.95" @@ -707,6 +1365,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -731,6 +1395,18 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "llm_models_spider" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59df3fc75e3bb80f00651ec1fefc7a1a8d8dc89a709be4225831d183cce75bb2" + [[package]] name = "lock_api" version = "0.4.14" @@ -747,19 +1423,63 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] -name = "memchr" -version = "2.8.0" +name = "lol_html" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "6888e8653f6e49cb2924c660fc367a8beeb6239b71e117fa082153c6ea44d427" +dependencies = [ + "bitflags", + "cfg-if", + "cssparser", + "encoding_rs", + "foldhash 0.2.0", + "hashbrown 0.16.1", + "memchr", + "mime", + "precomputed-hash", + "selectors", + "thiserror 2.0.18", +] [[package]] -name = "mime" -version = "0.3.17" +name = "lru-slab" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] -name = "mio" +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" @@ -786,6 +1506,27 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" + [[package]] name = "num-traits" version = "0.2.19" @@ -795,6 +1536,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -880,6 +1631,111 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros 0.13.1", + "phf_shared 0.13.1", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_codegen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.6", +] + +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator 0.11.3", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -901,6 +1757,27 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "prettyplease" version = "0.2.37" @@ -920,6 +1797,89 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "publicsuffix" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf" +dependencies = [ + "idna", + "psl-types", +] + +[[package]] +name = "quick-xml" +version = "0.39.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +dependencies = [ + "memchr", + "serde", + "tokio", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.45" @@ -929,12 +1889,79 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -944,6 +1971,35 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + [[package]] name = "reqwest" version = "0.12.28" @@ -953,7 +2009,9 @@ dependencies = [ "base64", "bytes", "encoding_rs", + "futures-channel", "futures-core", + "futures-util", "h2", "http", "http-body", @@ -968,6 +2026,8 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", "rustls-pki-types", "serde", "serde_json", @@ -975,12 +2035,60 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "reqwest" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +dependencies = [ + "base64", + "bytes", + "cookie", + "cookie_store", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", ] @@ -998,6 +2106,21 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1017,28 +2140,71 @@ version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ + "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ + "web-time", "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1056,6 +2222,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.29" @@ -1094,6 +2269,25 @@ dependencies = [ "libc", ] +[[package]] +name = "selectors" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fdfed56cd634f04fe8b9ddf947ae3dc493483e819593d2ba17df9ad05db8b2" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "log", + "new_debug_unreachable", + "phf 0.13.1", + "phf_codegen 0.13.1", + "precomputed-hash", + "rustc-hash", + "servo_arc", + "smallvec", +] + [[package]] name = "semver" version = "1.0.28" @@ -1143,6 +2337,16 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_regex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +dependencies = [ + "regex", + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1155,6 +2359,35 @@ dependencies = [ "serde", ] +[[package]] +name = "servo_arc" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1165,54 +2398,246 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" name = "signal-hook-registry" version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +dependencies = [ + "serde", +] + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "sophon-cli" +version = "0.1.0" +dependencies = [ + "async-trait", + "cargo-husky", + "chrono", + "clap", + "dotenvy", + "reqwest 0.12.28", + "serde", + "serde_json", + "spider", + "thiserror 1.0.69", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "spider" +version = "2.51.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611ae6d3b51422455d6c04730884d91baaa071c2919eca298062bc76df0894ab" +dependencies = [ + "ahash", + "aho-corasick", + "auto_encoder", + "base64", + "bytes", + "case_insensitive_string", + "chromey", + "cookie", + "dashmap", + "fastrand", + "h2", + "hashbrown 0.17.0", + "home", + "lazy_static", + "libc", + "log", + "lol_html", + "memchr", + "num_cpus", + "percent-encoding", + "phf 0.13.1", + "pin-project-lite", + "quick-xml", + "rand 0.9.4", + "regex", + "reqwest 0.13.3", + "serde", + "serde_json", + "serde_regex", + "simdutf8", + "smallvec", + "spider_agent_types", + "spider_fingerprint", + "string-interner", + "string_concat", + "strum", + "tokio", + "tokio-stream", + "tower", + "url", + "which", +] + +[[package]] +name = "spider_agent_types" +version = "2.51.139" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50ef3e0833162bebe6616124a9612f1a564deeb93165e30fe81c9881822f81ad" +dependencies = [ + "aho-corasick", + "llm_models_spider", + "memchr", + "serde", + "serde_json", +] + +[[package]] +name = "spider_chromiumoxide_cdp" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "425d07927c18fd157c4680cee8c7868e4a9bf4723bda1a4951c3316e3abdea4b" +dependencies = [ + "serde", + "serde_json", + "spider_chromiumoxide_pdl", + "spider_chromiumoxide_types", +] + +[[package]] +name = "spider_chromiumoxide_pdl" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4090793264cfc7533186505f7cc2198ef03325dfcbfe32d0e6814c6ca759051f" +dependencies = [ + "either", + "heck 0.4.1", + "once_cell", + "proc-macro2", + "quote", + "regex", + "serde", + "serde_json", + "spider_chromiumoxide_types", +] + +[[package]] +name = "spider_chromiumoxide_types" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7be30fbd75f45897bedcecde72e67ccad7cb4e8d11b92f34b7301c0f62592bf3" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "spider_fingerprint" +version = "2.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a74484e5e741d794810fa463c1b276d8913edf797abbfe83cbd55c4a6f141c40" dependencies = [ - "errno", - "libc", + "aho-corasick", + "case_insensitive_string", + "http", + "lazy_static", + "phf 0.11.3", + "phf_codegen 0.11.3", + "rand 0.9.4", + "reqwest 0.12.28", + "serde", + "serde_json", + "url", ] [[package]] -name = "slab" -version = "0.4.12" +name = "spider_network_blocker" +version = "0.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" +checksum = "5f681aa5665a44c0c6de41aab2dc84e8e4b0048abc0b7f52ed74e150c330eadf" +dependencies = [ + "arc-swap", + "convert_case", + "hashbrown 0.17.0", + "phf 0.13.1", + "phf_codegen 0.13.1", + "serde", + "url", +] [[package]] -name = "smallvec" -version = "1.15.1" +name = "stable_deref_trait" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] -name = "socket2" -version = "0.6.3" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] -name = "sophon-cli" -version = "0.1.0" +name = "string-interner" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23de088478b31c349c9ba67816fa55d9355232d63c3afea8bf513e31f0f1d2c0" dependencies = [ - "async-trait", - "chrono", - "clap", - "dotenvy", - "reqwest", + "hashbrown 0.15.5", "serde", - "serde_json", - "thiserror", - "tokio", ] [[package]] -name = "stable_deref_trait" -version = "1.2.1" +name = "string_concat" +version = "0.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +checksum = "c3c3ee6129eec20fed59acf2e9cfb3ffd20d0bbe39fe334c22af0edc56dfe752" [[package]] name = "strsim" @@ -1220,6 +2645,27 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1297,7 +2743,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -1311,6 +2766,57 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -1321,6 +2827,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.51.1" @@ -1369,6 +2890,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -1393,8 +2937,10 @@ dependencies = [ "pin-project-lite", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1403,13 +2949,18 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ + "async-compression", "bitflags", "bytes", + "futures-core", "futures-util", "http", "http-body", + "http-body-util", "iri-string", "pin-project-lite", + "tokio", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -1434,9 +2985,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -1444,6 +3007,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -1452,12 +3045,40 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.4", + "sha1", + "thiserror 2.0.18", +] + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -1494,12 +3115,34 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -1610,6 +3253,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -1632,6 +3288,52 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "which" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81995fafaaaf6ae47a7d0cc83c67caf92aeb7e5331650ae6ff856f7c0c60c459" +dependencies = [ + "libc", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -1784,6 +3486,16 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winreg" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d6f32a0ff4a9f6f01231eb2059cc85479330739333e0e58cadf03b6af2cca10" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -1800,7 +3512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" dependencies = [ "anyhow", - "heck", + "heck 0.5.0", "wit-parser", ] @@ -1811,7 +3523,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", - "heck", + "heck 0.5.0", "indexmap", "prettyplease", "syn", @@ -1901,6 +3613,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.7" @@ -1966,3 +3698,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 27d356b..a8811ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,12 @@ name = "sophon-cli" version = "0.1.0" edition = "2024" +description = "Provider-agnostic search CLI for Brave Search and Exa" +license = "MIT" +repository = "https://github.com/alchemiststudiosDOTai/sophon" +readme = "README.md" +keywords = ["search", "cli", "brave", "exa"] +categories = ["command-line-utilities"] [dependencies] tokio = { version = "1", features = ["full"] } @@ -13,3 +19,23 @@ async-trait = "0.1" clap = { version = "4", features = ["derive"] } dotenvy = "0.15" chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +spider = { version = "2", default-features = false, features = ["chrome", "chrome_intercept", "chrome_stealth"] } + +[[test]] +name = "search_service_integration" +path = "tests/integration/search_service_test.rs" + +[[test]] +name = "provider_registry_integration" +path = "tests/integration/provider_registry_test.rs" + +[[test]] +name = "cli_integration" +path = "tests/integration/cli_test.rs" + +[dev-dependencies.cargo-husky] +version = "1.5.0" +default-features = false +features = ["user-hooks"] diff --git a/HARNESS.md b/HARNESS.md index ef03c53..33ed95a 100644 --- a/HARNESS.md +++ b/HARNESS.md @@ -4,6 +4,15 @@ phase: Research date: "2026-04-14" owner: "agent" tags: [research, harness, sophon-cli, rust] + +when_to_read: + - "When validating local checks, architecture gates, tests, docs builds, or CI coverage." + - "When changing the repository harness or deciding which command proves the repo is healthy." +summary: "Harness map for sophon-cli, describing the canonical check command, test layers, architecture boundaries, documentation gates, and known validation gaps." +ontology_relations: + - relation: "defines" + target: "repository-harness" + note: "Documents the checks that protect changes in this repository." --- # sophon-cli – Harness Map @@ -12,11 +21,12 @@ A living map of the mechanical checks, policies, workflows, and artifacts that m ## Canonical Entry Point -- `justfile:1-5` defines the `check` recipe: +- `justfile:1-6` defines the `check` recipe: 1. `cargo fmt --check` 2. `cargo clippy -- -D warnings -W clippy::complexity -W clippy::cognitive_complexity` 3. `cargo test` - 4. `mdbook build` + 4. `python3 scripts/check_markdown_frontmatter.py` + 5. `mdbook build` There is no Makefile, npm script, or other local entrypoint. `just check` is the canonical umbrella command. @@ -28,6 +38,7 @@ There is no Makefile, npm script, or other local entrypoint. `just check` is the | Format | `cargo fmt --check` | `Cargo.toml` edition 2024 | Rust style consistency | | Lint / complexity | `cargo clippy -- -D warnings -W clippy::complexity -W clippy::cognitive_complexity` | `Cargo.toml` | Correctness + complexity ceiling | | Tests | `cargo test` | `Cargo.toml` | Behavioral verification | +| Docs metadata | `python3 scripts/check_markdown_frontmatter.py` | `scripts/check_markdown_frontmatter.py` | Required Markdown frontmatter except `AGENTS.md`, `README.md`, and `docs/SUMMARY.md` | | Docs build | `mdbook build` | `book.toml` | Documentation compiles | ### Layer 2: Architecture Boundaries @@ -37,6 +48,7 @@ There is no Makefile, npm script, or other local entrypoint. `just check` is the | transport-isolation | `src/transport/` | `crate::providers`, `crate::cli`, `crate::app` | `tests/architecture_test.rs` | | provider-isolation | `src/providers/` | `crate::cli`, `crate::app` | `tests/architecture_test.rs` | | app-isolation | `src/app/` | `crate::cli` | `tests/architecture_test.rs` | +| bootstrap-isolation | `src/bootstrap/` | `crate::cli` | `tests/architecture_test.rs` | | render_text-isolation | all except `src/cli/` | `render_text` | `tests/architecture_test.rs` | Architecture boundary tests run as part of `cargo test`. @@ -47,40 +59,46 @@ No structural rule engine is currently configured (no ast-grep, semgrep, or cust ### Layer 4: Behavioral Verification | Test Suite | Command | Location | Notes | |------------|---------|----------|-------| -| Unit tests (inline) | `cargo test` | `src/**/*.rs` under `#[cfg(test)]` | 7 tests across 4 modules | +| Unit tests (inline) | `cargo test` | `src/**/*.rs` under `#[cfg(test)]` | 22 tests across source modules | | Mapper tests | `cargo test` | `src/providers/brave/mapper.rs` | 4 tests: web, news, images, videos DTO→domain mapping | | Provider tests | `cargo test` | `src/providers/brave/client.rs` | 1 mock-HTTP test for `BraveProvider::search` | | App-layer tests | `cargo test` | `src/app/search_service.rs` | 1 mock-provider test for `SearchService` delegation | | Output tests | `cargo test` | `src/cli/output.rs` | 1 text-rendering test with mixed result types | -| Architecture tests | `cargo test` | `tests/architecture_test.rs` | 5 source-scan tests enforcing layer boundaries | +| Architecture tests | `cargo test` | `tests/architecture_test.rs` | 6 source-scan tests enforcing layer boundaries | No snapshot, golden, or integration test suites exist. ### Layer 5: Docs Ratchet | Check | Command | Allowlist | Notes | |-------|---------|-----------|-------| -| Docs build | `mdbook build` | n/a | Fails if markdown or `book.toml` is malformed | +| Frontmatter | `python3 scripts/check_markdown_frontmatter.py` | `AGENTS.md`, `README.md`, `docs/SUMMARY.md` | Requires `title`, `when_to_read`, `summary`, and structured `ontology_relations` | +| Docs build | `mdbook build` | n/a | Fails if markdown or `book.toml` is malformed; `scripts/mdbook_strip_frontmatter.py` strips metadata from rendered HTML | + +No link checker or nav check is configured. -No link checker, frontmatter validator, or nav check is configured. +### Layer 5.5: Git Hooks +| Hook | Source | Installed By | Runs | +|------|--------|--------------|------| +| pre-push | `.cargo-husky/hooks/pre-push` | `cargo-husky` dev dependency during `cargo test` | `just check` | ### Layer 6: CI Matrix -No CI is currently configured. There is no `.github/workflows/`, `.gitlab-ci.yml`, or equivalent. +| Workflow | Trigger | Checks | +|----------|---------|--------| +| `.github/workflows/validate-agents.yml` | pull requests and pushes to `main` | Verifies key `AGENTS.md` referenced paths exist, installs `just`/`mdbook`/`cargo-udeps`, and runs `just check` + `just hygiene` | ### Layer 7: Evidence Workflow -| Artifact | Location | Triggers | Format | -|----------|----------|----------|--------| -| Plan | `.artifacts/plan/2026-04-14_search-cli/PLAN.md` | Manual (plan-phase) | Markdown | -| Tickets | `.artifacts/plan/2026-04-14_search-cli/tickets/T*.md` | Plan decomposition | Markdown | -| Execution log | `.artifacts/execute/2026-04-14_search-cli.md` | Per-task updates | Markdown | +| Artifact | Location | Tracking | Notes | +|----------|----------|----------|-------| +| Local agent artifacts | `.artifacts/` | Ignored by Git | Research, planning, execution logs, and generated design notes may exist locally but are not repository sources of truth | -These are human-maintained research/execution artifacts, not mechanically enforced. +These are human-maintained local artifacts, not mechanically enforced. ### Layer 8: Operator Surface | Surface | Location | Purpose | Usage | |---------|----------|---------|-------| -| PRD | `PRD.md` | Product requirements & design rules | Read before implementing | -| Plan | `.artifacts/plan/2026-04-14_search-cli/PLAN.md` | Implementation plan | Execute-phase reference | -| Execution log | `.artifacts/execute/2026-04-14_search-cli.md` | Debug history & task status | Update after each ticket | +| README | `README.md` | User-facing package overview | Read for installation, configuration, and CLI examples | +| mdBook docs | `docs/` | Maintainer and user documentation | Read for architecture and quickstart details | +| Harness map | `HARNESS.md` | Checks, hooks, and validation chain | Read before changing repository gates | `AGENTS.md` exists at the repository root. There is no `.codex/` directory inside the repo. @@ -90,28 +108,38 @@ Ordered list of checks as executed by the canonical entry point: 1. `cargo fmt --check` 2. `cargo clippy -- -D warnings -W clippy::complexity -W clippy::cognitive_complexity` 3. `cargo test` -4. `mdbook build` +4. `python3 scripts/check_markdown_frontmatter.py` +5. `mdbook build` + +Ordered list of hygiene checks as executed by the hygiene entry point: +1. `cargo +nightly udeps` +2. `npx --yes jscpd@4.0.5` +3. `bash scripts/check_tech_debt.sh` +4. `bash scripts/check_large_files.sh` ## Quick Reference - **Run all local checks:** `just check` +- **Run all hygiene checks:** `just hygiene` - **Run tests only:** `cargo test` - **Run lint only:** `cargo clippy -- -D warnings -W clippy::complexity -W clippy::cognitive_complexity` - **Run formatter check only:** `cargo fmt --check` -- **Run CI locally:** Not applicable (no CI configured) +- **Run CI locally:** `just check && just hygiene` covers workflow command checks; path existence checks are in `.github/workflows/validate-agents.yml` - **Add a new check:** Edit `justfile` and append to the `check` recipe ## Source Index | File | What It Contributes | |------|---------------------| -| `justfile:1-5` | Canonical local check gate | +| `justfile:1-6` | Canonical local check gate | +| `.cargo-husky/hooks/pre-push` | Cargo-managed pre-push hook that runs `just check` | +| `scripts/check_markdown_frontmatter.py` | Markdown frontmatter and ontology relation validator | +| `scripts/mdbook_strip_frontmatter.py` | mdBook preprocessor that keeps metadata out of rendered HTML | | `AGENTS.md` | Operator-facing navigational map | +| `.github/workflows/validate-agents.yml` | Pull-request AGENTS path and canonical harness check | | `docs/` | mdBook source: intro, architecture, quickstart | | `book.toml` | mdBook configuration | -| `Cargo.toml` | Project manifest, dependencies, edition 2024 | +| `Cargo.toml` | Project manifest, dependencies, edition 2024, cargo-husky hook installer | +| `src/bootstrap/provider_registry.rs` | Built-in provider registry and service construction tests | | `src/providers/brave/mapper.rs` | 4 unit tests for DTO→domain mapping | | `src/providers/brave/client.rs` | 1 mock-HTTP unit test for Brave provider | | `src/app/search_service.rs` | 1 mock-provider unit test for SearchService | | `src/cli/output.rs` | 1 unit test for text output rendering | -| `.artifacts/plan/2026-04-14_search-cli/PLAN.md` | Implementation plan (evidence) | -| `.artifacts/execute/2026-04-14_search-cli.md` | Execution log (evidence) | -| `PRD.md` | Product requirements & operator guidance | diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..098c48b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Sophon Relay + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index a0604c8..f2806a7 100644 --- a/README.md +++ b/README.md @@ -4,24 +4,74 @@ ![sophon](docs/sophon.png) -A provider-agnostic Rust CLI that queries the Brave Search API and prints normalized text results. +A provider-agnostic Rust CLI that queries Brave Search, Exa, or every environment-enabled provider and prints normalized text results. -## Quick start +## Install ```bash -# Set your API key -echo "BRAVE_API_KEY=your_key_here" > .env +cargo install sophon-cli +``` -# Run a search +## Quick start + +```bash +# Run locally from the repo cargo run -- "rust programming" -# About +# Choose a provider explicitly +cargo run -- "rust programming" --provider brave +cargo run -- "rust programming" --provider exa + +# Query every provider enabled by environment variables +cargo run -- "rust async trait" --provider all + +# After results, scrape each deduplicated result URL with headless Chrome (Spider). +# --scrape-limit is pages per result URL seed, not a top-N search-result limit. +cargo run -- "rust crates" --scrape --scrape-limit 5 --scrape-timeout-seconds 60 + +# Show package info cargo run -- --about # Run all checks just check ``` +## Configuration + +Set the API key for the provider you want to use: + +```bash +# Brave +echo "BRAVE_API_KEY=your_key_here" > .env + +# Exa +echo "EXA_API_KEY=your_key_here" > .env +``` + +You can also export the variables directly in your shell instead of using `.env`. `--provider all` queries every provider enabled by the current environment; for example, set both `BRAVE_API_KEY` and `EXA_API_KEY` to fan out to both providers. + +## Example usage + +```bash +# Web search with Brave +sophon-cli "rust programming" --provider brave + +# News search with Brave +sophon-cli "open source ai" --provider brave --search-type news --limit 3 + +# Exa search +sophon-cli "vector database benchmarks" --provider exa --limit 5 + +# All configured providers, with per-provider successes and failures +sophon-cli "rust async trait" --provider all +``` + +## Supported providers + +- `brave` for web, news, images, and video search +- `exa` for Exa search results mapped into the shared domain model +- `all` to query every configured provider in stable order and print per-provider failures when one provider rejects or fails a request + ## Docs See the [architecture docs](docs/architecture.md) for the typed input-to-output flow and layer boundaries. diff --git a/book.toml b/book.toml index 48b255c..4cba0bb 100644 --- a/book.toml +++ b/book.toml @@ -4,4 +4,8 @@ language = "en" src = "docs" title = "sophon-cli Documentation" +[preprocessor.strip-frontmatter] +command = "python3 scripts/mdbook_strip_frontmatter.py" +renderer = ["html"] + [output.html] diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index c3a111e..e1873a3 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -3,4 +3,6 @@ [Introduction](intro.md) - [Architecture](architecture.md) +- [Current Dependency Architecture Map](dependency-architecture-map.md) +- [Response Shape](response-shape.md) - [Quick Start](quickstart.md) diff --git a/docs/architecture.md b/docs/architecture.md index ecf35b0..b8f5aab 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,8 +1,20 @@ +--- +title: "Architecture" +when_to_read: + - "When reading or editing the mdBook documentation surface." + - "When checking how the CLI architecture, quickstart, and user-facing docs fit together." +summary: "mdBook documentation page for sophon-cli: Architecture. It contributes user and maintainer guidance that is built by the docs gate." +ontology_relations: + - relation: "part_of" + target: "docs/SUMMARY.md" + note: "Belongs to the mdBook documentation set." +--- + # Architecture ## Design principle -The codebase is split into four strictly ordered layers. **No layer may import from a layer above it.** This prevents the CLI from leaking into the domain and keeps providers interchangeable. +The codebase is split into four strictly ordered runtime layers plus a narrow bootstrap composition layer. **No runtime layer may import from a layer above it.** This prevents the CLI from leaking into the domain and keeps providers interchangeable. ``` cli (top) @@ -14,6 +26,8 @@ providers transport ↑ domain (bottom) + +bootstrap composes app + providers + transport at startup ``` ## Full request lifecycle @@ -23,33 +37,41 @@ domain (bottom) └── user runs: cargo run -- "query" --search-type news --limit 3 2. src/main.rs + └── initializes tracing and dotenv + └── delegates runtime execution to src/cli/runner.rs + +3. src/cli/runner.rs └── CliArgs::parse() produces CliArgs { query, provider, search_type, limit, ... } - └── selects BraveProvider or ExaProvider - └── maps CliArgs → SearchQuery + └── src/cli/request.rs maps CliArgs + query text → SearchQuery + └── selects single-provider or all-provider mode + └── requests services from ProviderRegistry + └── renders stdout through src/cli/output.rs -3. src/app/search_service.rs - └── SearchService::search(SearchQuery) awaits - └── delegates to dyn SearchProvider +4. src/app/search_service.rs or src/app/fanout_search_service.rs + └── SearchService::search(SearchQuery) awaits one provider + └── FanoutSearchService::search_all(SearchQuery) awaits enabled providers sequentially + └── delegates to dyn SearchProvider trait objects -4. src/providers/*/client.rs +5. src/providers/*/client.rs └── provider-specific SearchProvider::search(&SearchQuery) └── Brave builds GET endpoint + query params └── Exa builds POST /search JSON body └── calls HttpClient::get_json() or HttpClient::post_json() -5. src/transport/http.rs +6. src/transport/http.rs └── ReqwestHttpClient executes HTTP GET └── on success: deserializes JSON into BraveNewsResponse └── on failure: maps status code → SearchError -6. src/providers/brave/mapper.rs +7. src/providers/brave/mapper.rs └── map_news_response(BraveNewsResponse) → SearchResponse └── transforms DTOs into domain SearchResult::News items -7. src/cli/output.rs - └── render_text(&SearchResponse) → String +8. src/cli/output.rs + └── render_text(&SearchResponse) → String for single-provider output + └── render_fanout_text(&SearchBatchResponse) → String for all-provider output -8. src/main.rs +9. src/cli/runner.rs └── println!("{}", rendered_string) ``` @@ -60,11 +82,13 @@ Every public function that crosses a module boundary uses a domain type. | Boundary | Function | Input type | Output type | |----------|----------|------------|-------------| | CLI → App | `SearchService::search` | `SearchQuery` | `Result` | +| CLI → App | `FanoutSearchService::search_all` | `SearchQuery` | `SearchBatchResponse` | | App → Provider | `SearchProvider::search` | `&SearchQuery` | `Result` | | Provider → Transport | `HttpClient::{get_json, post_json}` | `url, headers, params/body` | `Result` | | Transport → Provider | (JSON response body) | bytes | provider DTOs such as `Brave*Response` or `ExaSearchResponse` | | Provider → Domain | `map_*_response` | provider DTOs | `SearchResponse` | -| App → CLI | `render_text` | `&SearchResponse` | `String` | +| CLI rendering | `render_text` | `&SearchResponse` | `String` | +| CLI rendering | `render_fanout_text` | `&SearchBatchResponse` | `String` | ## Domain type reference @@ -90,7 +114,7 @@ pub struct SearchResponse { pub query: String, pub provider: String, pub results: Vec, - pub total_estimated: Option, + pub total_estimated: Option, pub next_page: Option, } ``` @@ -104,6 +128,21 @@ pub enum SearchResult { } ``` +Fan-out results stay provider-agnostic in the domain layer: + +```rust +pub struct SearchBatchResponse { + pub query: String, + pub responses: Vec, + pub failures: Vec, +} + +pub struct ProviderSearchFailure { + pub provider: String, + pub error: SearchError, +} +``` + ### Provider trait ```rust @@ -133,8 +172,8 @@ Errors are created at the layer where the failure occurs and bubble upward uncha 1. **Transport layer** — `reqwest` failures, non-2xx HTTP status, or JSON decode errors become `SearchError::Transport`, `SearchError::Provider`, or `SearchError::Decode`. 2. **Provider layer** — can surface `SearchError` directly; does not wrap in another error type. -3. **App layer** — `SearchService` returns the `SearchError` untouched. -4. **CLI layer** — `main.rs` matches on `SearchError` and prints a human-readable message to `stderr`, then exits with code `1`. +3. **App layer** — `SearchService` returns the `SearchError` untouched; `FanoutSearchService` records per-provider failures in `SearchBatchResponse` and continues to later providers. +4. **CLI layer** — `src/cli/runner.rs` matches on single-provider `SearchError` and prints a human-readable message to `stderr`, or renders fan-out successes and failures and returns exit code `1` when no provider succeeded. This keeps error handling simple: there is only one error type in the public API. @@ -162,12 +201,15 @@ Unsupported Exa inputs are rejected at runtime instead of being ignored: `Images ## Runtime provider selection -`main.rs` remains the only place that chooses a concrete provider. `SearchQuery`, `SearchResponse`, `SearchProvider`, and `SearchService` stay unchanged. +`src/main.rs` remains a thin process entrypoint. It initializes process-level concerns and delegates to `src/cli/runner.rs`, which owns user-surface branching, query normalization, output rendering, and exit-code calculation. Concrete provider construction remains in `src/bootstrap/provider_registry.rs`, where typed provider config, HTTP transport, provider clients, `SearchService`, and `FanoutSearchService` are composed. -- `--provider brave` loads `BraveConfig` and constructs `BraveProvider` -- `--provider exa` loads `ExaConfig` and constructs `ExaProvider` +- `--provider brave` uses `ProviderRegistry::build(ProviderId::Brave)`; the registry includes it only when `BRAVE_API_KEY` is configured +- `--provider exa` uses `ProviderRegistry::build(ProviderId::Exa)`; the registry includes it only when `EXA_API_KEY` is configured +- `--provider all` uses `ProviderRegistry::build_all_enabled()` to build a `FanoutSearchService` from every configured provider in stable order - omitting `--provider` still selects Brave +`FanoutSearchService` is application-layer orchestration over multiple domain `SearchProvider` trait objects. It does not render output; fan-out rendering remains in the CLI layer through `render_fanout_text`. + ## Architecture enforcement The rules are verified by `tests/architecture_test.rs`. These tests scan source files and fail if a forbidden import pattern is found. @@ -177,7 +219,8 @@ The rules are verified by `tests/architecture_test.rs`. These tests scan source | `src/domain/` | `crate::providers::`, `crate::transport::`, `crate::cli::`, `crate::app::` | | `src/transport/` | `crate::providers::`, `crate::cli::`, `crate::app::` | | `src/providers/` | `crate::cli::`, `crate::app::` | -| `src/app/` | `crate::cli::` | +| `src/app/` | `crate::cli::`, `crate::bootstrap::`, `crate::providers::`, `crate::transport::` | +| `src/bootstrap/` | `crate::cli::` | | Any layer except `src/cli/` | `render_text` | Run them with the rest of the suite: diff --git a/docs/dependency-architecture-map.html b/docs/dependency-architecture-map.html new file mode 100644 index 0000000..b6f8338 --- /dev/null +++ b/docs/dependency-architecture-map.html @@ -0,0 +1,369 @@ + + + + + +sophon-cli Current Dependency Architecture Map + + + + + + +
+
+
current import map
+

sophon-cli dependency architecture

+

Accurate source dependency map after the CLI runner refactor. Arrows mean importer depends on target. The entrypoint delegates to the CLI surface; bootstrap owns concrete provider and HTTP wiring; app services orchestrate domain trait objects; providers and transport depend downward on domain contracts.

+
+ +
+ + sophon-cli current dependency architecture map + Layered Rust module import map from entrypoint to CLI, bootstrap, app services, provider adapters, transport, and domain core. + + + + + + + + + + + + + + + + + + + + + + + + + + entry + + cli user surface + + bootstrap composition root + + application services + + adapters + + domain core + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + only runtime import from main + runner asks registry for services + bootstrap owns concrete construction + app sees SearchProvider trait objects only + all adapters return domain contracts + + + + + + src/main.rs + tracing + dotenv + run_from_env + + + + cli::runner + about, missing query + provider mode, exit code + + + + cli::args + clap flags + enums + + + + cli::request + CliArgs to SearchQuery + + + + cli::output + render_text + fanout + + + + bootstrap::provider_registry + ProviderRegistry + ProviderId + builds services and adapters + + + + app::SearchService + single SearchProvider orchestration + + + + app::FanoutSearchService + sequential provider fan-out + + + + brave::client/config/dto + SearchProvider + GET API DTOs + + + + brave::mapper + DTOs to SearchResponse + + + + exa::client/config/dto + SearchProvider + POST DTOs + + + + exa::mapper + DTOs to SearchResponse + + + + transport::http + HttpClient + ReqwestHttpClient + + + + domain::query/types + SearchQuery, SearchType + + + + domain::provider + SearchProvider trait + + + + domain::result/error + responses, failures, errors + + + + + + Accuracy notes + Deleted binary-private helper is intentionally absent. main imports only cli::runner at runtime. app imports domain only. + bootstrap imports app, providers, transport, and domain. providers import transport and domain. transport imports domain::SearchError. + + +
+ +
+
+

CLI surface

+
    +
  • * `cli::runner` owns user-facing branching and rendering.
    • +
  • * `cli::request` is the CLI-to-domain query boundary.
    • +
  • * `main` delegates to `run_from_env()` and handles exit.
    • +
+
+
+

Composition

+
    +
  • * `ProviderRegistry` is the concrete composition root.
    • +
  • * It builds app services from Brave/Exa and reqwest transport.
    • +
  • * The CLI asks for services; it does not construct providers.
    • +
+
+
+

Domain direction

+
    +
  • * App services import only domain contracts.
    • +
  • * Providers implement domain traits and use transport.
    • +
  • * Domain has no upward imports.
    • +
+
+
+ +
Generated for docs/dependency-architecture-map.html.
+
+ + diff --git a/docs/dependency-architecture-map.md b/docs/dependency-architecture-map.md new file mode 100644 index 0000000..edd0ee2 --- /dev/null +++ b/docs/dependency-architecture-map.md @@ -0,0 +1,35 @@ +--- +title: "Current Dependency Architecture Map" +when_to_read: + - "When you need to see the current Rust import/dependency direction as a clean architecture map." +summary: "Current architecture-style map of sophon-cli module import direction." +ontology_relations: + - relation: "part_of" + target: "docs/SUMMARY.md" + note: "Belongs to the mdBook documentation set." + - relation: "explains" + target: "docs/architecture.md" + note: "Visualizes the concrete import direction behind the architecture layers." +--- + +# Current Dependency Architecture Map + +This is the current/actual Rust import shape after the runtime organization refactor: `src/main.rs` delegates to the CLI runner, the CLI surface performs user-facing branching and rendering, bootstrap owns concrete construction, app services orchestrate domain provider traits, and domain remains the bottom layer. + +Read every dependency as: + +```text +importer -> imported dependency +``` + +Open the standalone visual page: + +[Current dependency architecture map](dependency-architecture-map.html) + + + +If the embedded frame is cramped, open the standalone page above. diff --git a/docs/intro.md b/docs/intro.md index bf8cd6c..9f55c36 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -1,20 +1,33 @@ +--- +title: "sophon-cli" +when_to_read: + - "When reading or editing the mdBook documentation surface." + - "When checking how the CLI architecture, quickstart, and user-facing docs fit together." +summary: "mdBook documentation page for sophon-cli: sophon-cli. It contributes user and maintainer guidance that is built by the docs gate." +ontology_relations: + - relation: "part_of" + target: "docs/SUMMARY.md" + note: "Belongs to the mdBook documentation set." +--- + # sophon-cli -A provider-agnostic Rust CLI that queries the Brave Search API and prints normalized text results. +A provider-agnostic Rust CLI that queries Brave Search, Exa, or every environment-enabled provider and prints normalized text results. ## What it does - Parses CLI arguments - Builds a provider-agnostic `SearchQuery` -- Delegates to a `SearchProvider` (currently Brave) -- Renders results as human-readable text +- Delegates to a single `SearchProvider` for `--provider brave` or `--provider exa` +- Fans out sequentially to all environment-enabled providers for `--provider all` +- Renders single-provider or per-provider fan-out results as human-readable text ## Project structure - `src/domain/` — pure types and traits; no HTTP, no CLI - `src/transport/` — `HttpClient` trait + `reqwest` adapter - `src/providers/brave/` — Brave-specific DTOs, mapper, and client -- `src/app/` — `SearchService` orchestrator +- `src/app/` — `SearchService` and `FanoutSearchService` orchestrators - `src/cli/` — argument parsing and output rendering - `tests/architecture_test.rs` — boundary tests enforcing layer isolation diff --git a/docs/quickstart.md b/docs/quickstart.md index 27d84c4..3f62d60 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,9 +1,21 @@ +--- +title: "Quick Start" +when_to_read: + - "When reading or editing the mdBook documentation surface." + - "When checking how the CLI architecture, quickstart, and user-facing docs fit together." +summary: "mdBook documentation page for sophon-cli: Quick Start. It contributes user and maintainer guidance that is built by the docs gate." +ontology_relations: + - relation: "part_of" + target: "docs/SUMMARY.md" + note: "Belongs to the mdBook documentation set." +--- + # Quick Start ## Prerequisites - Rust toolchain (edition 2024) -- `BRAVE_API_KEY` in a `.env` file at the project root +- `BRAVE_API_KEY` and/or `EXA_API_KEY` in a `.env` file at the project root ## Install the task runner @@ -23,6 +35,18 @@ just check cargo run -- "rust programming" ``` +## Run all configured providers + +Set both provider keys when you want `--provider all` to query Brave and Exa in one run: + +```bash +export BRAVE_API_KEY=your_brave_key +export EXA_API_KEY=your_exa_key +cargo run -- "rust async trait" --provider all +``` + +`--provider all` includes only providers enabled by the current environment variables. If neither `BRAVE_API_KEY` nor `EXA_API_KEY` is available, the command exits non-zero and prints `no configured providers; set BRAVE_API_KEY and/or EXA_API_KEY`. + ## Search news ```bash diff --git a/docs/response-shape.md b/docs/response-shape.md new file mode 100644 index 0000000..4d1549b --- /dev/null +++ b/docs/response-shape.md @@ -0,0 +1,223 @@ +--- +title: "Response Shape" +when_to_read: + - "When integrating with sophon-cli output or consuming SearchResponse/SearchBatchResponse." + - "When mapping provider DTOs into domain results or adding new result types." +summary: "Reference for every field in the domain search response, including single-provider, fan-out, error, and CLI text representations." +ontology_relations: + - relation: "part_of" + target: "docs/SUMMARY.md" + note: "Belongs to the mdBook documentation set." +--- + +# Response Shape + +When you run a search, the domain layer returns provider-agnostic types. The CLI renders them as plain text; if you are using the library surface directly, you work with these structs and enums. + +## Single provider + +```rust +pub struct SearchResponse { + pub query: String, + pub provider: String, + pub results: Vec, + pub total_estimated: Option, + pub next_page: Option, +} +``` + +| Field | Meaning | +|-------|---------| +| `query` | Normalized query text that was sent to the provider. | +| `provider` | Provider identifier, e.g. `"brave"` or `"exa"`. | +| `results` | Ordered list of hits. Each item is one of four variants. | +| `total_estimated` | Provider-reported total hit count, if available. Not rendered in CLI text output. | +| `next_page` | Opaque token for fetching the next page, if the provider supports pagination. Not rendered in CLI text output. | + +--- + +## `SearchResult` variants + +### `SearchResult::Web` + +```rust +pub struct WebResult { + pub title: String, + pub url: String, + pub snippet: Option, + pub display_url: Option, +} +``` + +**CLI text rendering** + +```text +1. [Rust Programming Language] + URL: https://www.rust-lang.org + A language empowering everyone to build reliable software. +``` + +- `display_url` is not shown in text output. + +--- + +### `SearchResult::News` + +```rust +pub struct NewsResult { + pub title: String, + pub url: String, + pub snippet: Option, + pub source: Option, + pub published_at: Option, +} +``` + +**CLI text rendering** + +```text +2. [NEWS] Rust 1.85 Released + URL: https://blog.rust-lang.org/2025/02/20/Rust-1.85.0.html + Source: Rust Blog + Major compiler improvements and new language features. +``` + +- `published_at` is not shown in text output. + +--- + +### `SearchResult::Image` + +```rust +pub struct ImageResult { + pub title: String, + pub url: String, + pub thumbnail_url: Option, + pub source: Option, +} +``` + +**CLI text rendering** + +```text +3. [IMAGE] Rust Logo + URL: https://www.rust-lang.org/static/images/rust-logo-blk.svg +``` + +- `thumbnail_url` and `source` are not shown in text output. + +--- + +### `SearchResult::Video` + +```rust +pub struct VideoResult { + pub title: String, + pub url: String, + pub thumbnail_url: Option, + pub duration: Option, + pub published_at: Option, +} +``` + +**CLI text rendering** + +```text +4. [VIDEO] Rust Tutorial + URL: https://example.com/rust-tutorial +``` + +- `thumbnail_url`, `duration`, and `published_at` are not shown in text output. + +--- + +## Multi-provider fanout (`--provider all`) + +When multiple providers are queried, the result is a `SearchBatchResponse`: + +```rust +pub struct SearchBatchResponse { + pub query: String, + pub responses: Vec, + pub failures: Vec, +} + +pub struct ProviderSearchFailure { + pub provider: String, + pub error: SearchError, +} +``` + +| Field | Meaning | +|-------|---------| +| `query` | The shared query text. | +| `responses` | One `SearchResponse` per provider that succeeded. | +| `failures` | One entry per provider that failed; the error is preserved so callers know why. | + +**CLI text rendering** + +```text +Query: rust programming +Providers succeeded: 2 +Providers failed: 0 + +== brave == +Provider: brave +Query: rust programming +Results: 3 +1. [Rust Programming Language] + URL: https://www.rust-lang.org + ... + +== exa == +Provider: exa +Query: rust programming +Results: 2 +1. [Systems Programming with Rust] + URL: https://example.com/intro + ... +``` + +If a provider fails: + +```text +== Failures == +- exa: invalid query: unsupported +``` + +--- + +## Error shape + +The single public error type across all layers is `SearchError`: + +```rust +pub enum SearchError { + InvalidQuery(String), + Unauthorized, + RateLimited, + ProviderError(String), + NetworkError(String), + SerializationError(String), + Unexpected(String), +} +``` + +In text output, errors are formatted as lower-case messages prefixed by the variant: + +- `InvalidQuery("foo")` → `invalid query: foo` +- `Unauthorized` → `unauthorized` +- `RateLimited` → `rate limited` +- `ProviderError("down")` → `provider error: down` +- `NetworkError("timeout")` → `network error: timeout` +- `SerializationError("bad json")` → `serialization error: bad json` +- `Unexpected("oops")` → `unexpected error: oops` + +--- + +## Summary + +- **One provider** → `SearchResponse` → `render_text` +- **All providers** → `SearchBatchResponse` → `render_fanout_text` +- **Each hit** → `SearchResult::{Web,News,Image,Video}` with typed fields +- **CLI text** shows `title`, `url`, and the most common human-readable fields; optional metadata (`display_url`, `published_at`, `thumbnail_url`, `duration`, `total_estimated`, `next_page`) is present in the domain type but omitted from text output. diff --git a/justfile b/justfile index 8525a6f..bf087f9 100644 --- a/justfile +++ b/justfile @@ -1,6 +1,24 @@ -# Check gate: formatter, linter (+ complexity), tests, docs build +# Check gate: formatter, linter (+ complexity), tests, docs metadata, docs build check: cargo fmt --check cargo clippy -- -D warnings -W clippy::complexity -W clippy::cognitive_complexity cargo test + python3 scripts/check_markdown_frontmatter.py mdbook build + +# Hygiene gate: dependencies, duplication, tech debt markers, large files +hygiene: udeps duplicates tech-debt large-files + +# Uses nightly because cargo-udeps relies on nightly internals. +udeps: + if ! cargo udeps --version >/dev/null 2>&1; then cargo install cargo-udeps --locked; fi + cargo +nightly udeps + +duplicates: + npx --yes jscpd@4.0.5 + +tech-debt: + bash scripts/check_tech_debt.sh + +large-files: + bash scripts/check_large_files.sh diff --git a/scripts/check_large_files.sh b/scripts/check_large_files.sh new file mode 100644 index 0000000..a10d2dc --- /dev/null +++ b/scripts/check_large_files.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +max_bytes=512000 +max_kb=500 +allowlist_file="scripts/large_files_allowlist.txt" + +is_allowlisted() { + local candidate="$1" + if [ ! -f "${allowlist_file}" ]; then + return 1 + fi + + while IFS= read -r pattern; do + [ -z "${pattern}" ] && continue + [[ "${pattern}" =~ ^# ]] && continue + if [ "${candidate}" = "${pattern}" ]; then + return 0 + fi + done <"${allowlist_file}" + + return 1 +} + +offenders=() +while IFS= read -r -d '' file; do + if [ ! -f "${file}" ]; then + continue + fi + if is_allowlisted "${file}"; then + continue + fi + + size_bytes="$(wc -c <"${file}" | tr -d '[:space:]')" + if [ "${size_bytes}" -gt "${max_bytes}" ]; then + offenders+=("${file}:${size_bytes}") + fi +done < <(git ls-files -z) + +if [ "${#offenders[@]}" -gt 0 ]; then + echo "error: tracked files exceed ${max_kb} KB (${max_bytes} bytes):" + for offender in "${offenders[@]}"; do + file="${offender%%:*}" + size="${offender##*:}" + printf ' - %s (%s bytes)\n' "${file}" "${size}" + done + exit 1 +fi + +echo "Large file check passed." diff --git a/scripts/check_markdown_frontmatter.py b/scripts/check_markdown_frontmatter.py new file mode 100755 index 0000000..640f9a1 --- /dev/null +++ b/scripts/check_markdown_frontmatter.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Validate required Markdown frontmatter for tracked docs.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +REQUIRED_KEYS = ("title", "when_to_read", "summary", "ontology_relations") +RELATION_KEYS = ("relation", "target", "note") + + +def tracked_markdown_files() -> list[Path]: + result = subprocess.run( + ["git", "ls-files", "*.md"], + check=True, + capture_output=True, + text=True, + ) + skipped = {"AGENTS.md", "README.md", "docs/SUMMARY.md"} + return [Path(line) for line in result.stdout.splitlines() if line and line not in skipped] + + +def frontmatter_lines(path: Path) -> tuple[list[str] | None, str | None]: + text = path.read_text(encoding="utf-8") + if not text.startswith("---\n"): + return None, "missing YAML frontmatter at top of file" + + end = text.find("\n---\n", 4) + if end == -1: + return None, "missing closing YAML frontmatter delimiter" + + return text[4:end].splitlines(), None + + +def top_level_keys(lines: list[str]) -> set[str]: + keys: set[str] = set() + for line in lines: + if not line or line.startswith((" ", "\t", "-")): + continue + if ":" in line: + keys.add(line.split(":", 1)[0].strip()) + return keys + + +def ontology_relation_errors(lines: list[str]) -> list[str]: + errors: list[str] = [] + entries: list[set[str]] = [] + current: set[str] | None = None + in_relations = False + + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + + if not line.startswith((" ", "\t", "-")) and ":" in line: + in_relations = line.split(":", 1)[0].strip() == "ontology_relations" + current = None + continue + + if not in_relations: + continue + + if stripped.startswith("- "): + if current is not None: + entries.append(current) + current = set() + remainder = stripped[2:] + if ":" in remainder: + current.add(remainder.split(":", 1)[0].strip()) + continue + + if current is not None and ":" in stripped: + current.add(stripped.split(":", 1)[0].strip()) + + if current is not None: + entries.append(current) + + if not entries: + return ["ontology_relations must contain at least one relation entry"] + + for index, entry in enumerate(entries, start=1): + missing = [key for key in RELATION_KEYS if key not in entry] + if missing: + errors.append( + f"ontology_relations entry {index} missing key(s): {', '.join(missing)}" + ) + + return errors + + +def validate(path: Path) -> list[str]: + lines, frontmatter_error = frontmatter_lines(path) + if frontmatter_error is not None: + return [frontmatter_error] + + assert lines is not None + keys = top_level_keys(lines) + errors = [ + f"missing required frontmatter key: {key}" + for key in REQUIRED_KEYS + if key not in keys + ] + + if "ontology_relations" in keys: + errors.extend(ontology_relation_errors(lines)) + + return errors + + +def main() -> int: + failures: dict[Path, list[str]] = {} + for path in tracked_markdown_files(): + errors = validate(path) + if errors: + failures[path] = errors + + if failures: + print("Markdown frontmatter validation failed:", file=sys.stderr) + for path, errors in failures.items(): + print(f"- {path}", file=sys.stderr) + for error in errors: + print(f" - {error}", file=sys.stderr) + return 1 + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_tech_debt.sh b/scripts/check_tech_debt.sh new file mode 100644 index 0000000..ee16345 --- /dev/null +++ b/scripts/check_tech_debt.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +if ! command -v rg >/dev/null 2>&1; then + echo "error: ripgrep (rg) is required for tech debt checks." + exit 1 +fi + +allowlist_file="scripts/tech_debt_allowlist.txt" +matches_file="$(mktemp)" +cleanup() { + rm -f "${matches_file}" +} +trap cleanup EXIT + +include_globs=( + "*.rs" + "*.toml" +) + +exclude_globs=( + "target/**" + ".git/**" + "node_modules/**" +) + +rg_args=(--line-number --with-filename --no-heading "(TODO|FIXME)") +for glob in "${include_globs[@]}"; do + rg_args+=(--glob "${glob}") +done +for glob in "${exclude_globs[@]}"; do + rg_args+=(--glob "!${glob}") +done + +rg_status=0 +rg "${rg_args[@]}" . >"${matches_file}" || rg_status=$? + +if [ "${rg_status}" -gt 1 ]; then + echo "error: ripgrep exited with status ${rg_status}." + exit 1 +fi + +if [ "${rg_status}" -eq 0 ] && [ -f "${allowlist_file}" ]; then + # Build a sanitized allowlist that ignores blank lines and comments. + sanitized_allowlist="$(mktemp)" + while IFS= read -r line; do + [ -z "${line}" ] && continue + [[ "${line}" =~ ^[[:space:]]*# ]] && continue + printf '%s\n' "${line}" >>"${sanitized_allowlist}" + done <"${allowlist_file}" + + if [ -s "${sanitized_allowlist}" ]; then + filtered_file="$(mktemp)" + grep_exit=0 + grep -F -v -f "${sanitized_allowlist}" "${matches_file}" >"${filtered_file}" || grep_exit=$? + if [ "${grep_exit}" -gt 1 ]; then + echo "error: grep filtering failed with exit code ${grep_exit}." + rm -f "${sanitized_allowlist}" "${filtered_file}" + exit 1 + fi + mv "${filtered_file}" "${matches_file}" + fi + rm -f "${sanitized_allowlist}" +fi + +if [ -s "${matches_file}" ]; then + echo "error: found TODO/FIXME markers." + echo "Add exact match substrings to ${allowlist_file} to allow specific lines." + cat "${matches_file}" + exit 1 +fi + +echo "Tech debt marker check passed." diff --git a/scripts/large_files_allowlist.txt b/scripts/large_files_allowlist.txt new file mode 100644 index 0000000..934b989 --- /dev/null +++ b/scripts/large_files_allowlist.txt @@ -0,0 +1,2 @@ +# Relative paths allowed to exceed the large-file threshold. +docs/sophon.png diff --git a/scripts/mdbook_strip_frontmatter.py b/scripts/mdbook_strip_frontmatter.py new file mode 100644 index 0000000..ab37ae4 --- /dev/null +++ b/scripts/mdbook_strip_frontmatter.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""mdBook preprocessor that strips YAML frontmatter from rendered chapters.""" + +from __future__ import annotations + +import json +import sys +from typing import Any + + +def strip_frontmatter(content: str) -> str: + """Remove a leading YAML frontmatter block from Markdown content.""" + if not content.startswith("---\n"): + return content + + end = content.find("\n---\n", 4) + if end == -1: + return content + + return content[end + len("\n---\n") :] + + +def strip_content_fields(value: Any) -> None: + """Recursively strip frontmatter from any mdBook chapter content field.""" + if isinstance(value, dict): + content = value.get("content") + if isinstance(content, str): + value["content"] = strip_frontmatter(content) + + for child in value.values(): + strip_content_fields(child) + return + + if isinstance(value, list): + for item in value: + strip_content_fields(item) + + +def main(argv: list[str]) -> int: + if len(argv) >= 2 and argv[1] == "supports": + return 0 + + _context, book = json.load(sys.stdin) + strip_content_fields(book) + json.dump(book, sys.stdout) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/scripts/tech_debt_allowlist.txt b/scripts/tech_debt_allowlist.txt new file mode 100644 index 0000000..6e74a45 --- /dev/null +++ b/scripts/tech_debt_allowlist.txt @@ -0,0 +1,2 @@ +# Add line substrings that should be ignored by check_tech_debt.sh. +# Keep this list minimal and review entries regularly. diff --git a/sophon-cli-architecture.html b/sophon-cli-architecture.html new file mode 100644 index 0000000..686eec3 --- /dev/null +++ b/sophon-cli-architecture.html @@ -0,0 +1,304 @@ + + + + + + sophon-cli Architecture Diagram + + + + +
+
+
+
+

sophon-cli Architecture

+
+

Module map and data flow from main.rs through the provider-agnostic search pipeline

+
+ +
+ + + + + + + + + + + + + + + + + args + + + + parse + + + + query + limit + + + + build + + + + build + + + + search() + + + + search_all() + + + + trait call + + + + trait call + + + + HTTP GET + + + + HTTP POST + + + + JSON + + + + JSON + + + + render + + + + init + + + + load + + + + + + Terminal + stdin / stdout / stderr + + + + + CliArgs + clap parser + + + + + main.rs + entrypoint · tokio::main + run_single_provider + + + + + output.rs + render · render_fanout + + + + + ProviderRegistry + bootstrap from env + + + + + SearchService + single provider + + + + + FanoutSearchService + parallel fan-out + + + + + Domain Layer + SearchProvider trait · SearchQuery + SearchResponse · error · types + + + + + BraveProvider + client · dto · mapper · config + + + + + ExaProvider + client · dto · mapper · config + + + + + ReqwestHttpClient + HttpClient trait impl + + + + + Brave Search API + api.search.brave.com + + + + + Exa API + api.exa.ai + + + + + tracing + EnvFilter → stderr + + + + + dotenvy + .env loader + + + Legend + + + CLI / Frontend + + + Application + + + Domain / Core + + + Provider / Transport + + + External API + + + Observability + + + Async call + +
+ + +
+
+
+
+

CLI Layer

+
+
    +
  • • CliArgs — clap derive parser with query, provider, and filters
    • +
  • • CliSearchType / CliSafeSearch / CliProvider value enums
    • +
  • • output.rs — render_text and render_fanout_text formatters
    • +
  • • --about flag and structured help text
    • +
+
+ +
+
+
+

Application Layer

+
+
    +
  • • main.rs — tokio::main entrypoint, run_single_provider helper
    • +
  • • ProviderRegistry — env-based builder registry (Brave + Exa)
    • +
  • • SearchService — single-provider orchestration wrapper
    • +
  • • FanoutSearchService — parallel search across all enabled providers
    • +
+
+ +
+
+
+

Provider & Transport Layer

+
+
    +
  • • BraveProvider & ExaProvider — SearchProvider trait impls
    • +
  • • Per-provider DTOs, mappers, and config modules
    • +
  • • HttpClient trait — abstracts GET/POST JSON transport
    • +
  • • ReqwestHttpClient — auth, rate-limit, and decode error handling
    • +
+
+
+ +

sophon-cli • Rust Edition 2024 • Provider-agnostic search CLI

+
+ + diff --git a/src/app/fanout_search_service.rs b/src/app/fanout_search_service.rs new file mode 100644 index 0000000..158a0d4 --- /dev/null +++ b/src/app/fanout_search_service.rs @@ -0,0 +1,128 @@ +use crate::domain::{ + ProviderSearchFailure, SearchBatchResponse, SearchProvider, SearchQuery, SearchResponse, +}; + +pub struct FanoutSearchService { + providers: Vec>, +} + +impl FanoutSearchService { + pub fn new(providers: Vec>) -> Self { + Self { providers } + } + + pub async fn search_all(&self, query: SearchQuery) -> SearchBatchResponse { + let mut responses: Vec = Vec::new(); + let mut failures: Vec = Vec::new(); + + for provider in &self.providers { + let provider_id = provider.id(); + match provider.search(&query).await { + Ok(response) => responses.push(response), + Err(error) => failures.push(ProviderSearchFailure { + provider: provider_id, + error, + }), + } + } + + SearchBatchResponse { + query: query.text, + responses, + failures, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::domain::{ + ProviderCapabilities, SearchError, SearchProvider, SearchQuery, SearchResponse, SearchType, + }; + use async_trait::async_trait; + + enum MockOutcome { + Success(SearchResponse), + Failure(&'static str), + } + + struct MockProvider { + id: &'static str, + outcome: MockOutcome, + } + + #[async_trait] + impl SearchProvider for MockProvider { + fn id(&self) -> String { + self.id.to_string() + } + + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + web: true, + ..ProviderCapabilities::default() + } + } + + async fn search(&self, _query: &SearchQuery) -> Result { + match &self.outcome { + MockOutcome::Success(response) => Ok(response.clone()), + MockOutcome::Failure(message) => Err(SearchError::Provider(message.to_string())), + } + } + } + + fn response(provider: &str) -> SearchResponse { + SearchResponse { + query: "rust".to_string(), + provider: provider.to_string(), + results: vec![], + total_estimated: None, + next_page: None, + } + } + + fn query() -> SearchQuery { + SearchQuery { + text: "rust".to_string(), + search_type: SearchType::Web, + limit: None, + offset: None, + safe_search: None, + country: None, + language: None, + time_range: None, + } + } + + #[tokio::test] + async fn search_all_preserves_provider_order_and_records_failures() { + let service = FanoutSearchService::new(vec![ + Box::new(MockProvider { + id: "brave", + outcome: MockOutcome::Success(response("brave")), + }), + Box::new(MockProvider { + id: "broken", + outcome: MockOutcome::Failure("temporary outage"), + }), + Box::new(MockProvider { + id: "exa", + outcome: MockOutcome::Success(response("exa")), + }), + ]); + + let batch = service.search_all(query()).await; + + assert_eq!(batch.query, "rust"); + assert_eq!(batch.responses[0].provider, "brave"); + assert_eq!(batch.responses[1].provider, "exa"); + assert_eq!(batch.failures.len(), 1); + assert_eq!(batch.failures[0].provider, "broken"); + assert_eq!( + batch.failures[0].error.to_string(), + "provider error: temporary outage" + ); + } +} diff --git a/src/app/mod.rs b/src/app/mod.rs index e0a164b..577c446 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -1 +1,2 @@ +pub mod fanout_search_service; pub mod search_service; diff --git a/src/app/search_service.rs b/src/app/search_service.rs index ed06923..1f8055f 100644 --- a/src/app/search_service.rs +++ b/src/app/search_service.rs @@ -1,7 +1,4 @@ -use crate::domain::error::SearchError; -use crate::domain::provider::SearchProvider; -use crate::domain::query::SearchQuery; -use crate::domain::result::SearchResponse; +use crate::domain::{SearchError, SearchProvider, SearchQuery, SearchResponse}; pub struct SearchService { provider: Box, @@ -12,16 +9,21 @@ impl SearchService { Self { provider } } + #[tracing::instrument(skip(self), fields(query = %query.text, provider = %self.provider.id()))] pub async fn search(&self, query: SearchQuery) -> Result { - self.provider.search(&query).await + tracing::debug!("delegating search to provider"); + let result = self.provider.search(&query).await; + if let Err(ref e) = result { + tracing::warn!(error = %e, "provider search returned error"); + } + result } } #[cfg(test)] mod tests { use super::*; - use crate::domain::provider::ProviderCapabilities; - use crate::domain::types::SearchType; + use crate::domain::{ProviderCapabilities, SearchType}; use async_trait::async_trait; struct MockProvider { @@ -37,12 +39,7 @@ mod tests { fn capabilities(&self) -> ProviderCapabilities { ProviderCapabilities { web: true, - news: false, - images: false, - videos: false, - pagination: false, - safe_search: false, - time_range_filter: false, + ..ProviderCapabilities::default() } } diff --git a/src/bootstrap/mod.rs b/src/bootstrap/mod.rs new file mode 100644 index 0000000..0fe533b --- /dev/null +++ b/src/bootstrap/mod.rs @@ -0,0 +1 @@ +pub mod provider_registry; diff --git a/src/bootstrap/provider_registry.rs b/src/bootstrap/provider_registry.rs new file mode 100644 index 0000000..3427ae4 --- /dev/null +++ b/src/bootstrap/provider_registry.rs @@ -0,0 +1,316 @@ +use std::collections::HashMap; +use std::fmt; + +use crate::app::{fanout_search_service::FanoutSearchService, search_service::SearchService}; +use crate::domain::SearchProvider; +use crate::providers::{ + brave::{client::BraveProvider, config::BraveConfig}, + exa::{client::ExaProvider, config::ExaConfig}, +}; +use crate::transport::http::ReqwestHttpClient; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ProviderId { + Brave, + Exa, +} + +impl fmt::Display for ProviderId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ProviderId::Brave => formatter.write_str("brave"), + ProviderId::Exa => formatter.write_str("exa"), + } + } +} + +pub type ProviderBuilder = Box Box + Send + Sync>; + +pub struct ProviderRegistry { + builders: HashMap, +} + +#[derive(Debug, thiserror::Error)] +pub enum BuildSearchServiceError { + #[error("provider `{provider}` is unavailable; configured providers: {available:?}")] + ProviderUnavailable { + provider: ProviderId, + available: Vec, + }, + #[error("no configured providers; set BRAVE_API_KEY and/or EXA_API_KEY")] + NoProvidersAvailable, +} + +impl ProviderRegistry { + pub fn empty() -> Self { + Self { + builders: HashMap::new(), + } + } + + pub fn production_from_env() -> Self { + let mut registry = Self::empty(); + + match BraveConfig::from_env() { + Ok(config) => { + registry.register( + ProviderId::Brave, + Box::new(move || { + Box::new(BraveProvider::new(ReqwestHttpClient::new(), config.clone())) + }), + ); + } + Err(std::env::VarError::NotPresent | std::env::VarError::NotUnicode(_)) => {} + } + + match ExaConfig::from_env() { + Ok(config) => { + registry.register( + ProviderId::Exa, + Box::new(move || { + Box::new(ExaProvider::new(ReqwestHttpClient::new(), config.clone())) + }), + ); + } + Err(std::env::VarError::NotPresent | std::env::VarError::NotUnicode(_)) => {} + } + + registry + } + + pub fn register(&mut self, id: ProviderId, builder: ProviderBuilder) { + self.builders.insert(id, builder); + } + + pub fn available_providers(&self) -> Vec { + [ProviderId::Brave, ProviderId::Exa] + .into_iter() + .filter(|id| self.builders.contains_key(id)) + .collect() + } + + pub fn build(&self, provider: ProviderId) -> Result { + let builder = self.builders.get(&provider).ok_or_else(|| { + BuildSearchServiceError::ProviderUnavailable { + provider, + available: self.available_providers(), + } + })?; + + Ok(SearchService::new(builder())) + } + + pub fn build_all_enabled(&self) -> Result { + let provider_ids = self.available_providers(); + if provider_ids.is_empty() { + return Err(BuildSearchServiceError::NoProvidersAvailable); + } + + let providers = provider_ids + .into_iter() + .map(|provider_id| { + let builder = self + .builders + .get(&provider_id) + .expect("available provider has a registered builder"); + builder() + }) + .collect(); + + Ok(FanoutSearchService::new(providers)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::domain::{ + ProviderCapabilities, SearchError, SearchQuery, SearchResponse, SearchType, + }; + use async_trait::async_trait; + use std::ffi::OsString; + use std::sync::{Mutex, OnceLock}; + + struct MockProvider; + + #[async_trait] + impl SearchProvider for MockProvider { + fn id(&self) -> String { + "mock".to_string() + } + + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + web: true, + news: true, + images: false, + videos: false, + pagination: false, + safe_search: false, + time_range_filter: false, + } + } + + async fn search(&self, _query: &SearchQuery) -> Result { + Ok(SearchResponse { + query: "mock".to_string(), + provider: "mock".to_string(), + results: vec![], + total_estimated: None, + next_page: None, + }) + } + } + + struct NamedMockProvider { + id: &'static str, + } + + #[async_trait] + impl SearchProvider for NamedMockProvider { + fn id(&self) -> String { + self.id.to_string() + } + + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + web: true, + news: true, + images: false, + videos: false, + pagination: false, + safe_search: false, + time_range_filter: false, + } + } + + async fn search(&self, query: &SearchQuery) -> Result { + Ok(SearchResponse { + query: query.text.clone(), + provider: self.id.to_string(), + results: vec![], + total_estimated: None, + next_page: None, + }) + } + } + + fn search_query() -> SearchQuery { + SearchQuery { + text: "rust".to_string(), + search_type: SearchType::Web, + limit: None, + offset: None, + safe_search: None, + country: None, + language: None, + time_range: None, + } + } + + fn env_lock() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + } + + fn restore_env_var(name: &str, value: Option) { + match value { + Some(value) => unsafe { + std::env::set_var(name, value); + }, + None => unsafe { + std::env::remove_var(name); + }, + } + } + + #[test] + fn empty_registry_reports_provider_unavailable() { + let registry = ProviderRegistry::empty(); + + match registry.build(ProviderId::Brave) { + Err(BuildSearchServiceError::ProviderUnavailable { + provider, + available, + }) => { + assert_eq!(provider, ProviderId::Brave); + assert!(available.is_empty()); + } + Ok(_) => panic!("expected ProviderUnavailable error"), + Err(other) => panic!("expected ProviderUnavailable error, got {other}"), + } + } + + #[test] + fn production_registry_only_includes_configured_providers() { + let _guard = env_lock().lock().unwrap(); + let original_brave = std::env::var_os("BRAVE_API_KEY"); + let original_exa = std::env::var_os("EXA_API_KEY"); + + unsafe { + std::env::set_var("BRAVE_API_KEY", "test-brave-key"); + std::env::remove_var("EXA_API_KEY"); + } + + let registry = ProviderRegistry::production_from_env(); + + restore_env_var("BRAVE_API_KEY", original_brave); + restore_env_var("EXA_API_KEY", original_exa); + + assert_eq!(registry.available_providers(), vec![ProviderId::Brave]); + } + + #[test] + fn registered_provider_is_available_and_builds_service() { + let mut registry = ProviderRegistry::empty(); + registry.register(ProviderId::Brave, Box::new(|| Box::new(MockProvider))); + + assert_eq!(registry.available_providers(), vec![ProviderId::Brave]); + assert!(registry.build(ProviderId::Brave).is_ok()); + } + + #[test] + fn available_providers_are_returned_in_stable_order() { + let mut registry = ProviderRegistry::empty(); + registry.register(ProviderId::Exa, Box::new(|| Box::new(MockProvider))); + registry.register(ProviderId::Brave, Box::new(|| Box::new(MockProvider))); + + assert_eq!( + registry.available_providers(), + vec![ProviderId::Brave, ProviderId::Exa] + ); + } + + #[tokio::test] + async fn build_all_enabled_uses_stable_order_and_rejects_empty_registry() { + let empty = ProviderRegistry::empty(); + match empty.build_all_enabled() { + Err(BuildSearchServiceError::NoProvidersAvailable) => {} + Ok(_) => panic!("expected NoProvidersAvailable error"), + Err(other) => panic!("expected NoProvidersAvailable error, got {other}"), + } + + let mut registry = ProviderRegistry::empty(); + registry.register( + ProviderId::Exa, + Box::new(|| Box::new(NamedMockProvider { id: "exa" })), + ); + registry.register( + ProviderId::Brave, + Box::new(|| Box::new(NamedMockProvider { id: "brave" })), + ); + + let service = registry + .build_all_enabled() + .expect("fan-out service builds"); + let batch = service.search_all(search_query()).await; + + let providers: Vec<&str> = batch + .responses + .iter() + .map(|response| response.provider.as_str()) + .collect(); + assert_eq!(providers, vec!["brave", "exa"]); + assert!(batch.failures.is_empty()); + } +} diff --git a/src/cli/args.rs b/src/cli/args.rs index bc4601e..3d5a4c3 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -1,6 +1,7 @@ -use crate::domain::types::SearchType; use clap::{Parser, ValueEnum}; +use crate::domain::{SafeSearch, SearchType}; + #[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)] pub enum CliSearchType { Web, @@ -27,12 +28,12 @@ pub enum CliSafeSearch { Strict, } -impl From for crate::domain::types::SafeSearch { +impl From for SafeSearch { fn from(val: CliSafeSearch) -> Self { match val { - CliSafeSearch::Off => crate::domain::types::SafeSearch::Off, - CliSafeSearch::Moderate => crate::domain::types::SafeSearch::Moderate, - CliSafeSearch::Strict => crate::domain::types::SafeSearch::Strict, + CliSafeSearch::Off => SafeSearch::Off, + CliSafeSearch::Moderate => SafeSearch::Moderate, + CliSafeSearch::Strict => SafeSearch::Strict, } } } @@ -41,6 +42,7 @@ impl From for crate::domain::types::SafeSearch { pub enum CliProvider { Brave, Exa, + All, } #[derive(Parser, Debug)] @@ -73,6 +75,21 @@ pub struct CliArgs { #[arg(long)] pub language: Option, + + /// After search, scrape each deduplicated result URL with headless Chrome (Spider). + #[arg(long)] + pub scrape: bool, + + /// Max pages to crawl per result URL seed when scraping; this does not limit search results. + #[arg( + long, + help = "Max pages to crawl per result URL seed; does not limit search results" + )] + pub scrape_limit: Option, + + /// Seconds before giving up scraping a single result URL seed (default when omitted: 120). + #[arg(long)] + pub scrape_timeout_seconds: Option, } #[cfg(test)] @@ -89,4 +106,41 @@ mod tests { let default_args = CliArgs::parse_from(["sophon-cli", "rust"]); assert_eq!(default_args.provider, CliProvider::Brave); } + + #[test] + fn test_cli_provider_parses_all_and_defaults_to_brave() { + let all_args = CliArgs::try_parse_from(["sophon-cli", "rust", "--provider", "all"]) + .expect("all provider parses"); + assert_eq!(all_args.provider, CliProvider::All); + + let default_args = + CliArgs::try_parse_from(["sophon-cli", "rust"]).expect("default provider parses"); + assert_eq!(default_args.provider, CliProvider::Brave); + } + + #[test] + fn test_cli_parse_scrape_defaults_and_explicit_flags() { + let baseline = CliArgs::try_parse_from(["sophon-cli", "rust"]).expect("parses query"); + assert!(!baseline.scrape); + assert!(baseline.scrape_limit.is_none()); + assert!(baseline.scrape_timeout_seconds.is_none()); + + let with_scrape = + CliArgs::try_parse_from(["sophon-cli", "rust", "--scrape"]).expect("parses scrape"); + assert!(with_scrape.scrape); + + let full = CliArgs::try_parse_from([ + "sophon-cli", + "--scrape", + "--scrape-limit", + "5", + "--scrape-timeout-seconds", + "30", + "rust", + ]) + .expect("parses scrape options"); + assert!(full.scrape); + assert_eq!(full.scrape_limit, Some(5)); + assert_eq!(full.scrape_timeout_seconds, Some(30)); + } } diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 3f2a241..b5354c0 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,2 +1,5 @@ pub mod args; pub mod output; +pub mod request; +pub mod runner; +pub mod scrape; diff --git a/src/cli/output.rs b/src/cli/output.rs index 80b9a90..1b8a4a5 100644 --- a/src/cli/output.rs +++ b/src/cli/output.rs @@ -1,4 +1,7 @@ -use crate::domain::result::{SearchResponse, SearchResult}; +use crate::cli::scrape::ScrapedSite; +use crate::domain::{SearchBatchResponse, SearchResponse, SearchResult}; + +const SCRAPED_CONTENT_PREVIEW_CHARS: usize = 2_000; pub fn render_text(response: &SearchResponse) -> String { let mut lines = vec![ @@ -13,7 +16,10 @@ pub fn render_text(response: &SearchResponse) -> String { lines.push(format!("{}. [{}]", i + 1, r.title)); lines.push(format!(" URL: {}", r.url)); if let Some(s) = &r.snippet { - lines.push(format!(" {}", s)); + let t = s.trim(); + if !t.is_empty() { + lines.push(format!(" {}", t)); + } } } SearchResult::News(r) => { @@ -22,6 +28,12 @@ pub fn render_text(response: &SearchResponse) -> String { if let Some(s) = &r.source { lines.push(format!(" Source: {}", s)); } + if let Some(s) = &r.snippet { + let t = s.trim(); + if !t.is_empty() { + lines.push(format!(" {}", t)); + } + } } SearchResult::Image(r) => { lines.push(format!("{}. [IMAGE] {}", i + 1, r.title)); @@ -37,10 +49,84 @@ pub fn render_text(response: &SearchResponse) -> String { lines.join("\n") } +pub fn render_scraped_sites(sites: &[ScrapedSite]) -> String { + let mut lines = vec!["== Scraped pages ==".to_string(), String::new()]; + for site in sites { + lines.push(format!("Seed: {}", site.seed_url)); + lines.push(format!("Time: {:?}", site.duration)); + lines.push(format!("Page limit per seed: {}", site.page_limit)); + if let Some(err) = &site.error { + lines.push(format!("Error: {err}")); + } else { + lines.push(format!("Extracted content pages: {}", site.pages.len())); + for (i, page) in site.pages.iter().enumerate() { + lines.push(format!(" Page {}: {}", i + 1, page.url)); + lines.push(format!(" Status: {}", page.status_code)); + lines.push(" Content excerpt:".to_string()); + for line in preview_content(&page.content).lines() { + lines.push(format!(" {}", line)); + } + } + + lines.push(String::new()); + lines.push(format!( + "Crawl telemetry - visited URLs: {}", + site.visited_urls.len() + )); + for url in &site.visited_urls { + lines.push(format!(" - {}", url)); + } + } + lines.push(String::new()); + } + lines.join("\n") +} + +fn preview_content(content: &str) -> String { + let trimmed = content.trim(); + if trimmed.chars().count() <= SCRAPED_CONTENT_PREVIEW_CHARS { + return trimmed.to_string(); + } + + let mut preview: String = trimmed + .chars() + .take(SCRAPED_CONTENT_PREVIEW_CHARS) + .collect(); + preview.push_str("\n ... [content truncated]"); + preview +} + +pub fn render_fanout_text(response: &SearchBatchResponse) -> String { + let mut lines = vec![ + format!("Query: {}", response.query), + format!("Providers succeeded: {}", response.responses.len()), + format!("Providers failed: {}", response.failures.len()), + String::new(), + ]; + + for provider_response in &response.responses { + lines.push(format!("== {} ==", provider_response.provider)); + lines.push(render_text(provider_response)); + lines.push(String::new()); + } + + if !response.failures.is_empty() { + lines.push("== Failures ==".to_string()); + for failure in &response.failures { + lines.push(format!("- {}: {}", failure.provider, failure.error)); + } + } + + lines.join("\n") +} + #[cfg(test)] mod tests { use super::*; - use crate::domain::result::*; + use crate::cli::scrape::ScrapedSite; + use crate::domain::{ + ImageResult, NewsResult, ProviderSearchFailure, SearchError, VideoResult, WebResult, + }; #[test] fn test_render_text_mixed_results() { @@ -57,7 +143,7 @@ mod tests { SearchResult::News(NewsResult { title: "Rust News".to_string(), url: "https://example.com/news".to_string(), - snippet: None, + snippet: Some("Breaking update".to_string()), source: Some("Example".to_string()), published_at: None, }), @@ -84,7 +170,74 @@ mod tests { assert!(text.contains("Rust Lang")); assert!(text.contains("https://rust-lang.org")); assert!(text.contains("[NEWS] Rust News")); + assert!(text.contains("Breaking update")); assert!(text.contains("[IMAGE] Rust Logo")); assert!(text.contains("[VIDEO] Rust Tutorial")); } + + #[test] + fn test_render_scraped_sites_shows_content_separate_from_visited_urls() { + let ok = ScrapedSite { + seed_url: "https://example.com".into(), + duration: std::time::Duration::from_millis(100), + page_limit: 5, + pages: vec![crate::cli::scrape::ScrapedPage { + url: "https://example.com".into(), + status_code: 200, + content: "Real scraped page body".into(), + }], + visited_urls: vec!["https://example.com/a".into()], + error: None, + }; + let err = ScrapedSite { + seed_url: "https://bad".into(), + duration: std::time::Duration::from_secs(1), + page_limit: 5, + pages: vec![], + visited_urls: vec![], + error: Some("boom".into()), + }; + let text = render_scraped_sites(&[ok, err]); + assert!(text.contains("== Scraped pages ==")); + assert!(text.contains("Seed: https://example.com")); + assert!(text.contains("Page limit per seed: 5")); + assert!(text.contains("Extracted content pages: 1")); + assert!(text.contains("Content excerpt:")); + assert!(text.contains("Real scraped page body")); + assert!(text.contains("Crawl telemetry - visited URLs: 1")); + assert!(text.contains("https://example.com/a")); + assert!(text.contains("Error: boom")); + } + + #[test] + fn test_render_fanout_text_includes_successes_and_failures() { + let response = SearchBatchResponse { + query: "rust".to_string(), + responses: vec![SearchResponse { + query: "rust".to_string(), + provider: "brave".to_string(), + results: vec![SearchResult::Web(WebResult { + title: "Rust Lang".to_string(), + url: "https://rust-lang.org".to_string(), + snippet: Some("Safe systems".to_string()), + display_url: None, + })], + total_estimated: None, + next_page: None, + }], + failures: vec![ProviderSearchFailure { + provider: "exa".to_string(), + error: SearchError::InvalidQuery("unsupported".to_string()), + }], + }; + + let text = render_fanout_text(&response); + + assert!(text.contains("Query: rust")); + assert!(text.contains("Providers succeeded: 1")); + assert!(text.contains("Providers failed: 1")); + assert!(text.contains("== brave ==")); + assert!(text.contains("Rust Lang")); + assert!(text.contains("- exa: invalid query: unsupported")); + } } diff --git a/src/cli/request.rs b/src/cli/request.rs new file mode 100644 index 0000000..53aaf1c --- /dev/null +++ b/src/cli/request.rs @@ -0,0 +1,53 @@ +use crate::cli::args::CliArgs; +use crate::domain::SearchQuery; + +pub fn build_search_query(query_text: String, args: &CliArgs) -> SearchQuery { + SearchQuery { + text: query_text, + search_type: args.search_type.into(), + limit: args.limit, + offset: args.offset, + safe_search: args.safe_search.map(|safe_search| safe_search.into()), + country: args.country.clone(), + language: args.language.clone(), + time_range: None, + } +} + +#[cfg(test)] +mod tests { + use crate::cli::args::{CliArgs, CliProvider, CliSafeSearch, CliSearchType}; + use crate::cli::request::build_search_query; + use crate::domain::{SafeSearch, SearchType}; + + #[test] + fn build_search_query_maps_all_supported_fields() { + let args = CliArgs { + query: Some("ignored parsed query".to_string()), + about: false, + search_type: CliSearchType::News, + provider: CliProvider::All, + limit: Some(7), + offset: Some(14), + safe_search: Some(CliSafeSearch::Strict), + country: Some("US".to_string()), + language: Some("en".to_string()), + scrape: false, + scrape_limit: None, + scrape_timeout_seconds: None, + }; + + let query = build_search_query("rust search".to_string(), &args); + + assert_eq!(query.text, "rust search"); + assert_eq!(query.search_type, SearchType::News); + assert_eq!(query.limit, Some(7)); + assert_eq!(query.offset, Some(14)); + assert_eq!(query.safe_search, Some(SafeSearch::Strict)); + assert_eq!(query.country, Some("US".to_string())); + assert_eq!(query.language, Some("en".to_string())); + assert_eq!(query.time_range, None); + assert_eq!(args.country, Some("US".to_string())); + assert_eq!(args.language, Some("en".to_string())); + } +} diff --git a/src/cli/runner.rs b/src/cli/runner.rs new file mode 100644 index 0000000..715828c --- /dev/null +++ b/src/cli/runner.rs @@ -0,0 +1,131 @@ +use clap::Parser; + +use crate::bootstrap::provider_registry::{ProviderId, ProviderRegistry}; +use crate::cli::args::{CliArgs, CliProvider}; +use crate::cli::output::{render_fanout_text, render_scraped_sites, render_text}; +use crate::cli::request::build_search_query; +use crate::cli::scrape::{ + deduped_urls_from_batch, deduped_urls_from_response, page_limit_for_scrape, scrape_seed_urls, + timeout_duration_for_scrape, +}; +use crate::domain::SearchQuery; + +pub async fn run_from_env() -> i32 { + let args = CliArgs::parse(); + run(args).await +} + +pub async fn run(args: CliArgs) -> i32 { + if args.about { + print_about(); + return 0; + } + + let query_text = match args.query.clone() { + Some(query) => query, + None => { + eprintln!("Error: missing query. Use --help for usage or --about for more info."); + return 1; + } + }; + + let query = build_search_query(query_text, &args); + let registry = ProviderRegistry::production_from_env(); + + match args.provider { + CliProvider::Brave => run_single_provider(®istry, ProviderId::Brave, query, &args).await, + CliProvider::Exa => run_single_provider(®istry, ProviderId::Exa, query, &args).await, + CliProvider::All => run_all_enabled(®istry, query, &args).await, + } +} + +fn print_about() { + println!("sophon-cli — a provider-agnostic search CLI"); + println!(); + println!("Named after the Sophon from Cixin Liu's Three-Body Problem trilogy:"); + println!("a sentient proton supercomputer that performs near-infinite computation"); + println!("across vast distances. This tiny CLI delegates its heavy lifting to"); + println!("distant search APIs the same way."); + println!(); + println!("Currently supports Brave Search (web, news, images, video) and Exa."); +} + +async fn run_single_provider( + registry: &ProviderRegistry, + provider_id: ProviderId, + query: SearchQuery, + args: &CliArgs, +) -> i32 { + tracing::info!(provider = %provider_id, query = %query.text, "initializing search service"); + + let service = match registry.build(provider_id) { + Ok(service) => service, + Err(error) => { + tracing::error!(%error, "failed to build provider"); + eprintln!("{error}"); + return 1; + } + }; + + match service.search(query).await { + Ok(response) => { + tracing::info!(result_count = response.results.len(), total_estimated = ?response.total_estimated, "search completed"); + let mut printed = render_text(&response); + if args.scrape { + let seeds = deduped_urls_from_response(&response); + let scraped = scrape_seed_urls( + &seeds, + page_limit_for_scrape(args), + timeout_duration_for_scrape(args), + ) + .await; + if !scraped.is_empty() { + printed.push('\n'); + printed.push_str(&render_scraped_sites(&scraped)); + } + } + println!("{}", printed); + 0 + } + Err(error) => { + tracing::error!(error = %error, "search failed"); + eprintln!("Search failed: {}", error); + 1 + } + } +} + +async fn run_all_enabled(registry: &ProviderRegistry, query: SearchQuery, args: &CliArgs) -> i32 { + tracing::info!(query = %query.text, "initializing all-enabled provider fan-out service"); + let service = match registry.build_all_enabled() { + Ok(service) => service, + Err(error) => { + tracing::error!(%error, "failed to build fan-out providers"); + eprintln!("{error}"); + return 1; + } + }; + + let response = service.search_all(query).await; + tracing::info!( + successful_providers = response.responses.len(), + failed_providers = response.failures.len(), + "fan-out search completed" + ); + let mut printed = render_fanout_text(&response); + if args.scrape { + let seeds = deduped_urls_from_batch(&response); + let scraped = scrape_seed_urls( + &seeds, + page_limit_for_scrape(args), + timeout_duration_for_scrape(args), + ) + .await; + if !scraped.is_empty() { + printed.push('\n'); + printed.push_str(&render_scraped_sites(&scraped)); + } + } + println!("{}", printed); + if response.responses.is_empty() { 1 } else { 0 } +} diff --git a/src/cli/scrape.rs b/src/cli/scrape.rs new file mode 100644 index 0000000..998c88e --- /dev/null +++ b/src/cli/scrape.rs @@ -0,0 +1,328 @@ +//! Post-search URL extraction and optional Spider/Chrome crawling (CLI layer only). + +use std::collections::HashSet; +use std::time::{Duration, Instant}; + +use spider::features::chrome_common::RequestInterceptConfiguration; +use spider::website::Website; + +use crate::domain::{SearchBatchResponse, SearchResponse, SearchResult}; + +/// Default max pages per seed when `--scrape-limit` is omitted. +pub const DEFAULT_SCRAPE_PAGE_LIMIT: u32 = 10; + +/// Default per-seed scrape timeout when `--scrape-timeout-seconds` is omitted. +pub const DEFAULT_SCRAPE_TIMEOUT_SECS: u64 = 120; + +pub fn page_limit_for_scrape(cli: &crate::cli::args::CliArgs) -> u32 { + cli.scrape_limit + .and_then(|n| u32::try_from(n).ok()) + .unwrap_or(DEFAULT_SCRAPE_PAGE_LIMIT) +} + +pub fn timeout_duration_for_scrape(cli: &crate::cli::args::CliArgs) -> Duration { + Duration::from_secs( + cli.scrape_timeout_seconds + .unwrap_or(DEFAULT_SCRAPE_TIMEOUT_SECS), + ) +} + +/// Extract result URLs in search order. +pub fn urls_from_results(results: &[SearchResult]) -> Vec { + results + .iter() + .map(|r| url_from_result(r).to_string()) + .collect() +} + +fn url_from_result(r: &SearchResult) -> &str { + match r { + SearchResult::Web(w) => &w.url, + SearchResult::News(n) => &n.url, + SearchResult::Image(i) => &i.url, + SearchResult::Video(v) => &v.url, + } +} + +/// Deduplicate URLs preserving first-seen order. +pub fn dedup_ordered(urls: Vec) -> Vec { + let mut seen = HashSet::::new(); + let mut out = Vec::new(); + for u in urls { + if seen.insert(u.clone()) { + out.push(u); + } + } + out +} + +pub fn deduped_urls_from_response(response: &SearchResponse) -> Vec { + dedup_ordered(urls_from_results(&response.results)) +} + +pub fn deduped_urls_from_batch(batch: &SearchBatchResponse) -> Vec { + let mut urls = Vec::new(); + for response in &batch.responses { + urls.extend(urls_from_results(&response.results)); + } + dedup_ordered(urls) +} + +/// One page captured by Spider while crawling a seed URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScrapedPage { + pub url: String, + pub status_code: u16, + pub content: String, +} + +/// One scraped seed URL: content pages plus crawl telemetry from Spider. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScrapedSite { + pub seed_url: String, + pub duration: Duration, + pub page_limit: u32, + pub pages: Vec, + pub visited_urls: Vec, + pub error: Option, +} + +struct CrawlResult { + pages: Vec, + visited_urls: Vec, +} + +/// Scrape each seed sequentially with a per-seed timeout. +pub async fn scrape_seed_urls( + seeds: &[String], + page_limit_per_site: u32, + timeout_per_seed: Duration, +) -> Vec { + let mut out = Vec::with_capacity(seeds.len()); + for seed in seeds { + let start = Instant::now(); + let crawl = + tokio::time::timeout(timeout_per_seed, crawl_one_seed(seed, page_limit_per_site)); + match crawl.await { + Err(_) => { + out.push(ScrapedSite { + seed_url: seed.clone(), + duration: start.elapsed(), + page_limit: page_limit_per_site, + pages: vec![], + visited_urls: vec![], + error: Some(format!("timed out after {timeout_per_seed:?}")), + }); + } + Ok(Ok(crawl_result)) => { + out.push(ScrapedSite { + seed_url: seed.clone(), + duration: start.elapsed(), + page_limit: page_limit_per_site, + pages: crawl_result.pages, + visited_urls: crawl_result.visited_urls, + error: None, + }); + } + Ok(Err(message)) => { + out.push(ScrapedSite { + seed_url: seed.clone(), + duration: start.elapsed(), + page_limit: page_limit_per_site, + pages: vec![], + visited_urls: vec![], + error: Some(message), + }); + } + } + } + out +} + +async fn crawl_one_seed(seed: &str, page_limit: u32) -> Result { + let spider_page_limit = page_limit.max(2); + let mut website: Website = Website::new(seed) + .with_limit(spider_page_limit) + .with_chrome_intercept(RequestInterceptConfiguration::new(true)) + .with_stealth(true) + .build() + .map_err(|e| e.to_string())?; + + website.scrape().await; + + let mut pages: Vec = website + .get_pages() + .map(|pages| { + pages + .iter() + .map(|page| ScrapedPage { + url: page.get_url().to_string(), + status_code: page.status_code.as_u16(), + content: page.get_content(), + }) + .collect() + }) + .unwrap_or_default(); + pages.truncate(page_limit as usize); + + let links = website.get_all_links_visited().await; + let mut visited_urls: Vec = links.into_iter().map(|u| u.to_string()).collect(); + visited_urls.truncate(page_limit as usize); + Ok(CrawlResult { + pages, + visited_urls, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::domain::{ + ImageResult, NewsResult, ProviderSearchFailure, SearchBatchResponse, SearchError, + SearchResponse, SearchResult, VideoResult, WebResult, + }; + + #[test] + fn urls_from_results_collects_each_variant_url() { + let results = vec![ + SearchResult::Web(WebResult { + title: "w".into(), + url: "https://a".into(), + snippet: None, + display_url: None, + }), + SearchResult::News(NewsResult { + title: "n".into(), + url: "https://b".into(), + snippet: None, + source: None, + published_at: None, + }), + SearchResult::Image(ImageResult { + title: "i".into(), + url: "https://c.png".into(), + thumbnail_url: None, + source: None, + }), + SearchResult::Video(VideoResult { + title: "v".into(), + url: "https://d".into(), + thumbnail_url: None, + duration: None, + published_at: None, + }), + ]; + assert_eq!( + urls_from_results(&results), + vec![ + "https://a".to_string(), + "https://b".to_string(), + "https://c.png".to_string(), + "https://d".to_string(), + ] + ); + } + + #[test] + fn dedup_ordered_keeps_first_occurrence_order() { + let d = dedup_ordered(vec![ + "https://x".into(), + "https://y".into(), + "https://x".into(), + "https://z".into(), + "https://y".into(), + ]); + assert_eq!( + d, + vec![ + "https://x".to_string(), + "https://y".to_string(), + "https://z".to_string(), + ] + ); + } + + #[test] + fn deduped_urls_from_response_preserves_order_and_dedupes() { + let response = SearchResponse { + query: "rust".to_string(), + provider: "brave".to_string(), + results: vec![ + SearchResult::Web(WebResult { + title: "a".into(), + url: "https://dup".into(), + snippet: None, + display_url: None, + }), + SearchResult::News(NewsResult { + title: "b".into(), + url: "https://dup".into(), + snippet: None, + source: None, + published_at: None, + }), + SearchResult::Web(WebResult { + title: "c".into(), + url: "https://only-once".into(), + snippet: None, + display_url: None, + }), + ], + total_estimated: None, + next_page: None, + }; + assert_eq!( + deduped_urls_from_response(&response), + vec!["https://dup".to_string(), "https://only-once".to_string(),] + ); + } + + #[test] + fn deduped_urls_from_batch_flattens_providers_then_dedupes() { + let batch = SearchBatchResponse { + query: "q".into(), + responses: vec![ + SearchResponse { + query: "q".into(), + provider: "brave".into(), + results: vec![SearchResult::Web(WebResult { + title: "1".into(), + url: "https://dup".into(), + snippet: None, + display_url: None, + })], + total_estimated: None, + next_page: None, + }, + SearchResponse { + query: "q".into(), + provider: "exa".into(), + results: vec![ + SearchResult::Web(WebResult { + title: "2".into(), + url: "https://dup".into(), + snippet: None, + display_url: None, + }), + SearchResult::Web(WebResult { + title: "3".into(), + url: "https://only-exa".into(), + snippet: None, + display_url: None, + }), + ], + total_estimated: None, + next_page: None, + }, + ], + failures: vec![ProviderSearchFailure { + provider: "broken".into(), + error: SearchError::Provider("err".into()), + }], + }; + assert_eq!( + deduped_urls_from_batch(&batch), + vec!["https://dup".to_string(), "https://only-exa".to_string(),] + ); + } +} diff --git a/src/domain/mod.rs b/src/domain/mod.rs index 8a088c3..964cc7b 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -3,3 +3,12 @@ pub mod provider; pub mod query; pub mod result; pub mod types; + +pub use error::SearchError; +pub use provider::{ProviderCapabilities, SearchProvider}; +pub use query::SearchQuery; +pub use result::{ + ImageResult, NewsResult, PageToken, ProviderSearchFailure, SearchBatchResponse, SearchResponse, + SearchResult, VideoResult, WebResult, +}; +pub use types::{SafeSearch, SearchType, TimeRange}; diff --git a/src/domain/provider.rs b/src/domain/provider.rs index 88dab2a..a4fad4a 100644 --- a/src/domain/provider.rs +++ b/src/domain/provider.rs @@ -1,9 +1,8 @@ -use crate::domain::error::SearchError; -use crate::domain::query::SearchQuery; -use crate::domain::result::SearchResponse; use async_trait::async_trait; -#[derive(Debug, Clone)] +use super::{SearchError, SearchQuery, SearchResponse}; + +#[derive(Debug, Clone, Default)] #[allow(dead_code)] pub struct ProviderCapabilities { pub web: bool, diff --git a/src/domain/query.rs b/src/domain/query.rs index 2fc02f6..6127932 100644 --- a/src/domain/query.rs +++ b/src/domain/query.rs @@ -1,4 +1,4 @@ -use crate::domain::types::*; +use super::{SafeSearch, SearchType, TimeRange}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct SearchQuery { diff --git a/src/domain/result.rs b/src/domain/result.rs index 572df8d..290aa1b 100644 --- a/src/domain/result.rs +++ b/src/domain/result.rs @@ -1,3 +1,5 @@ +use super::SearchError; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct PageToken(pub String); @@ -10,6 +12,19 @@ pub struct SearchResponse { pub next_page: Option, } +#[derive(Debug)] +pub struct SearchBatchResponse { + pub query: String, + pub responses: Vec, + pub failures: Vec, +} + +#[derive(Debug)] +pub struct ProviderSearchFailure { + pub provider: String, + pub error: SearchError, +} + #[derive(Debug, Clone, PartialEq)] pub enum SearchResult { Web(WebResult), @@ -51,3 +66,36 @@ pub struct VideoResult { pub duration: Option, pub published_at: Option, } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn search_batch_response_can_hold_success_and_failure() { + let response = SearchResponse { + query: "rust".to_string(), + provider: "brave".to_string(), + results: vec![], + total_estimated: None, + next_page: None, + }; + let failure = ProviderSearchFailure { + provider: "exa".to_string(), + error: SearchError::InvalidQuery("unsupported".to_string()), + }; + let batch = SearchBatchResponse { + query: "rust".to_string(), + responses: vec![response], + failures: vec![failure], + }; + + assert_eq!(batch.query, "rust"); + assert_eq!(batch.responses[0].provider, "brave"); + assert_eq!(batch.failures[0].provider, "exa"); + assert_eq!( + batch.failures[0].error.to_string(), + "invalid query: unsupported" + ); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..f199ffc --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,6 @@ +pub mod app; +pub mod bootstrap; +pub mod cli; +pub mod domain; +mod providers; +mod transport; diff --git a/src/main.rs b/src/main.rs index 88f0577..b96dd0b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,89 +1,16 @@ -mod app; -mod cli; -mod domain; -mod providers; -mod transport; - -use app::search_service::SearchService; -use clap::Parser; -use cli::args::{CliArgs, CliProvider}; -use cli::output::render_text; -use domain::query::SearchQuery; -use providers::brave::client::BraveProvider; -use providers::brave::config::BraveConfig; -use providers::exa::client::ExaProvider; -use providers::exa::config::ExaConfig; -use transport::http::ReqwestHttpClient; +use tracing_subscriber::EnvFilter; #[tokio::main] async fn main() { - dotenvy::dotenv().ok(); - - let args = CliArgs::parse(); - - if args.about { - println!("sophon-cli — a provider-agnostic search CLI"); - println!(); - println!("Named after the Sophon from Cixin Liu's Three-Body Problem trilogy:"); - println!("a sentient proton supercomputer that performs near-infinite computation"); - println!("across vast distances. This tiny CLI delegates its heavy lifting to"); - println!("distant search APIs the same way."); - println!(); - println!("Currently supports Brave Search (web, news, images, video)."); - return; - } - - let query_text = match args.query { - Some(q) => q, - None => { - eprintln!("Error: missing query. Use --help for usage or --about for more info."); - std::process::exit(1); - } - }; + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .with_writer(std::io::stderr) + .init(); - let query = SearchQuery { - text: query_text, - search_type: args.search_type.into(), - limit: args.limit, - offset: args.offset, - safe_search: args.safe_search.map(|s| s.into()), - country: args.country, - language: args.language, - time_range: None, - }; - - let service = match args.provider { - CliProvider::Brave => { - let config = match BraveConfig::from_env() { - Ok(config) => config, - Err(error) => { - eprintln!("Failed to load Brave config: {}", error); - std::process::exit(1); - } - }; - let provider = BraveProvider::new(ReqwestHttpClient::new(), config); - SearchService::new(Box::new(provider)) - } - CliProvider::Exa => { - let config = match ExaConfig::from_env() { - Ok(config) => config, - Err(error) => { - eprintln!("Failed to load Exa config: {}", error); - std::process::exit(1); - } - }; - let provider = ExaProvider::new(ReqwestHttpClient::new(), config); - SearchService::new(Box::new(provider)) - } - }; + dotenvy::dotenv().ok(); - match service.search(query).await { - Ok(response) => { - println!("{}", render_text(&response)); - } - Err(e) => { - eprintln!("Search failed: {}", e); - std::process::exit(1); - } + let exit_code = sophon_cli::cli::runner::run_from_env().await; + if exit_code != 0 { + std::process::exit(exit_code); } } diff --git a/src/providers/brave/client.rs b/src/providers/brave/client.rs index f2c4604..b8e7109 100644 --- a/src/providers/brave/client.rs +++ b/src/providers/brave/client.rs @@ -1,14 +1,17 @@ -use crate::domain::error::SearchError; -use crate::domain::provider::{ProviderCapabilities, SearchProvider}; -use crate::domain::query::SearchQuery; -use crate::domain::result::SearchResponse; -use crate::domain::types::SearchType; -use crate::providers::brave::config::BraveConfig; -use crate::providers::brave::dto::*; -use crate::providers::brave::mapper::*; -use crate::transport::http::HttpClient; use async_trait::async_trait; +use crate::domain::{ + ProviderCapabilities, SafeSearch, SearchError, SearchProvider, SearchQuery, SearchResponse, + SearchType, TimeRange, +}; +use crate::transport::http::HttpClient; + +use super::config::BraveConfig; +use super::dto::{BraveImagesResponse, BraveNewsResponse, BraveVideosResponse, BraveWebResponse}; +use super::mapper::{ + map_images_response, map_news_response, map_videos_response, map_web_response, +}; + pub struct BraveProvider { client: C, config: BraveConfig, @@ -38,6 +41,7 @@ impl SearchProvider for BraveProvider { } } + #[tracing::instrument(skip(self), fields(query = %query.text, search_type = ?query.search_type))] async fn search(&self, query: &SearchQuery) -> Result { let endpoint = match query.search_type { SearchType::Web => "web/search", @@ -56,9 +60,9 @@ impl SearchProvider for BraveProvider { } if let Some(ss) = query.safe_search { let val = match ss { - crate::domain::types::SafeSearch::Off => "off", - crate::domain::types::SafeSearch::Moderate => "moderate", - crate::domain::types::SafeSearch::Strict => "strict", + SafeSearch::Off => "off", + SafeSearch::Moderate => "moderate", + SafeSearch::Strict => "strict", }; params.push(("safesearch".to_string(), val.to_string())); } @@ -70,10 +74,10 @@ impl SearchProvider for BraveProvider { } if let Some(ref tr) = query.time_range { let val = match tr { - crate::domain::types::TimeRange::Day => "day", - crate::domain::types::TimeRange::Week => "week", - crate::domain::types::TimeRange::Month => "month", - crate::domain::types::TimeRange::Year => "year", + TimeRange::Day => "day", + TimeRange::Week => "week", + TimeRange::Month => "month", + TimeRange::Year => "year", }; params.push(("freshness".to_string(), val.to_string())); } @@ -110,7 +114,7 @@ impl SearchProvider for BraveProvider { #[cfg(test)] mod tests { use super::*; - use crate::domain::types::{SafeSearch, SearchType}; + use crate::domain::SearchResult; use crate::transport::http::HttpClient; use async_trait::async_trait; use serde::Serialize; @@ -187,7 +191,7 @@ mod tests { assert_eq!(resp.total_estimated, Some(100)); assert_eq!(resp.results.len(), 1); match &resp.results[0] { - crate::domain::result::SearchResult::Web(r) => { + SearchResult::Web(r) => { assert_eq!(r.title, "Rust"); assert_eq!(r.url, "https://rust-lang.org"); } diff --git a/src/providers/brave/config.rs b/src/providers/brave/config.rs index da711dd..1cc5b1f 100644 --- a/src/providers/brave/config.rs +++ b/src/providers/brave/config.rs @@ -7,6 +7,10 @@ pub struct BraveConfig { impl BraveConfig { pub fn from_env() -> Result { let api_key = std::env::var("BRAVE_API_KEY")?; + if api_key.trim().is_empty() { + return Err(std::env::VarError::NotPresent); + } + Ok(Self { api_key, base_url: "https://api.search.brave.com/res/v1".to_string(), diff --git a/src/providers/brave/mapper.rs b/src/providers/brave/mapper.rs index 3cbc5da..f1b9d64 100644 --- a/src/providers/brave/mapper.rs +++ b/src/providers/brave/mapper.rs @@ -1,5 +1,8 @@ -use crate::domain::result::*; -use crate::providers::brave::dto::*; +use crate::domain::{ + ImageResult, NewsResult, SearchResponse, SearchResult, VideoResult, WebResult, +}; + +use super::dto::{BraveImagesResponse, BraveNewsResponse, BraveVideosResponse, BraveWebResponse}; pub fn map_web_response(dto: BraveWebResponse) -> SearchResponse { let query_text = dto.query.and_then(|q| q.original).unwrap_or_default(); @@ -94,6 +97,11 @@ pub fn map_videos_response(dto: BraveVideosResponse) -> SearchResponse { #[cfg(test)] mod tests { + use super::super::dto::{ + BraveImageResult, BraveImagesResponse, BraveNewsResponse, BraveNewsResult, + BraveNewsResults, BraveQuery, BraveThumbnail, BraveVideoResult, BraveVideosResponse, + BraveVideosResults, BraveWebResponse, BraveWebResult, BraveWebResults, + }; use super::*; #[test] diff --git a/src/providers/exa/client.rs b/src/providers/exa/client.rs index eea9ebc..66ce041 100644 --- a/src/providers/exa/client.rs +++ b/src/providers/exa/client.rs @@ -1,15 +1,22 @@ -use crate::domain::error::SearchError; -use crate::domain::provider::{ProviderCapabilities, SearchProvider}; -use crate::domain::query::SearchQuery; -use crate::domain::result::SearchResponse; -use crate::domain::types::{SafeSearch, SearchType, TimeRange}; -use crate::providers::exa::config::ExaConfig; -use crate::providers::exa::dto::{ExaContentsRequest, ExaSearchRequest, ExaSearchResponse}; -use crate::providers::exa::mapper::{map_news_response, map_web_response}; -use crate::transport::http::HttpClient; use async_trait::async_trait; use chrono::{Duration, Utc}; +use crate::domain::{ + ProviderCapabilities, SafeSearch, SearchError, SearchProvider, SearchQuery, SearchResponse, + SearchType, TimeRange, +}; +use crate::transport::http::HttpClient; + +use super::config::ExaConfig; +use super::dto::{ + ExaContentsRequest, ExaHighlightsRequest, ExaSearchRequest, ExaSearchResponse, + ExaSummaryRequest, +}; +use super::mapper::{map_news_response, map_web_response}; + +/// Budget for Exa per-result highlights (API); mapper applies a shorter CLI-facing cap. +const EXA_HIGHLIGHTS_MAX_CHARACTERS: u32 = 1200; + pub struct ExaProvider { client: C, config: ExaConfig, @@ -67,7 +74,16 @@ impl ExaProvider { Some(SafeSearch::Moderate | SafeSearch::Strict) => Some(true), None => None, }, - contents: ExaContentsRequest { text: true }, + contents: ExaContentsRequest { + text: None, + highlights: Some(ExaHighlightsRequest { + max_characters: EXA_HIGHLIGHTS_MAX_CHARACTERS, + query: Some(query.text.clone()), + }), + summary: Some(ExaSummaryRequest { + query: query.text.clone(), + }), + }, }) } } @@ -90,6 +106,7 @@ impl SearchProvider for ExaProvider { } } + #[tracing::instrument(skip(self), fields(query = %query.text, search_type = ?query.search_type))] async fn search(&self, query: &SearchQuery) -> Result { let request = self.build_request(query)?; let response: ExaSearchResponse = self @@ -127,13 +144,11 @@ fn published_date_window(time_range: Option<&TimeRange>) -> (Option, Opt #[cfg(test)] mod tests { + use super::super::config::ExaConfig; use super::ExaProvider; - use crate::domain::error::SearchError; - use crate::domain::provider::SearchProvider; - use crate::domain::query::SearchQuery; - use crate::domain::result::SearchResult; - use crate::domain::types::{SafeSearch, SearchType, TimeRange}; - use crate::providers::exa::config::ExaConfig; + use crate::domain::{ + SafeSearch, SearchError, SearchProvider, SearchQuery, SearchResult, SearchType, TimeRange, + }; use crate::transport::http::HttpClient; use async_trait::async_trait; use serde::Serialize; @@ -181,10 +196,28 @@ mod tests { assert_eq!(body.get("numResults"), Some(&Value::from(3))); assert_eq!(body.get("moderation"), Some(&Value::Bool(true))); assert_eq!(body.get("type"), Some(&Value::String("auto".to_string()))); + let contents = body.get("contents").unwrap(); + assert!(contents.get("text").is_none()); + assert_eq!( + contents + .get("highlights") + .and_then(|h| h.get("maxCharacters")) + .and_then(Value::as_u64), + Some(u64::from(super::EXA_HIGHLIGHTS_MAX_CHARACTERS)) + ); + assert_eq!( + contents + .get("highlights") + .and_then(|h| h.get("query")) + .and_then(Value::as_str), + Some("ai news") + ); assert_eq!( - body.get("contents") - .and_then(|contents| contents.get("text")), - Some(&Value::Bool(true)) + contents + .get("summary") + .and_then(|s| s.get("query")) + .and_then(Value::as_str), + Some("ai news") ); let start = body diff --git a/src/providers/exa/config.rs b/src/providers/exa/config.rs index 8a7bfc6..a2319b0 100644 --- a/src/providers/exa/config.rs +++ b/src/providers/exa/config.rs @@ -7,6 +7,10 @@ pub struct ExaConfig { impl ExaConfig { pub fn from_env() -> Result { let api_key = std::env::var("EXA_API_KEY")?; + if api_key.trim().is_empty() { + return Err(std::env::VarError::NotPresent); + } + Ok(Self { api_key, base_url: "https://api.exa.ai".to_string(), diff --git a/src/providers/exa/dto.rs b/src/providers/exa/dto.rs index 1ae6818..355ce96 100644 --- a/src/providers/exa/dto.rs +++ b/src/providers/exa/dto.rs @@ -24,7 +24,27 @@ pub struct ExaSearchRequest { #[derive(Debug, Clone, Serialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ExaContentsRequest { - pub text: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub highlights: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ExaHighlightsRequest { + pub max_characters: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub query: Option, +} + +/// Request-time `contents.summary` object with query (see Exa search API). +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ExaSummaryRequest { + pub query: String, } #[derive(Debug, Clone, Deserialize, PartialEq, Eq)] @@ -43,6 +63,8 @@ pub struct ExaResult { pub url: Option, pub published_date: Option, pub author: Option, + #[serde(default)] + pub highlights: Vec, pub text: Option, pub summary: Option, } @@ -77,9 +99,38 @@ mod tests { url: Some("https://example.com".to_string()), published_date: Some("2026-04-15T00:00:00.000Z".to_string()), author: None, + highlights: vec![], text: None, summary: Some("Example summary".to_string()), }] ); } + + #[test] + fn test_exa_search_response_deserializes_highlights_and_summary() { + let json = r#"{ + "requestId": "req_456", + "searchType": "auto", + "results": [ + { + "title": "Article", + "url": "https://example.com/a", + "highlights": ["First excerpt.", "Second excerpt."], + "summary": "One-line overview", + "text": "FULL PAGE MARKDOWN WOULD BE HERE" + } + ] + }"#; + + let response: ExaSearchResponse = serde_json::from_str(json).unwrap(); + + assert_eq!(response.results.len(), 1); + let r = &response.results[0]; + assert_eq!( + r.highlights, + vec!["First excerpt.".to_string(), "Second excerpt.".to_string()] + ); + assert_eq!(r.summary.as_deref(), Some("One-line overview")); + assert_eq!(r.text.as_deref(), Some("FULL PAGE MARKDOWN WOULD BE HERE")); + } } diff --git a/src/providers/exa/mapper.rs b/src/providers/exa/mapper.rs index f31c9fc..8bdc4b7 100644 --- a/src/providers/exa/mapper.rs +++ b/src/providers/exa/mapper.rs @@ -1,29 +1,39 @@ -use crate::domain::result::{NewsResult, SearchResponse, SearchResult, WebResult}; -use crate::providers::exa::dto::{ExaResult, ExaSearchResponse}; +use crate::domain::{NewsResult, SearchResponse, SearchResult, WebResult}; + +use super::dto::{ExaResult, ExaSearchResponse}; + +/// Maximum characters for `snippet` shown in the CLI (after join / trim). +const SNIPPET_DISPLAY_MAX_CHARS: usize = 500; + +const HIGHLIGHT_JOIN: &str = " … "; pub fn map_web_response(query: &str, dto: ExaSearchResponse) -> SearchResponse { - SearchResponse { - query: query.to_string(), - provider: "exa".to_string(), - total_estimated: None, - next_page: None, - results: dto - .results - .into_iter() - .map(|result| { - let snippet = preferred_snippet(&result); - SearchResult::Web(WebResult { - title: result.title.unwrap_or_default(), - url: result.url.unwrap_or_default(), - snippet, - display_url: None, - }) - }) - .collect(), - } + map_response(query, dto, |result, snippet| { + SearchResult::Web(WebResult { + title: result.title.unwrap_or_default(), + url: result.url.unwrap_or_default(), + snippet, + display_url: None, + }) + }) } pub fn map_news_response(query: &str, dto: ExaSearchResponse) -> SearchResponse { + map_response(query, dto, |result, snippet| { + SearchResult::News(NewsResult { + title: result.title.unwrap_or_default(), + url: result.url.unwrap_or_default(), + snippet, + source: result.author, + published_at: result.published_date, + }) + }) +} + +fn map_response(query: &str, dto: ExaSearchResponse, map_result: F) -> SearchResponse +where + F: Fn(ExaResult, Option) -> SearchResult, +{ SearchResponse { query: query.to_string(), provider: "exa".to_string(), @@ -34,27 +44,67 @@ pub fn map_news_response(query: &str, dto: ExaSearchResponse) -> SearchResponse .into_iter() .map(|result| { let snippet = preferred_snippet(&result); - SearchResult::News(NewsResult { - title: result.title.unwrap_or_default(), - url: result.url.unwrap_or_default(), - snippet, - source: result.author, - published_at: result.published_date, - }) + map_result(result, snippet) }) .collect(), } } fn preferred_snippet(result: &ExaResult) -> Option { - result.summary.clone().or_else(|| result.text.clone()) + if let Some(s) = result + .summary + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + { + return Some(cap_snippet_chars(s, SNIPPET_DISPLAY_MAX_CHARS)); + } + + let joined = join_highlights(&result.highlights); + if joined.is_empty() { + return None; + } + + Some(cap_snippet_chars(&joined, SNIPPET_DISPLAY_MAX_CHARS)) +} + +fn join_highlights(highlights: &[String]) -> String { + highlights + .iter() + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .collect::>() + .join(HIGHLIGHT_JOIN) +} + +fn cap_snippet_chars(s: &str, max_chars: usize) -> String { + let count = s.chars().count(); + if count <= max_chars { + return s.to_string(); + } + let mut out: String = s.chars().take(max_chars.saturating_sub(1)).collect(); + out.push('…'); + out } #[cfg(test)] mod tests { use super::map_news_response; - use crate::domain::result::SearchResult; - use crate::providers::exa::dto::{ExaResult, ExaSearchResponse}; + use crate::domain::SearchResult; + + use super::super::dto::{ExaResult, ExaSearchResponse}; + + fn sample_result() -> ExaResult { + ExaResult { + title: Some("Example headline".to_string()), + url: Some("https://example.com/news".to_string()), + published_date: Some("2026-04-15T00:00:00.000Z".to_string()), + author: Some("Example Reporter".to_string()), + highlights: vec![], + text: None, + summary: Some("Short summary".to_string()), + } + } #[test] fn test_map_news_response_prefers_summary_and_preserves_author() { @@ -62,12 +112,9 @@ mod tests { request_id: Some("req_123".to_string()), search_type: Some("auto".to_string()), results: vec![ExaResult { - title: Some("Example headline".to_string()), - url: Some("https://example.com/news".to_string()), - published_date: Some("2026-04-15T00:00:00.000Z".to_string()), - author: Some("Example Reporter".to_string()), - text: Some("Longer body text".to_string()), - summary: Some("Short summary".to_string()), + highlights: vec!["Longer body text".to_string()], + text: Some("IGNORED FULL TEXT".to_string()), + ..sample_result() }], }; @@ -92,4 +139,87 @@ mod tests { other => panic!("expected news result, got {other:?}"), } } + + #[test] + fn test_map_prefers_summary_over_highlights() { + let dto = ExaSearchResponse { + request_id: None, + search_type: None, + results: vec![ExaResult { + summary: Some("Summary wins".to_string()), + highlights: vec!["Highlight A".to_string(), "Highlight B".to_string()], + text: Some("x".repeat(50_000)), + ..sample_result() + }], + }; + let response = map_news_response("q", dto); + match &response.results[0] { + SearchResult::News(r) => assert_eq!(r.snippet.as_deref(), Some("Summary wins")), + other => panic!("{other:?}"), + } + } + + #[test] + fn test_map_whitespace_summary_falls_through_to_highlights() { + let dto = ExaSearchResponse { + request_id: None, + search_type: None, + results: vec![ExaResult { + summary: Some(" \t ".to_string()), + highlights: vec!["Only this".to_string()], + text: Some("x".repeat(10_000)), + ..sample_result() + }], + }; + let response = map_news_response("q", dto); + match &response.results[0] { + SearchResult::News(r) => assert_eq!(r.snippet.as_deref(), Some("Only this")), + other => panic!("{other:?}"), + } + } + + #[test] + fn test_map_highlights_joined_and_capped() { + let a = "a".repeat(350); + let b = "b".repeat(350); + let dto = ExaSearchResponse { + request_id: None, + search_type: None, + results: vec![ExaResult { + summary: None, + highlights: vec![a, b], + text: Some("SHOULD NOT APPEAR".to_string()), + ..sample_result() + }], + }; + let response = map_news_response("q", dto); + match &response.results[0] { + SearchResult::News(r) => { + let s = r.snippet.as_ref().unwrap(); + assert!(!s.contains("SHOULD NOT APPEAR")); + assert!(s.chars().count() <= 500); + assert!(s.ends_with('…')); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn test_map_no_concise_fields_returns_none_despite_huge_text() { + let dto = ExaSearchResponse { + request_id: None, + search_type: None, + results: vec![ExaResult { + summary: None, + highlights: vec![], + text: Some("x".repeat(100_000)), + ..sample_result() + }], + }; + let response = map_news_response("q", dto); + match &response.results[0] { + SearchResult::News(r) => assert!(r.snippet.is_none()), + other => panic!("{other:?}"), + } + } } diff --git a/src/transport/http.rs b/src/transport/http.rs index 64ac382..43b3326 100644 --- a/src/transport/http.rs +++ b/src/transport/http.rs @@ -1,8 +1,9 @@ -use crate::domain::error::SearchError; use async_trait::async_trait; use reqwest::{Client, Response}; use serde::Serialize; +use crate::domain::SearchError; + #[async_trait] pub trait HttpClient: Send + Sync { async fn get_json( @@ -34,12 +35,18 @@ pub struct ReqwestHttpClient { client: Client, } -impl ReqwestHttpClient { - pub fn new() -> Self { +impl Default for ReqwestHttpClient { + fn default() -> Self { Self { client: Client::new(), } } +} + +impl ReqwestHttpClient { + pub fn new() -> Self { + Self::default() + } async fn decode_response(response: Response) -> Result where @@ -66,6 +73,7 @@ impl ReqwestHttpClient { #[async_trait] impl HttpClient for ReqwestHttpClient { + #[tracing::instrument(skip(self, headers), fields(url = %url))] async fn get_json( &self, url: &str, @@ -85,9 +93,11 @@ impl HttpClient for ReqwestHttpClient { .await .map_err(|e| SearchError::Transport(e.to_string()))?; + tracing::debug!(status = %resp.status(), "received HTTP response"); Self::decode_response(resp).await } + #[tracing::instrument(skip(self, headers, body), fields(url = %url))] async fn post_json( &self, url: &str, @@ -108,6 +118,7 @@ impl HttpClient for ReqwestHttpClient { .await .map_err(|e| SearchError::Transport(e.to_string()))?; + tracing::debug!(status = %resp.status(), "received HTTP response"); Self::decode_response(resp).await } } diff --git a/tests/architecture_test.rs b/tests/architecture_test.rs index a5e566d..1b2b4ff 100644 --- a/tests/architecture_test.rs +++ b/tests/architecture_test.rs @@ -29,11 +29,22 @@ fn test_providers_do_not_import_cli_or_app() { } #[test] -fn test_app_does_not_import_cli() { - let forbidden = ["use crate::cli::"]; +fn test_app_depends_only_on_domain_contracts() { + let forbidden = [ + "use crate::cli::", + "use crate::bootstrap::", + "use crate::providers::", + "use crate::transport::", + ]; check_dir_for_forbidden_patterns("src/app", &forbidden); } +#[test] +fn test_bootstrap_does_not_import_cli() { + let forbidden = ["use crate::cli::"]; + check_dir_for_forbidden_patterns("src/bootstrap", &forbidden); +} + #[test] fn test_render_text_only_called_from_cli() { let forbidden_dirs = ["src/domain", "src/transport", "src/providers", "src/app"]; @@ -48,6 +59,160 @@ fn test_render_text_only_called_from_cli() { } } +#[test] +fn test_t001_cli_request_module_import_contract() { + let cli_mod = read_repo_file("src/cli/mod.rs"); + assert!( + cli_mod.contains("pub mod request;"), + "T001 requires src/cli/mod.rs to expose the request module" + ); + + let request = read_repo_file("src/cli/request.rs"); + assert!( + request.contains("crate::cli::args") && request.contains("CliArgs"), + "T001 request module must import CliArgs from the CLI args boundary" + ); + assert!( + request.contains("crate::domain") && request.contains("SearchQuery"), + "T001 request module must import SearchQuery from the domain boundary" + ); + assert!( + request.contains("pub fn build_search_query"), + "T001 request module must expose build_search_query" + ); +} + +#[test] +fn test_t002_cli_runner_module_import_contract() { + let cli_mod = read_repo_file("src/cli/mod.rs"); + assert!( + cli_mod.contains("pub mod runner;"), + "T002 requires src/cli/mod.rs to expose the runner module" + ); + + let runner = read_repo_file("src/cli/runner.rs"); + let required_patterns = [ + ("use clap::Parser", "parse CLI args in run_from_env"), + ("CliArgs", "accept parsed CLI args"), + ("CliProvider", "branch on provider selection"), + ( + "build_search_query", + "delegate query normalization to cli::request", + ), + ( + "ProviderRegistry", + "request provider services from bootstrap", + ), + ( + "ProviderId", + "identify single-provider runs through bootstrap IDs", + ), + ( + "render_fanout_text", + "render fan-out output through the CLI output boundary", + ), + ( + "pub async fn run_from_env() -> i32", + "expose env-backed runner entrypoint", + ), + ( + "pub async fn run(args: CliArgs) -> i32", + "expose parsed-args runner entrypoint", + ), + ]; + + for (pattern, reason) in required_patterns { + assert!( + runner.contains(pattern), + "T002 runner import contract missing {pattern:?} to {reason}" + ); + } +} + +#[test] +fn test_t003_entrypoint_import_contract() { + let main = read_repo_file("src/main.rs"); + let forbidden_patterns = [ + "use clap::Parser", + "sophon_cli::bootstrap::", + "sophon_cli::domain::", + "sophon_cli::cli::args", + "sophon_cli::cli::output", + "mod single_provider_search", + "single_provider_search::", + ]; + + for pattern in forbidden_patterns { + assert!( + !main.contains(pattern), + "T003 requires src/main.rs to drop direct import/declaration {pattern:?}" + ); + } + + assert!( + main.contains("sophon_cli::cli::runner::run_from_env().await"), + "T003 requires src/main.rs to delegate runtime execution to cli::runner::run_from_env" + ); + assert!( + !Path::new("src/single_provider_search.rs").exists(), + "T003 requires src/single_provider_search.rs to be removed after its logic moves to cli::runner" + ); +} + +#[test] +fn test_entrypoint_delegates_only_to_cli_surface() { + assert_entrypoint_delegates_only_to_cli_surface(); +} + +#[test] +fn test_t004_ideal_dependency_direction_import_contract() { + assert_entrypoint_delegates_only_to_cli_surface(); + + let forbidden_app_patterns = [ + "use crate::cli::", + "use crate::bootstrap::", + "use crate::providers::", + "use crate::transport::", + ]; + check_dir_for_forbidden_patterns("src/app", &forbidden_app_patterns); +} + +#[test] +fn test_t005_current_dependency_map_reflects_refactor() { + let current_map = read_repo_file("docs/dependency-architecture-map.html"); + assert!( + current_map.contains("cli::runner"), + "T005 current dependency map should show cli::runner after the refactor" + ); + assert!( + !current_map.contains("single_provider_search"), + "T005 current dependency map should no longer mention single_provider_search" + ); +} + +fn assert_entrypoint_delegates_only_to_cli_surface() { + let main = read_repo_file("src/main.rs"); + let forbidden_entrypoint_patterns = [ + "sophon_cli::bootstrap::", + "sophon_cli::domain::", + "sophon_cli::cli::output", + "mod single_provider_search", + "single_provider_search::", + ]; + + for pattern in forbidden_entrypoint_patterns { + assert!( + !main.contains(pattern), + "T004 requires the entrypoint to delegate only to the CLI surface; found {pattern:?}" + ); + } + + assert!( + !Path::new("src/single_provider_search.rs").exists(), + "T004 requires the binary-private single_provider_search helper to be absent" + ); +} + fn check_dir_for_forbidden_patterns(dir: &str, forbidden: &[&str]) { visit_rust_files(dir, &|path, content| { for pat in forbidden { @@ -61,6 +226,12 @@ fn check_dir_for_forbidden_patterns(dir: &str, forbidden: &[&str]) { }); } +fn read_repo_file(path: &str) -> String { + fs::read_to_string(path).unwrap_or_else(|error| { + panic!("Expected {path} to exist for import-organization contract test: {error}") + }) +} + fn visit_rust_files(dir: &str, callback: &dyn Fn(&Path, &str)) { let path = Path::new(dir); if !path.exists() { diff --git a/tests/common/cli.rs b/tests/common/cli.rs new file mode 100644 index 0000000..18b6106 --- /dev/null +++ b/tests/common/cli.rs @@ -0,0 +1,44 @@ +#![allow(dead_code)] + +use std::process::{Command, Output}; + +pub fn run_cli(args: &[&str]) -> Output { + cli_command(args) + .env_remove("RUST_LOG") + .output() + .expect("sophon-cli runs") +} + +pub fn run_cli_without_keys(args: &[&str]) -> Output { + cli_command(args) + .env("BRAVE_API_KEY", "") + .env("EXA_API_KEY", "") + .env_remove("RUST_LOG") + .current_dir(std::env::temp_dir()) + .output() + .expect("sophon-cli runs") +} + +fn cli_command(args: &[&str]) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_sophon-cli")); + command.args(args); + command +} + +pub fn stdout_text(output: &Output) -> String { + String::from_utf8_lossy(&output.stdout).into_owned() +} + +pub fn stderr_text(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).into_owned() +} + +pub fn assert_stdout_empty(output: &Output) { + let stdout = stdout_text(output); + assert!(stdout.is_empty(), "unexpected stdout: {stdout}"); +} + +pub fn assert_stderr_empty(output: &Output) { + let stderr = stderr_text(output); + assert!(stderr.is_empty(), "unexpected stderr: {stderr}"); +} diff --git a/tests/fanout_cli_test.rs b/tests/fanout_cli_test.rs new file mode 100644 index 0000000..5fe505a --- /dev/null +++ b/tests/fanout_cli_test.rs @@ -0,0 +1,16 @@ +#[path = "common/cli.rs"] +mod cli; + +#[test] +fn provider_all_without_config_exits_nonzero_with_no_provider_error() { + let output = cli::run_cli_without_keys(&["rust", "--provider", "all"]); + + assert!(!output.status.success()); + cli::assert_stdout_empty(&output); + + let stderr = cli::stderr_text(&output); + assert!( + stderr.contains("no configured providers"), + "stderr did not contain no configured providers: {stderr}" + ); +} diff --git a/tests/integration/cli_test.rs b/tests/integration/cli_test.rs new file mode 100644 index 0000000..ecd3051 --- /dev/null +++ b/tests/integration/cli_test.rs @@ -0,0 +1,111 @@ +#[path = "../common/cli.rs"] +mod cli; + +use clap::Parser; +use sophon_cli::cli::args::{CliArgs, CliProvider, CliSafeSearch, CliSearchType}; + +#[test] +fn cli_about_flag_prints_description() { + let output = cli::run_cli(&["--about"]); + + assert!(output.status.success()); + cli::assert_stderr_empty(&output); + + let stdout = cli::stdout_text(&output); + assert!(stdout.contains("sophon-cli")); + assert!(stdout.contains("Three-Body Problem")); + assert!(stdout.contains("Brave Search")); + assert!(stdout.contains("Exa")); +} + +#[test] +fn cli_help_flag_prints_usage() { + let output = cli::run_cli(&["--help"]); + + assert!(output.status.success()); + cli::assert_stderr_empty(&output); + + let stdout = cli::stdout_text(&output); + assert!(stdout.contains("--provider")); + assert!(stdout.contains("--search-type")); + assert!(stdout.contains("--limit")); + assert!(stdout.contains("--about")); + assert!(stdout.contains("--scrape")); + assert!(stdout.contains("--safe-search")); +} + +#[test] +fn cli_missing_query_exits_with_error() { + let output = cli::run_cli(&[]); + + assert!(!output.status.success()); + cli::assert_stdout_empty(&output); + + let stderr = cli::stderr_text(&output); + assert!( + stderr.contains("missing query"), + "stderr did not contain missing query: {stderr}" + ); +} + +#[test] +fn cli_brave_provider_without_key_exits_with_provider_unavailable_error() { + let output = cli::run_cli_without_keys(&["rust", "--provider", "brave"]); + + assert_explicit_provider_unavailable(output, "brave"); +} + +#[test] +fn cli_exa_provider_without_key_exits_with_provider_unavailable_error() { + let output = cli::run_cli_without_keys(&["rust", "--provider", "exa"]); + + assert_explicit_provider_unavailable(output, "exa"); +} + +#[test] +fn cli_with_explicit_arguments_parses_correctly() { + let args = CliArgs::try_parse_from([ + "sophon-cli", + "rust search", + "--provider", + "all", + "--search-type", + "news", + "--limit", + "3", + "--safe-search", + "strict", + "--country", + "US", + "--language", + "en", + ]) + .expect("explicit args parse"); + + assert_eq!(args.query.as_deref(), Some("rust search")); + assert_eq!(args.provider, CliProvider::All); + assert_eq!(args.search_type, CliSearchType::News); + assert_eq!(args.limit, Some(3)); + assert_eq!(args.safe_search, Some(CliSafeSearch::Strict)); + assert_eq!(args.country.as_deref(), Some("US")); + assert_eq!(args.language.as_deref(), Some("en")); +} + +fn assert_explicit_provider_unavailable(output: std::process::Output, provider: &str) { + assert!(!output.status.success()); + cli::assert_stdout_empty(&output); + + let stderr = cli::stderr_text(&output); + assert!( + stderr.contains(&format!("provider `{provider}` is unavailable")), + "stderr did not contain {provider} unavailable error: {stderr}" + ); + assert!( + stderr.contains("configured providers: []"), + "stderr did not include configured provider list: {stderr}" + ); + assert!( + !stderr.contains("no configured providers") && !stderr.contains("NoProvidersAvailable"), + "explicit provider should not use fan-out no-provider error: {stderr}" + ); +} diff --git a/tests/integration/provider_registry_test.rs b/tests/integration/provider_registry_test.rs new file mode 100644 index 0000000..86a34f9 --- /dev/null +++ b/tests/integration/provider_registry_test.rs @@ -0,0 +1,257 @@ +use async_trait::async_trait; +use sophon_cli::bootstrap::provider_registry::{ + BuildSearchServiceError, ProviderBuilder, ProviderId, ProviderRegistry, +}; +use sophon_cli::domain::{ + ProviderCapabilities, SearchError, SearchProvider, SearchQuery, SearchResponse, SearchType, +}; +use std::ffi::OsString; +use std::sync::{Mutex, MutexGuard, OnceLock}; + +struct NamedProvider { + name: &'static str, +} + +#[async_trait] +impl SearchProvider for NamedProvider { + fn id(&self) -> String { + self.name.to_string() + } + + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities::default() + } + + async fn search(&self, query: &SearchQuery) -> Result { + Ok(SearchResponse { + query: query.text.clone(), + provider: self.name.to_string(), + results: vec![], + total_estimated: None, + next_page: None, + }) + } +} + +struct EnvGuard { + _lock: MutexGuard<'static, ()>, + brave: Option, + exa: Option, +} + +impl EnvGuard { + fn set(brave: Option<&str>, exa: Option<&str>) -> Self { + let lock = env_lock().lock().unwrap(); + let guard = Self { + _lock: lock, + brave: std::env::var_os("BRAVE_API_KEY"), + exa: std::env::var_os("EXA_API_KEY"), + }; + + set_env_var("BRAVE_API_KEY", brave); + set_env_var("EXA_API_KEY", exa); + guard + } +} + +impl Drop for EnvGuard { + fn drop(&mut self) { + restore_saved_env_var("BRAVE_API_KEY", self.brave.take()); + restore_saved_env_var("EXA_API_KEY", self.exa.take()); + } +} + +fn env_lock() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) +} + +fn set_env_var(name: &str, value: Option<&str>) { + match value { + Some(value) => unsafe { + std::env::set_var(name, value); + }, + None => unsafe { + std::env::remove_var(name); + }, + } +} + +fn restore_saved_env_var(name: &str, saved_value: Option) { + if let Some(saved_value) = saved_value { + unsafe { std::env::set_var(name, saved_value) }; + } else { + unsafe { std::env::remove_var(name) }; + } +} + +fn provider_builder(name: &'static str) -> ProviderBuilder { + Box::new(move || Box::new(NamedProvider { name })) +} + +fn search_query(text: &str) -> SearchQuery { + SearchQuery { + text: text.to_string(), + search_type: SearchType::Web, + limit: None, + offset: None, + safe_search: None, + country: None, + language: None, + time_range: None, + } +} + +#[test] +fn empty_registry_build_fails_with_provider_unavailable() { + let registry = ProviderRegistry::empty(); + let result = registry.build(ProviderId::Brave); + + match result { + Err(BuildSearchServiceError::ProviderUnavailable { + provider, + available, + }) => { + assert_eq!(provider, ProviderId::Brave); + assert!(available.is_empty()); + } + other => panic!( + "expected ProviderUnavailable error, got {:?}", + other.map(|_| ()) + ), + } +} + +#[test] +fn empty_registry_build_all_enabled_fails_with_no_providers() { + let registry = ProviderRegistry::empty(); + let result = registry.build_all_enabled(); + + match result { + Err(BuildSearchServiceError::NoProvidersAvailable) => {} + other => panic!( + "expected NoProvidersAvailable error, got {:?}", + other.map(|_| ()) + ), + } +} + +#[test] +fn registered_provider_is_available() { + let mut registry = ProviderRegistry::empty(); + registry.register(ProviderId::Brave, provider_builder("stub")); + + assert_eq!(registry.available_providers(), vec![ProviderId::Brave]); +} + +#[test] +fn available_providers_returned_in_stable_order() { + let mut registry = ProviderRegistry::empty(); + registry.register(ProviderId::Exa, provider_builder("exa")); + registry.register(ProviderId::Brave, provider_builder("brave")); + + assert_eq!( + registry.available_providers(), + vec![ProviderId::Brave, ProviderId::Exa] + ); +} + +#[test] +fn production_registry_without_keys_has_no_providers() { + let _env = EnvGuard::set(None, None); + + let registry = ProviderRegistry::production_from_env(); + + assert!(registry.available_providers().is_empty()); +} + +#[test] +fn production_registry_treats_empty_keys_as_unconfigured() { + let _env = EnvGuard::set(Some(" "), Some("")); + + let registry = ProviderRegistry::production_from_env(); + + assert!(registry.available_providers().is_empty()); +} + +#[test] +fn production_registry_includes_brave_only_when_only_brave_key_is_set() { + let _env = EnvGuard::set(Some("test-brave-key"), None); + + let registry = ProviderRegistry::production_from_env(); + + assert_eq!(registry.available_providers(), vec![ProviderId::Brave]); +} + +#[test] +fn production_registry_includes_exa_only_when_only_exa_key_is_set() { + let _env = EnvGuard::set(None, Some("test-exa-key")); + + let registry = ProviderRegistry::production_from_env(); + + assert_eq!(registry.available_providers(), vec![ProviderId::Exa]); +} + +#[test] +fn production_registry_includes_both_env_configured_providers_in_stable_order() { + let _env = EnvGuard::set(Some("test-brave-key"), Some("test-exa-key")); + + let registry = ProviderRegistry::production_from_env(); + + assert_eq!( + registry.available_providers(), + vec![ProviderId::Brave, ProviderId::Exa] + ); +} + +#[test] +fn production_registry_reports_explicit_provider_unavailable_when_only_other_provider_exists() { + let _env = EnvGuard::set(None, Some("test-exa-key")); + + let registry = ProviderRegistry::production_from_env(); + let result = registry.build(ProviderId::Brave); + + match result { + Err(BuildSearchServiceError::ProviderUnavailable { + provider, + available, + }) => { + assert_eq!(provider, ProviderId::Brave); + assert_eq!(available, vec![ProviderId::Exa]); + } + other => panic!( + "expected ProviderUnavailable error, got {:?}", + other.map(|_| ()) + ), + } +} + +#[tokio::test] +async fn registered_provider_builds_service_that_searches() { + let mut registry = ProviderRegistry::empty(); + registry.register(ProviderId::Brave, provider_builder("stub")); + + let service = registry.build(ProviderId::Brave).expect("build succeeds"); + let result = service + .search(search_query("integration test")) + .await + .unwrap(); + + assert_eq!(result.query, "integration test"); + assert_eq!(result.provider, "stub"); +} + +#[tokio::test] +async fn build_all_enabled_uses_stable_order() { + let mut registry = ProviderRegistry::empty(); + registry.register(ProviderId::Exa, provider_builder("exa")); + registry.register(ProviderId::Brave, provider_builder("brave")); + + let service = registry.build_all_enabled().expect("build succeeds"); + let batch = service.search_all(search_query("rust")).await; + + assert!(batch.failures.is_empty()); + assert_eq!(batch.responses.len(), 2); + assert_eq!(batch.responses[0].provider, "brave"); + assert_eq!(batch.responses[1].provider, "exa"); +} diff --git a/tests/integration/search_service_test.rs b/tests/integration/search_service_test.rs new file mode 100644 index 0000000..c4d9a04 --- /dev/null +++ b/tests/integration/search_service_test.rs @@ -0,0 +1,207 @@ +use async_trait::async_trait; +use sophon_cli::app::fanout_search_service::FanoutSearchService; +use sophon_cli::app::search_service::SearchService; +use sophon_cli::domain::{ + ProviderCapabilities, SearchError, SearchProvider, SearchQuery, SearchResponse, SearchResult, + SearchType, WebResult, +}; +use std::sync::{Arc, Mutex}; + +#[derive(Clone)] +enum ProviderOutcome { + Success(SearchResponse), + Failure(&'static str), +} + +struct SpyProvider { + id: &'static str, + seen_queries: Arc>>, + outcome: ProviderOutcome, +} + +#[async_trait] +impl SearchProvider for SpyProvider { + fn id(&self) -> String { + self.id.to_string() + } + + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + web: true, + ..ProviderCapabilities::default() + } + } + + async fn search(&self, query: &SearchQuery) -> Result { + self.seen_queries.lock().unwrap().push(query.clone()); + + match &self.outcome { + ProviderOutcome::Success(response) => Ok(response.clone()), + ProviderOutcome::Failure(message) => Err(SearchError::Provider((*message).to_string())), + } + } +} + +fn query() -> SearchQuery { + SearchQuery { + text: "distributed systems".to_string(), + search_type: SearchType::Web, + limit: Some(10), + offset: None, + safe_search: None, + country: None, + language: None, + time_range: None, + } +} + +fn response(provider: &str, total_estimated: Option) -> SearchResponse { + SearchResponse { + query: "provider supplied query".to_string(), + provider: provider.to_string(), + results: vec![], + total_estimated, + next_page: None, + } +} + +fn provider( + id: &'static str, + outcome: ProviderOutcome, +) -> (SpyProvider, Arc>>) { + let seen_queries = Arc::new(Mutex::new(Vec::new())); + ( + SpyProvider { + id, + seen_queries: Arc::clone(&seen_queries), + outcome, + }, + seen_queries, + ) +} + +fn web_result(title: &str, url: &str, snippet: Option<&str>) -> SearchResult { + SearchResult::Web(WebResult { + title: title.to_string(), + url: url.to_string(), + snippet: snippet.map(str::to_string), + display_url: None, + }) +} + +#[tokio::test] +async fn search_service_passes_full_query_to_provider_and_returns_exact_response() { + let expected_query = query(); + let expected_response = SearchResponse { + query: "canonical provider query".to_string(), + provider: "mock".to_string(), + results: vec![], + total_estimated: None, + next_page: None, + }; + let (provider, seen_queries) = + provider("mock", ProviderOutcome::Success(expected_response.clone())); + let service = SearchService::new(Box::new(provider)); + + let result = service.search(expected_query.clone()).await.unwrap(); + + assert_eq!(result, expected_response); + assert_eq!(*seen_queries.lock().unwrap(), vec![expected_query]); +} + +#[tokio::test] +async fn search_service_propagates_provider_errors() { + let (provider, seen_queries) = provider("broken", ProviderOutcome::Failure("network timeout")); + let expected_query = query(); + let service = SearchService::new(Box::new(provider)); + + let error = service + .search(expected_query.clone()) + .await + .expect_err("provider error should propagate"); + + match error { + SearchError::Provider(message) => assert_eq!(message, "network timeout"), + other => panic!("expected provider error, got {other}"), + } + assert_eq!(*seen_queries.lock().unwrap(), vec![expected_query]); +} + +#[tokio::test] +async fn search_service_preserves_result_count() { + let provider_response = SearchResponse { + query: "provider supplied query".to_string(), + provider: "brave".to_string(), + results: vec![ + web_result("First", "https://example.com/1", Some("snippet one")), + web_result("Second", "https://example.com/2", None), + ], + total_estimated: Some(2), + next_page: None, + }; + let (provider, _) = provider("brave", ProviderOutcome::Success(provider_response)); + let service = SearchService::new(Box::new(provider)); + + let result = service.search(query()).await.unwrap(); + + assert_eq!(result.results.len(), 2); + assert_eq!(result.total_estimated, Some(2)); +} + +#[tokio::test] +async fn fanout_service_aggregates_multiple_providers() { + let (brave, _) = provider( + "brave", + ProviderOutcome::Success(response("brave", Some(100))), + ); + let (exa, _) = provider("exa", ProviderOutcome::Success(response("exa", Some(50)))); + let service = FanoutSearchService::new(vec![Box::new(brave), Box::new(exa)]); + + let batch = service.search_all(query()).await; + + assert_eq!(batch.query, "distributed systems"); + assert_eq!(batch.responses.len(), 2); + assert!(batch.failures.is_empty()); + assert_eq!(batch.responses[0].provider, "brave"); + assert_eq!(batch.responses[0].total_estimated, Some(100)); + assert_eq!(batch.responses[1].provider, "exa"); + assert_eq!(batch.responses[1].total_estimated, Some(50)); +} + +#[tokio::test] +async fn fanout_service_records_failures_without_short_circuiting() { + let (working, _) = provider( + "working", + ProviderOutcome::Success(response("working", None)), + ); + let (broken, _) = provider("broken", ProviderOutcome::Failure("network timeout")); + let service = FanoutSearchService::new(vec![Box::new(working), Box::new(broken)]); + + let batch = service.search_all(query()).await; + + assert_eq!(batch.responses.len(), 1); + assert_eq!(batch.failures.len(), 1); + assert_eq!(batch.responses[0].provider, "working"); + assert_eq!(batch.failures[0].provider, "broken"); +} + +#[tokio::test] +async fn fanout_service_preserves_stable_order() { + let providers: Vec> = ["alpha", "beta", "gamma"] + .into_iter() + .map(|id| { + let (provider, _) = provider(id, ProviderOutcome::Success(response(id, None))); + Box::new(provider) as Box + }) + .collect(); + let service = FanoutSearchService::new(providers); + + let batch = service.search_all(query()).await; + let ids: Vec<&str> = batch + .responses + .iter() + .map(|response| response.provider.as_str()) + .collect(); + + assert_eq!(ids, vec!["alpha", "beta", "gamma"]); +}