diff --git a/README.md b/README.md index 7a74241..6bd590c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Search CLI — Web Search for AI Agents -**One binary, 13 providers, 13 modes, rank-fused results. The web search tool your AI agent is missing.** +**One binary, 14 providers, 13 modes, rank-fused results. The web search tool your AI agent is missing.**
@@ -19,7 +19,7 @@ --- -A single Rust binary that aggregates Brave, Serper, Exa, Linkup, Jina, Firecrawl, Tavily, SerpApi, Perplexity, Parallel, xAI, and more into one search interface. Built for AI agents from day one: structured JSON, semantic exit codes, self-describing `agent-info`, reciprocal rank fusion across providers, and a `usage` command that reports remaining API credits. +A single Rust binary that aggregates Brave, Serper, Exa, Linkup, Jina, Firecrawl, Tavily, SerpApi, Perplexity, Parallel, You.com, xAI, and more into one search interface. Built for AI agents from day one: structured JSON, semantic exit codes, self-describing `agent-info`, reciprocal rank fusion across providers, and a `usage` command that reports remaining API credits. [Install](#install) | [How It Works](#how-it-works) | [Features](#features) | [Providers](#providers) | [Contributing](#contributing) @@ -125,11 +125,11 @@ same routing registry the engine uses. | Mode | Use when | `-q` is | Providers used | |------|----------|---------|----------------| -| `general` | Any web lookup not covered below (default) | query | Parallel + Brave + Serper + Exa + Jina + Linkup + Tavily + Perplexity | -| `news` | Current events; add `-f day`/`-f week` | query | Parallel + Brave + Serper + Linkup + Tavily + Perplexity (news endpoints) | +| `general` | Any web lookup not covered below (default) | query | Parallel + Brave + Serper + Exa + Jina + Linkup + Tavily + You.com + Perplexity | +| `news` | Current events; add `-f day`/`-f week` | query | Parallel + Brave + Serper + Linkup + Tavily + You.com + Perplexity (news endpoints) | | `academic` | Papers/studies by topic (semantic + web) | query | Exa + Serper + Tavily + Perplexity | | `scholar` | Google Scholar records: citations, PDFs | query | Serper + SerpApi | -| `deep` | Max coverage; waits for all providers — use `-c 30` | query | Parallel + Brave (web + LLM Context) + Serper + Exa + Linkup + Tavily + Perplexity + xAI | +| `deep` | Max coverage; waits for all providers — use `-c 30` | query | Parallel + Brave (web + LLM Context) + Serper + Exa + Linkup + Tavily + You.com + Perplexity + xAI | | `people` | A person, their role, LinkedIn profile | query | Exa | | `social` | What's being said on X/Twitter | query | xAI (Grok) | | `patents` | Prior art, patent families | query | Serper | @@ -230,6 +230,7 @@ search "query" 2>/dev/null # suppress diagnostics | **[Tavily](https://tavily.com/)** | General + deep search, research-focused | Broad coverage, research queries | | **[SerpApi](https://serpapi.com/)** | Many engines: Google, Bing, YouTube, Baidu | Multi-engine coverage; only provider with a real balance API | | **[Perplexity](https://perplexity.ai/)** | LLM-synthesized answer with citations (Sonar) | When you want an answer with sources, not raw pages | +| **You.com** | Web + news search with freshness controls | Fresh web research, news queries | | **Browserless** | Cloud browser for Cloudflare/JS-heavy pages | Anti-bot bypass, pages that need a real browser | | **Stealth** | Built-in anti-bot scraper | Protected pages, no API key needed | | **[xAI](https://x.ai/)** | Only provider with native real-time X/Twitter search (Grok) | Live social signal, trending topics, account activity | @@ -303,11 +304,12 @@ export BRAVE_API_KEY=your-key export SERPER_API_KEY=your-key export EXA_API_KEY=your-key # ...also PERPLEXITY_API_KEY, JINA_API_KEY, LINKUP_API_KEY, FIRECRAWL_API_KEY, -# TAVILY_API_KEY, SERPAPI_API_KEY, BROWSERLESS_API_KEY, XAI_API_KEY, +# TAVILY_API_KEY, SERPAPI_API_KEY, YOUCOM_API_KEY, BROWSERLESS_API_KEY, XAI_API_KEY, # PARALLEL_API_KEY # Or the SEARCH_KEYS_ prefixed form: export SEARCH_KEYS_BRAVE=your-key +export SEARCH_KEYS_YOUCOM=your-key ``` Precedence (highest first): `_API_KEY` env → `SEARCH_KEYS_*` env → config file. diff --git a/src/cli.rs b/src/cli.rs index 451ffae..e114280 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -6,7 +6,7 @@ use clap::{Parser, Subcommand}; name = "search", version, about = "Agent-friendly multi-provider search CLI", - long_about = "Aggregates 13 search providers across 13 explicit search modes.\n\ + long_about = "Aggregates 14 search providers across 13 explicit search modes.\n\ You choose the mode (-m) and/or providers (-p); the CLI does not guess\n\ intent. Run `search agent-info` for the machine-readable capability map.\n\ Outputs colored tables for humans, JSON when piped to other tools.\n\n\ @@ -22,6 +22,7 @@ use clap::{Parser, Subcommand}; serpapi Many engines (Google, Bing, YouTube, Baidu, Scholar)\n \ perplexity LLM-synthesized answer with citations (Sonar)\n \ browserless Cloud browser for Cloudflare/JS-heavy pages\n \ + youcom You.com web search with live news results\n \ stealth Local anti-bot scraper (no API key)\n \ xai Only real-time X/Twitter search (Grok agentic)\n\n\ EXAMPLES:\n \ @@ -32,6 +33,7 @@ use clap::{Parser, Subcommand}; search search -q \"trending on twitter\" -m social # X/Twitter search\n \ search search -q \"query\" -p exa # force Exa only\n \ search search -q \"query\" -p exa,brave # only Exa + Brave\n \ + search search -q \"query\" -p youcom # force You.com only\n \ search --x \"AI agents\" # search X (Twitter) only\n \ search \"query\" --json | jq '.results[].url' # pipe JSON to jq" )] diff --git a/src/config.rs b/src/config.rs index 0be50b1..295ed5c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -38,6 +38,8 @@ pub struct ApiKeys { #[serde(default)] pub browserless: String, #[serde(default)] + pub youcom: String, + #[serde(default)] pub xai: String, } @@ -71,6 +73,7 @@ impl Default for AppConfig { serpapi: String::new(), perplexity: String::new(), browserless: String::new(), + youcom: String::new(), xai: String::new(), }, settings: Settings { @@ -135,6 +138,7 @@ pub const PROVIDER_KEYS: &[&str] = &[ "serpapi", "perplexity", "browserless", + "youcom", "xai", ]; @@ -191,6 +195,7 @@ pub fn config_show(config: &AppConfig) { &config.keys.browserless, "BROWSERLESS_API_KEY", ), + ("youcom", &config.keys.youcom, "YOUCOM_API_KEY"), ("xai", &config.keys.xai, "XAI_API_KEY"), ]; @@ -406,6 +411,12 @@ pub fn config_check(config: &AppConfig) { "BROWSERLESS_API_KEY", "Cloud browser for Cloudflare/JS-heavy pages", ), + ( + "youcom", + &config.keys.youcom, + "YOUCOM_API_KEY", + "You.com web search with live news results", + ), ( "xai", &config.keys.xai, diff --git a/src/main.rs b/src/main.rs index 4fcb219..9c012fa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -94,6 +94,7 @@ async fn main() { "api.exa.ai:443", "api.jina.ai:443", "api.tavily.com:443", + "ydc-index.io:443", "api.perplexity.ai:443", ]; for domain in domains { @@ -191,6 +192,7 @@ async fn main() { "https://api.search.brave.com/res/v1/web/search", "https://google.serper.dev/search", "https://api.exa.ai/search", + "https://ydc-index.io/v1/search", ]; for url in urls { let _ = app_c.client.head(url).send().await; diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 2eb428a..8e540b0 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -12,6 +12,7 @@ pub mod serper; pub mod stealth; pub mod tavily; pub mod xai; +pub mod youcom; use crate::context::AppContext; use crate::errors::SearchError; @@ -192,6 +193,7 @@ pub fn build_providers(ctx: &Arc) -> Vec> { Box::new(stealth::Stealth::new(ctx.clone())), Box::new(firecrawl::Firecrawl::new(ctx.clone())), Box::new(tavily::Tavily::new(ctx.clone())), + Box::new(youcom::YouCom::new(ctx.clone())), Box::new(browserless::Browserless::new(ctx.clone())), Box::new(perplexity::Perplexity::new(ctx.clone())), Box::new(serpapi::SerpApi::new(ctx.clone())), diff --git a/src/providers/youcom.rs b/src/providers/youcom.rs new file mode 100644 index 0000000..c551b4a --- /dev/null +++ b/src/providers/youcom.rs @@ -0,0 +1,272 @@ +use crate::context::AppContext; +use crate::errors::SearchError; +use crate::types::{SearchOpts, SearchResult}; +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::json; +use std::sync::Arc; +use std::time::Duration; + +pub struct YouCom { + ctx: Arc, +} + +impl YouCom { + pub fn new(ctx: Arc) -> Self { + Self { ctx } + } + + fn api_key(&self) -> String { + super::resolve_key(&self.ctx.config.keys.youcom, "YOUCOM_API_KEY") + } + + fn search_url(&self) -> &'static str { + "https://ydc-index.io/v1/search" + } + + fn build_query(&self, query: &str, opts: &SearchOpts) -> String { + let mut q = query.to_string(); + for domain in &opts.include_domains { + q = format!("{q} site:{domain}"); + } + for domain in &opts.exclude_domains { + q = format!("{q} -site:{domain}"); + } + q + } + + async fn query( + &self, + query: &str, + count: usize, + opts: &SearchOpts, + ) -> Result { + if self.api_key().is_empty() { + return Err(SearchError::AuthMissing { provider: "youcom" }); + } + + let client = &self.ctx.client; + let api_key = self.api_key(); + let q = self.build_query(query, opts); + let count_str = count.to_string(); + + super::retry_request(|| async { + let mut req = client + .get(self.search_url()) + .header("X-API-Key", api_key.as_str()) + .header("Accept", "application/json") + .query(&[("query", q.as_str()), ("count", count_str.as_str())]); + + if let Some(freshness) = opts.freshness.as_deref() { + req = req.query(&[("freshness", freshness)]); + } + + let resp = req.send().await?; + let status = resp.status(); + if status == reqwest::StatusCode::TOO_MANY_REQUESTS { + return Err(SearchError::RateLimited { provider: "youcom" }); + } + if !status.is_success() { + return Err(SearchError::Api { + provider: "youcom", + code: "api_error", + message: format!("HTTP {status}"), + status: Some(status.as_u16()), + }); + } + + Ok(resp.json::().await?) + }) + .await + } + + fn map_results(items: &[YouComResult], source: &str) -> Vec { + items + .iter() + .map(|item| { + let snippet = item + .snippets + .as_ref() + .filter(|snippets| !snippets.is_empty()) + .map(|snippets| snippets.join(" ... ")) + .or_else(|| item.description.clone()) + .unwrap_or_default(); + + SearchResult { + title: item.title.clone().unwrap_or_default(), + url: item.url.clone().unwrap_or_default(), + snippet, + source: source.to_string(), + published: item.page_age.clone(), + image_url: None, + extra: item + .favicon_url + .as_ref() + .map(|favicon_url| json!({ "favicon_url": favicon_url })), + } + }) + .collect() + } + + fn merge_sections(response: YouComResponse, news_only: bool) -> Vec { + let mut results = Vec::new(); + if let Some(results_by_type) = response.results { + if news_only { + if !results_by_type.news.is_empty() { + results.extend(Self::map_results(&results_by_type.news, "youcom_news")); + } else { + results.extend(Self::map_results(&results_by_type.web, "youcom_web")); + } + } else { + results.extend(Self::map_results(&results_by_type.web, "youcom_web")); + results.extend(Self::map_results(&results_by_type.news, "youcom_news")); + } + } + results + } +} + +#[derive(Debug, Deserialize)] +struct YouComResponse { + #[serde(default)] + results: Option, +} + +#[derive(Debug, Deserialize, Default)] +struct YouComSections { + #[serde(default)] + web: Vec, + #[serde(default)] + news: Vec, +} + +#[derive(Debug, Deserialize, Clone)] +struct YouComResult { + #[serde(default)] + title: Option, + #[serde(default)] + url: Option, + #[serde(default)] + description: Option, + #[serde(default)] + snippets: Option>, + #[serde(default, rename = "page_age")] + page_age: Option, + #[serde(default)] + favicon_url: Option, +} + +#[async_trait] +impl super::Provider for YouCom { + fn name(&self) -> &'static str { + "youcom" + } + + fn capabilities(&self) -> &[&'static str] { + &["general", "news", "deep"] + } + + fn env_keys(&self) -> &[&'static str] { + &["YOUCOM_API_KEY", "SEARCH_KEYS_YOUCOM"] + } + + fn is_configured(&self) -> bool { + !self.api_key().is_empty() + } + + fn timeout(&self) -> Duration { + Duration::from_secs(15) + } + + async fn search( + &self, + query: &str, + count: usize, + opts: &SearchOpts, + ) -> Result, SearchError> { + let response = self.query(query, count, opts).await?; + Ok(Self::merge_sections(response, false)) + } + + async fn search_news( + &self, + query: &str, + count: usize, + opts: &SearchOpts, + ) -> Result, SearchError> { + let mut news_opts = opts.clone(); + if news_opts.freshness.is_none() { + news_opts.freshness = Some("day".to_string()); + } + + let response = self.query(query, count, &news_opts).await?; + Ok(Self::merge_sections(response, true)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn merges_web_and_news_results() { + let response: YouComResponse = serde_json::from_str( + r#"{ + "results": { + "web": [ + { + "title": "Web Result", + "url": "https://example.com/web", + "description": "Web snippet", + "snippets": ["one", "two"], + "page_age": "2026-07-20T00:00:00" + } + ], + "news": [ + { + "title": "News Result", + "url": "https://example.com/news", + "description": "News snippet" + } + ] + } + }"#, + ) + .unwrap(); + + let merged = YouCom::merge_sections(response, false); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].title, "Web Result"); + assert_eq!(merged[0].snippet, "one ... two"); + assert_eq!(merged[0].published.as_deref(), Some("2026-07-20T00:00:00")); + assert_eq!(merged[1].source, "youcom_news"); + } + + #[test] + fn prefers_news_for_news_mode() { + let response: YouComResponse = serde_json::from_str( + r#"{ + "results": { + "web": [ + { + "title": "Web Result", + "url": "https://example.com/web" + } + ], + "news": [ + { + "title": "News Result", + "url": "https://example.com/news" + } + ] + } + }"#, + ) + .unwrap(); + + let merged = YouCom::merge_sections(response, true); + assert_eq!(merged.len(), 1); + assert_eq!(merged[0].title, "News Result"); + assert_eq!(merged[0].source, "youcom_news"); + } +} diff --git a/src/registry.rs b/src/registry.rs index 01fd765..040f21f 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -70,6 +70,7 @@ pub const MODES: &[ModeSpec] = &[ "jina", "linkup", "tavily", + "youcom", "perplexity", ], description: "Broad web search fused across all general-capable providers", @@ -79,7 +80,15 @@ pub const MODES: &[ModeSpec] = &[ mode: Mode::News, input: InputKind::Query, merge: MergeKind::Fused, - providers: &["parallel", "brave", "serper", "linkup", "tavily", "perplexity"], + providers: &[ + "parallel", + "brave", + "serper", + "linkup", + "tavily", + "youcom", + "perplexity", + ], description: "News-specific endpoints of the news-capable providers", when_to_use: "Current events and anything where recency dominates; combine with -f day or -f week.", }, @@ -110,6 +119,7 @@ pub const MODES: &[ModeSpec] = &[ "exa", "linkup", "tavily", + "youcom", "perplexity", "xai", ], @@ -256,6 +266,7 @@ mod tests { "linkup", "firecrawl", "tavily", + "youcom", "serpapi", "perplexity", "browserless", diff --git a/tests/integration.rs b/tests/integration.rs index 8d96fa7..3eb04ed 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -34,7 +34,7 @@ fn test_help_output() { .arg("--help") .assert() .success() - .stdout(predicate::str::contains("Aggregates 13 search providers")) + .stdout(predicate::str::contains("Aggregates 14 search providers")) .stdout(predicate::str::contains("brave")) .stdout(predicate::str::contains("serper")) .stdout(predicate::str::contains("exa")); @@ -92,6 +92,7 @@ fn test_providers_json() { assert!(names.contains(&"serper")); assert!(names.contains(&"exa")); assert!(names.contains(&"jina")); + assert!(names.contains(&"youcom")); assert!(names.contains(&"firecrawl")); assert!(names.contains(&"tavily")); } @@ -106,6 +107,7 @@ fn test_config_check() { .stdout(predicate::str::contains("serper")) .stdout(predicate::str::contains("exa")) .stdout(predicate::str::contains("jina")) + .stdout(predicate::str::contains("youcom")) .stdout(predicate::str::contains("firecrawl")) .stdout(predicate::str::contains("tavily")); }